1berry/wokada-voice-changer
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EXP. remove microphone stream 2

Browse Source
This commit is contained in:
wataru 2023-02-20 02:21:51 +09:00
parent de0be188ff
commit bde893b665
12 changed files with 518 additions and 478 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/demo/dist/index.js vendored
View File

File diff suppressed because one or more lines are too long

8
client/demo/src/106_convert_setting.tsx
View File

@ -25,18 +25,18 @@ export const useConvertSetting = (): ConvertSettingState => {
<div className="body-row split-3-2-1-4 left-padding-1 guided"> <div className="body-row split-3-2-1-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">Input Chunk Num(128sample/chunk)</div> <div className="body-item-title left-padding-1">Input Chunk Num(128sample/chunk)</div>
<div className="body-input-container"> <div className="body-input-container">
<input type="number" min={1} max={256} step={1} value={appState.streamerSetting.audioStreamerSetting.inputChunkNum} onChange={(e) => { <input type="number" min={1} max={256} step={1} value={appState.workletNodeSetting.workletNodeSetting.inputChunkNum} onChange={(e) => {
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, inputChunkNum: Number(e.target.value) }) appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, inputChunkNum: Number(e.target.value) })
}} /> }} />
</div> </div>
<div className="body-item-text"> <div className="body-item-text">
<div>buff: {(appState.streamerSetting.audioStreamerSetting.inputChunkNum * 128 * 1000 / 24000).toFixed(1)}ms</div> <div>buff: {(appState.workletNodeSetting.workletNodeSetting.inputChunkNum * 128 * 1000 / 48000).toFixed(1)}ms</div>
</div> </div>
<div className="body-item-text"></div> <div className="body-item-text"></div>
</div> </div>
) )
}, [appState.streamerSetting.audioStreamerSetting.inputChunkNum, appState.streamerSetting.updateAudioStreamerSetting]) }, [appState.workletNodeSetting.workletNodeSetting.inputChunkNum, appState.workletNodeSetting.updateWorkletNodeSetting])
const gpuRow = useMemo(() => { const gpuRow = useMemo(() => {
return ( return (

22
client/demo/src/107_advanced_setting.tsx
View File

@ -30,24 +30,24 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
<div className="body-row split-3-3-4 left-padding-1 guided"> <div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">MMVC Server</div> <div className="body-item-title left-padding-1">MMVC Server</div>
<div className="body-input-container"> <div className="body-input-container">
<input type="text" defaultValue={appState.streamerSetting.audioStreamerSetting.serverUrl} id="mmvc-server-url" className="body-item-input" /> <input type="text" defaultValue={appState.workletNodeSetting.workletNodeSetting.serverUrl} id="mmvc-server-url" className="body-item-input" />
</div> </div>
<div className="body-button-container"> <div className="body-button-container">
<div className="body-button" onClick={onSetServerClicked}>set</div> <div className="body-button" onClick={onSetServerClicked}>set</div>
</div> </div>
</div> </div>
) )
}, [appState.streamerSetting.audioStreamerSetting.serverUrl, appState.clientSetting.setServerUrl]) }, [appState.workletNodeSetting.workletNodeSetting.serverUrl, appState.clientSetting.setServerUrl])
const protocolRow = useMemo(() => { const protocolRow = useMemo(() => {
const onProtocolChanged = async (val: Protocol) => { const onProtocolChanged = async (val: Protocol) => {
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, protocol: val }) appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, protocol: val })
} }
return ( return (
<div className="body-row split-3-7 left-padding-1 guided"> <div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Protocol</div> <div className="body-item-title left-padding-1">Protocol</div>
<div className="body-select-container"> <div className="body-select-container">
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.protocol} onChange={(e) => { <select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.protocol} onChange={(e) => {
onProtocolChanged(e.target.value as onProtocolChanged(e.target.value as
Protocol) Protocol)
}}> }}>
@ -60,7 +60,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
</div> </div>
</div> </div>
) )
}, [appState.streamerSetting.audioStreamerSetting.protocol, appState.streamerSetting.updateAudioStreamerSetting]) }, [appState.workletNodeSetting.workletNodeSetting.protocol, appState.workletNodeSetting.updateWorkletNodeSetting])
const sampleRateRow = useMemo(() => { const sampleRateRow = useMemo(() => {
@ -87,8 +87,8 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
<div className="body-row split-3-7 left-padding-1 guided"> <div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Sending Sample Rate</div> <div className="body-item-title left-padding-1">Sending Sample Rate</div>
<div className="body-select-container"> <div className="body-select-container">
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.sendingSampleRate} onChange={(e) => { <select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.sendingSampleRate} onChange={(e) => {
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, sendingSampleRate: Number(e.target.value) as InputSampleRate }) appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, sendingSampleRate: Number(e.target.value) as InputSampleRate })
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, inputSampleRate: Number(e.target.value) as InputSampleRate }) appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, inputSampleRate: Number(e.target.value) as InputSampleRate })
}}> }}>
{ {
@ -100,7 +100,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
</div> </div>
</div> </div>
) )
}, [appState.streamerSetting.audioStreamerSetting.sendingSampleRate, appState.streamerSetting.updateAudioStreamerSetting, appState.serverSetting.updateServerSettings]) }, [appState.workletNodeSetting.workletNodeSetting.sendingSampleRate, appState.workletNodeSetting.updateWorkletNodeSetting, appState.serverSetting.updateServerSettings])
const bufferSizeRow = useMemo(() => { const bufferSizeRow = useMemo(() => {
return ( return (
@ -174,8 +174,8 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
<div className="body-row split-3-7 left-padding-1 guided"> <div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1 ">DownSamplingMode</div> <div className="body-item-title left-padding-1 ">DownSamplingMode</div>
<div className="body-select-container"> <div className="body-select-container">
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.downSamplingMode} onChange={(e) => { <select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.downSamplingMode} onChange={(e) => {
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, downSamplingMode: e.target.value as DownSamplingMode }) appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, downSamplingMode: e.target.value as DownSamplingMode })
}}> }}>
{ {
Object.values(DownSamplingMode).map(x => { Object.values(DownSamplingMode).map(x => {
@ -186,7 +186,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
</div> </div>
</div> </div>
) )
}, [appState.streamerSetting.audioStreamerSetting.downSamplingMode, appState.streamerSetting.updateAudioStreamerSetting]) }, [appState.workletNodeSetting.workletNodeSetting.downSamplingMode, appState.workletNodeSetting.updateWorkletNodeSetting])
const workletSettingRow = useMemo(() => { const workletSettingRow = useMemo(() => {

424
client/lib/src/AudioStreamer.ts
View File

@ -1,243 +1,243 @@
import { io, Socket } from "socket.io-client"; // import { io, Socket } from "socket.io-client";
import { DefaultEventsMap } from "@socket.io/component-emitter"; // import { DefaultEventsMap } from "@socket.io/component-emitter";
import { Duplex, DuplexOptions } from "readable-stream"; // import { Duplex, DuplexOptions } from "readable-stream";
import { AudioStreamerSetting, DefaultAudioStreamerSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const"; // import { AudioStreamerSetting, DefaultAudioStreamerSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
export type Callbacks = { // export type Callbacks = {
onVoiceReceived: (data: ArrayBuffer) => void // onVoiceReceived: (data: ArrayBuffer) => void
} // }
export type AudioStreamerListeners = { // export type AudioStreamerListeners = {
notifySendBufferingTime: (time: number) => void // notifySendBufferingTime: (time: number) => void
notifyResponseTime: (time: number) => void // notifyResponseTime: (time: number) => void
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void // notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
} // }
export class AudioStreamer extends Duplex { // export class AudioStreamer extends Duplex {
private setting: AudioStreamerSetting = DefaultAudioStreamerSetting // private setting: AudioStreamerSetting = DefaultAudioStreamerSetting
private callbacks: Callbacks // private callbacks: Callbacks
private audioStreamerListeners: AudioStreamerListeners // private audioStreamerListeners: AudioStreamerListeners
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null // private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
private requestChunks: ArrayBuffer[] = [] // private requestChunks: ArrayBuffer[] = []
// performance monitor // // performance monitor
private bufferStart = 0; // private bufferStart = 0;
constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) { // constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
super(options); // super(options);
this.callbacks = callbacks // this.callbacks = callbacks
this.audioStreamerListeners = audioStreamerListeners // this.audioStreamerListeners = audioStreamerListeners
this.createSocketIO() // this.createSocketIO()
} // }
private createSocketIO = () => { // private createSocketIO = () => {
if (this.socket) { // if (this.socket) {
this.socket.close() // this.socket.close()
} // }
if (this.setting.protocol === "sio") { // if (this.setting.protocol === "sio") {
this.socket = io(this.setting.serverUrl + "/test"); // this.socket = io(this.setting.serverUrl + "/test");
this.socket.on('connect_error', (err) => { // this.socket.on('connect_error', (err) => {
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`) // this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
}) // })
this.socket.on('connect', () => { // this.socket.on('connect', () => {
console.log(`[SIO] sonnect to ${this.setting.serverUrl}`) // console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
console.log(`[SIO] ${this.socket?.id}`) // console.log(`[SIO] ${this.socket?.id}`)
}); // });
this.socket.on('response', (response: any[]) => { // this.socket.on('response', (response: any[]) => {
const cur = Date.now() // const cur = Date.now()
const responseTime = cur - response[0] // const responseTime = cur - response[0]
const result = response[1] as ArrayBuffer // const result = response[1] as ArrayBuffer
if (result.byteLength < 128 * 2) { // if (result.byteLength < 128 * 2) {
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`) // this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
} else { // } else {
this.callbacks.onVoiceReceived(response[1]) // this.callbacks.onVoiceReceived(response[1])
this.audioStreamerListeners.notifyResponseTime(responseTime) // this.audioStreamerListeners.notifyResponseTime(responseTime)
} // }
}); // });
} // }
} // }
// Option Change // // Option Change
updateSetting = (setting: AudioStreamerSetting) => { // updateSetting = (setting: AudioStreamerSetting) => {
console.log(`[AudioStreamer] Updating AudioStreamer Setting,`, this.setting, setting) // console.log(`[AudioStreamer] Updating AudioStreamer Setting,`, this.setting, setting)
let recreateSocketIoRequired = false // let recreateSocketIoRequired = false
if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) { // if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
recreateSocketIoRequired = true // recreateSocketIoRequired = true
} // }
this.setting = setting // this.setting = setting
if (recreateSocketIoRequired) { // if (recreateSocketIoRequired) {
this.createSocketIO() // this.createSocketIO()
} // }
} // }
getSettings = (): AudioStreamerSetting => { // getSettings = (): AudioStreamerSetting => {
return this.setting // return this.setting
} // }
getSocketId = () => { // getSocketId = () => {
return this.socket?.id // return this.socket?.id
} // }
// Main Process // // Main Process
//// Pipe from mic stream // //// Pipe from mic stream
_write = (chunk: AudioBuffer, _encoding: any, callback: any) => { // _write = (chunk: AudioBuffer, _encoding: any, callback: any) => {
const buffer = chunk.getChannelData(0); // const buffer = chunk.getChannelData(0);
this._write_realtime(buffer) // this._write_realtime(buffer)
callback(); // callback();
} // }
_averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) { // _averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
if (originalSampleRate == destinationSamplerate) { // if (originalSampleRate == destinationSamplerate) {
return buffer; // return buffer;
} // }
if (destinationSamplerate > originalSampleRate) { // if (destinationSamplerate > originalSampleRate) {
throw "downsampling rate show be smaller than original sample rate"; // throw "downsampling rate show be smaller than original sample rate";
} // }
const sampleRateRatio = originalSampleRate / destinationSamplerate; // const sampleRateRatio = originalSampleRate / destinationSamplerate;
const newLength = Math.round(buffer.length / sampleRateRatio); // const newLength = Math.round(buffer.length / sampleRateRatio);
const result = new Float32Array(newLength); // const result = new Float32Array(newLength);
let offsetResult = 0; // let offsetResult = 0;
let offsetBuffer = 0; // let offsetBuffer = 0;
while (offsetResult < result.length) { // while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio); // var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
// Use average value of skipped samples // // Use average value of skipped samples
var accum = 0, count = 0; // var accum = 0, count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) { // for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
accum += buffer[i]; // accum += buffer[i];
count++; // count++;
} // }
result[offsetResult] = accum / count; // result[offsetResult] = accum / count;
// Or you can simply get rid of the skipped samples: // // Or you can simply get rid of the skipped samples:
// result[offsetResult] = buffer[nextOffsetBuffer]; // // result[offsetResult] = buffer[nextOffsetBuffer];
offsetResult++; // offsetResult++;
offsetBuffer = nextOffsetBuffer; // offsetBuffer = nextOffsetBuffer;
} // }
return result; // return result;
} // }
private _write_realtime = async (buffer: Float32Array) => { // private _write_realtime = async (buffer: Float32Array) => {
let downsampledBuffer: Float32Array | null = null // let downsampledBuffer: Float32Array | null = null
if (this.setting.sendingSampleRate == 48000) { // if (this.setting.sendingSampleRate == 48000) {
downsampledBuffer = buffer // downsampledBuffer = buffer
} else if (this.setting.downSamplingMode == DownSamplingMode.decimate) { // } else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
//////// (Kind 1) 間引き ////////// // //////// (Kind 1) 間引き //////////
// bufferSize個のデータ（48Khz）が入ってくる。 // // bufferSize個のデータ（48Khz）が入ってくる。
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。 // //// 48000Hz で入ってくるので間引いて24000Hzに変換する。
downsampledBuffer = new Float32Array(buffer.length / 2); // downsampledBuffer = new Float32Array(buffer.length / 2);
for (let i = 0; i < buffer.length; i++) { // for (let i = 0; i < buffer.length; i++) {
if (i % 2 == 0) { // if (i % 2 == 0) {
downsampledBuffer[i / 2] = buffer[i] // downsampledBuffer[i / 2] = buffer[i]
} // }
} // }
} else { // } else {
//////// (Kind 2) 平均 ////////// // //////// (Kind 2) 平均 //////////
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000) // // downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, this.setting.sendingSampleRate) // downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, this.setting.sendingSampleRate)
} // }
// Float to signed16 // // Float to signed16
const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2) // const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2)
const dataView = new DataView(arrayBuffer); // const dataView = new DataView(arrayBuffer);
for (let i = 0; i < downsampledBuffer.length; i++) { // for (let i = 0; i < downsampledBuffer.length; i++) {
let s = Math.max(-1, Math.min(1, downsampledBuffer[i])); // let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
s = s < 0 ? s * 0x8000 : s * 0x7FFF // s = s < 0 ? s * 0x8000 : s * 0x7FFF
dataView.setInt16(i * 2, s, true); // dataView.setInt16(i * 2, s, true);
} // }
// 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理 // // 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理
// const chunkByteSize = 256 // (const.ts ★1) // // const chunkByteSize = 256 // (const.ts ★1)
// const chunkByteSize = 256 * 2 // (const.ts ★1) // // const chunkByteSize = 256 * 2 // (const.ts ★1)
const chunkByteSize = (256 * 2) * (this.setting.sendingSampleRate / 48000) // (const.ts ★1) // const chunkByteSize = (256 * 2) * (this.setting.sendingSampleRate / 48000) // (const.ts ★1)
for (let i = 0; i < arrayBuffer.byteLength / chunkByteSize; i++) { // for (let i = 0; i < arrayBuffer.byteLength / chunkByteSize; i++) {
const ab = arrayBuffer.slice(i * chunkByteSize, (i + 1) * chunkByteSize) // const ab = arrayBuffer.slice(i * chunkByteSize, (i + 1) * chunkByteSize)
this.requestChunks.push(ab) // this.requestChunks.push(ab)
} // }
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。 // //// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
if (this.requestChunks.length < this.setting.inputChunkNum) { // if (this.requestChunks.length < this.setting.inputChunkNum) {
return // return
} // }
// リクエスト用の入れ物を作成 // // リクエスト用の入れ物を作成
const windowByteLength = this.requestChunks.reduce((prev, cur) => { // const windowByteLength = this.requestChunks.reduce((prev, cur) => {
return prev + cur.byteLength // return prev + cur.byteLength
}, 0) // }, 0)
const newBuffer = new Uint8Array(windowByteLength); // const newBuffer = new Uint8Array(windowByteLength);
// リクエストのデータをセット // // リクエストのデータをセット
this.requestChunks.reduce((prev, cur) => { // this.requestChunks.reduce((prev, cur) => {
newBuffer.set(new Uint8Array(cur), prev) // newBuffer.set(new Uint8Array(cur), prev)
return prev + cur.byteLength // return prev + cur.byteLength
}, 0) // }, 0)
// console.log("send buff length", newBuffer.length) // // console.log("send buff length", newBuffer.length)
this.sendBuffer(newBuffer) // this.sendBuffer(newBuffer)
this.requestChunks = [] // this.requestChunks = []
this.audioStreamerListeners.notifySendBufferingTime(Date.now() - this.bufferStart) // this.audioStreamerListeners.notifySendBufferingTime(Date.now() - this.bufferStart)
this.bufferStart = Date.now() // this.bufferStart = Date.now()
} // }
private sendBuffer = async (newBuffer: Uint8Array) => { // private sendBuffer = async (newBuffer: Uint8Array) => {
const timestamp = Date.now() // const timestamp = Date.now()
if (this.setting.protocol === "sio") { // if (this.setting.protocol === "sio") {
if (!this.socket) { // if (!this.socket) {
console.warn(`sio is not initialized`) // console.warn(`sio is not initialized`)
return // return
} // }
// console.log("emit!") // // console.log("emit!")
this.socket.emit('request_message', [ // this.socket.emit('request_message', [
timestamp, // timestamp,
newBuffer.buffer]); // newBuffer.buffer]);
} else { // } else {
const res = await postVoice( // const res = await postVoice(
this.setting.serverUrl + "/test", // this.setting.serverUrl + "/test",
timestamp, // timestamp,
newBuffer.buffer) // newBuffer.buffer)
if (res.byteLength < 128 * 2) { // if (res.byteLength < 128 * 2) {
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`) // this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
} else { // } else {
this.callbacks.onVoiceReceived(res) // this.callbacks.onVoiceReceived(res)
this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp) // this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
} // }
} // }
} // }
} // }
export const postVoice = async ( // export const postVoice = async (
url: string, // url: string,
timestamp: number, // timestamp: number,
buffer: ArrayBuffer) => { // buffer: ArrayBuffer) => {
const obj = { // const obj = {
timestamp, // timestamp,
buffer: Buffer.from(buffer).toString('base64') // buffer: Buffer.from(buffer).toString('base64')
}; // };
const body = JSON.stringify(obj); // const body = JSON.stringify(obj);
const res = await fetch(`${url}`, { // const res = await fetch(`${url}`, {
method: "POST", // method: "POST",
headers: { // headers: {
'Accept': 'application/json', // 'Accept': 'application/json',
'Content-Type': 'application/json' // 'Content-Type': 'application/json'
}, // },
body: body // body: body
}) // })
const receivedJson = await res.json() // const receivedJson = await res.json()
const changedVoiceBase64 = receivedJson["changedVoiceBase64"] // const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
const buf = Buffer.from(changedVoiceBase64, "base64") // const buf = Buffer.from(changedVoiceBase64, "base64")
const ab = new ArrayBuffer(buf.length); // const ab = new ArrayBuffer(buf.length);
const view = new Uint8Array(ab); // const view = new Uint8Array(ab);
for (let i = 0; i < buf.length; ++i) { // for (let i = 0; i < buf.length; ++i) {
view[i] = buf[i]; // view[i] = buf[i];
} // }
return ab // return ab
} // }

89
client/lib/src/VoiceChangerClient.ts
View File

@ -3,14 +3,12 @@ import { VoiceChangerWorkletNode, VoiceChangerWorkletListener } from "./VoiceCha
import workerjs from "raw-loader!../worklet/dist/index.js"; import workerjs from "raw-loader!../worklet/dist/index.js";
import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js"; import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
import { createDummyMediaStream, validateUrl } from "./util"; import { createDummyMediaStream, validateUrl } from "./util";
import { AudioStreamerSetting, DefaultVoiceChangerClientSetting, ServerSettingKey, VoiceChangerClientSetting, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletSetting } from "./const"; import { DefaultVoiceChangerClientSetting, ServerSettingKey, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "./const";
import MicrophoneStream from "microphone-stream";
import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer";
import { ServerConfigurator } from "./ServerConfigurator"; import { ServerConfigurator } from "./ServerConfigurator";
// オーディオデータの流れ // オーディオデータの流れ
// input node(mic or MediaStream) -> [vf node] -> microphne stream -> audio streamer -> // input node(mic or MediaStream) -> [vf node] -> [vc node] ->
// sio/rest server -> audio streamer-> vc node -> output node // sio/rest server -> [vc node] -> output node
import { BlockingQueue } from "./utils/BlockingQueue"; import { BlockingQueue } from "./utils/BlockingQueue";
@ -23,11 +21,8 @@ export class VoiceChangerClient {
private currentMediaStream: MediaStream | null = null private currentMediaStream: MediaStream | null = null
private currentMediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null private currentMediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null
private outputNodeFromVF: MediaStreamAudioDestinationNode | null = null
private inputGainNode: GainNode | null = null private inputGainNode: GainNode | null = null
private outputGainNode: GainNode | null = null private outputGainNode: GainNode | null = null
private micStream: MicrophoneStream | null = null
private audioStreamer!: AudioStreamer
private vcNode!: VoiceChangerWorkletNode private vcNode!: VoiceChangerWorkletNode
private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode
@ -41,13 +36,7 @@ export class VoiceChangerClient {
private sem = new BlockingQueue<number>(); private sem = new BlockingQueue<number>();
private callbacks: Callbacks = { constructor(ctx: AudioContext, vfEnable: boolean, voiceChangerWorkletListener: VoiceChangerWorkletListener) {
onVoiceReceived: (data: ArrayBuffer): void => {
this.vcNode.postReceivedVoice(data)
}
}
constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners, voiceChangerWorkletListener: VoiceChangerWorkletListener) {
this.sem.enqueue(0); this.sem.enqueue(0);
this.configurator = new ServerConfigurator() this.configurator = new ServerConfigurator()
this.ctx = ctx this.ctx = ctx
@ -62,14 +51,11 @@ export class VoiceChangerClient {
this.outputGainNode.gain.value = this.setting.outputGain this.outputGainNode.gain.value = this.setting.outputGain
this.vcNode.connect(this.outputGainNode) // vc node -> output node this.vcNode.connect(this.outputGainNode) // vc node -> output node
this.outputGainNode.connect(this.currentMediaStreamAudioDestinationNode) this.outputGainNode.connect(this.currentMediaStreamAudioDestinationNode)
// (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる)
this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, })
if (this.vfEnable) { if (this.vfEnable) {
this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' }) this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' })
const dummyMediaStream = createDummyMediaStream(this.ctx) const dummyMediaStream = createDummyMediaStream(this.ctx)
this.currentDevice = (await this.vf.createTransformDevice(dummyMediaStream)) || null; this.currentDevice = (await this.vf.createTransformDevice(dummyMediaStream)) || null;
this.outputNodeFromVF = this.ctx.createMediaStreamDestination();
} }
resolve() resolve()
}) })
@ -94,7 +80,6 @@ export class VoiceChangerClient {
// オペレーション // オペレーション
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
/// Operations /// /// Operations ///
// setup = async (input: string | MediaStream | null, bufferSize: BufferSize, echoCancel: boolean = true, noiseSuppression: boolean = true, noiseSuppression2: boolean = false) => {
setup = async () => { setup = async () => {
const lockNum = await this.lock() const lockNum = await this.lock()
@ -115,9 +100,7 @@ export class VoiceChangerClient {
//// Input デバイスがnullの時はmicStreamを止めてリターン //// Input デバイスがnullの時はmicStreamを止めてリターン
if (!this.setting.audioInput) { if (!this.setting.audioInput) {
console.log(`Input Setup=> client mic is disabled.`) console.log(`Input Setup=> client mic is disabled.`)
if (this.micStream) { this.vcNode.stopRecording()
this.micStream.pauseRecording()
}
await this.unlock(lockNum) await this.unlock(lockNum)
return return
} }
@ -143,17 +126,6 @@ export class VoiceChangerClient {
this.currentMediaStream = this.setting.audioInput this.currentMediaStream = this.setting.audioInput
} }
// create mic stream
if (this.micStream) {
this.micStream.unpipe()
this.micStream.destroy()
this.micStream = null
}
this.micStream = new MicrophoneStream({
objectMode: true,
bufferSize: this.setting.bufferSize,
context: this.ctx
})
// connect nodes. // connect nodes.
this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream) this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream)
this.inputGainNode = this.ctx.createGain() this.inputGainNode = this.ctx.createGain()
@ -163,11 +135,8 @@ export class VoiceChangerClient {
this.currentDevice.chooseNewInnerDevice(this.currentMediaStream) this.currentDevice.chooseNewInnerDevice(this.currentMediaStream)
const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node
this.inputGainNode.connect(voiceFocusNode.start) // input node -> vf node this.inputGainNode.connect(voiceFocusNode.start) // input node -> vf node
voiceFocusNode.end.connect(this.outputNodeFromVF!) voiceFocusNode.end.connect(this.vcNode)
// this.micStream.setStream(this.outputNodeFromVF!.stream) // vf node -> mic stream
} else { } else {
// const inputDestinationNodeForMicStream = this.ctx.createMediaStreamDestination()
// this.inputGainNode.connect(inputDestinationNodeForMicStream)
console.log("input___ media stream", this.currentMediaStream) console.log("input___ media stream", this.currentMediaStream)
this.currentMediaStream.getTracks().forEach(x => { this.currentMediaStream.getTracks().forEach(x => {
console.log("input___ media stream set", x.getSettings()) console.log("input___ media stream set", x.getSettings())
@ -177,17 +146,6 @@ export class VoiceChangerClient {
console.log("input___ media node", this.currentMediaStreamAudioSourceNode) console.log("input___ media node", this.currentMediaStreamAudioSourceNode)
console.log("input___ gain node", this.inputGainNode.channelCount, this.inputGainNode) console.log("input___ gain node", this.inputGainNode.channelCount, this.inputGainNode)
this.inputGainNode.connect(this.vcNode) this.inputGainNode.connect(this.vcNode)
// this.micStream.setStream(inputDestinationNodeForMicStream.stream) // input device -> mic stream
}
this.micStream.pipe(this.audioStreamer) // mic stream -> audio streamer
if (!this._isVoiceChanging) {
this.micStream.pauseRecording()
} else {
this.micStream.playRecording()
} }
console.log("Input Setup=> success") console.log("Input Setup=> success")
await this.unlock(lockNum) await this.unlock(lockNum)
@ -197,18 +155,14 @@ export class VoiceChangerClient {
} }
start = () => { start = () => {
if (!this.micStream) { this.vcNode.startRecording()
throw `Exception:${VOICE_CHANGER_CLIENT_EXCEPTION.ERR_MIC_STREAM_NOT_INITIALIZED}`
return
}
this.micStream.playRecording()
this._isVoiceChanging = true this._isVoiceChanging = true
} }
stop = () => { stop = () => {
if (!this.micStream) { return } this.vcNode.stopRecording()
this.micStream.pauseRecording()
this._isVoiceChanging = false this._isVoiceChanging = false
} }
get isVoiceChanging(): boolean { get isVoiceChanging(): boolean {
return this._isVoiceChanging return this._isVoiceChanging
} }
@ -231,7 +185,7 @@ export class VoiceChangerClient {
} }
} }
} }
this.audioStreamer.updateSetting({ ...this.audioStreamer.getSettings(), serverUrl: url }) this.vcNode.updateSetting({ ...this.vcNode.getSettings(), serverUrl: url })
this.configurator.setServerUrl(url) this.configurator.setServerUrl(url)
} }
@ -260,9 +214,6 @@ export class VoiceChangerClient {
if (reconstructInputRequired) { if (reconstructInputRequired) {
this.setup() this.setup()
} }
} }
setInputGain = (val: number) => { setInputGain = (val: number) => {
@ -301,17 +252,17 @@ export class VoiceChangerClient {
configureWorklet = (setting: WorkletSetting) => { configureWorklet = (setting: WorkletSetting) => {
this.vcNode.configure(setting) this.vcNode.configure(setting)
} }
startOutputRecordingWorklet = () => { startRecording = () => {
this.vcNode.startOutputRecordingWorklet() this.vcNode.startRecording()
} }
stopOutputRecordingWorklet = () => { stopRecording = () => {
this.vcNode.stopOutputRecordingWorklet() this.vcNode.stopRecording()
} }
//## Audio Streamer ##// //## Worklet Node ##//
updateAudioStreamerSetting = (setting: AudioStreamerSetting) => { updateWorkletNodeSetting = (setting: WorkletNodeSetting) => {
this.audioStreamer.updateSetting(setting) this.vcNode.updateSetting(setting)
} }
@ -320,16 +271,14 @@ export class VoiceChangerClient {
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
// Information // Information
getClientSettings = () => { getClientSettings = () => {
return this.audioStreamer.getSettings() return this.vcNode.getSettings()
} }
getServerSettings = () => { getServerSettings = () => {
return this.configurator.getSettings() return this.configurator.getSettings()
} }
getSocketId = () => { getSocketId = () => {
return this.audioStreamer.getSocketId() return this.vcNode.getSocketId()
} }
} }

227
client/lib/src/VoiceChangerWorkletNode.ts
View File

@ -1,17 +1,25 @@
import { VoiceChangerWorkletProcessorRequest } from "./@types/voice-changer-worklet-processor"; import { VoiceChangerWorkletProcessorRequest } from "./@types/voice-changer-worklet-processor";
import { WorkletSetting } from "./const"; import { DefaultWorkletNodeSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletNodeSetting, WorkletSetting } from "./const";
import { io, Socket } from "socket.io-client"; import { io, Socket } from "socket.io-client";
import { DefaultEventsMap } from "@socket.io/component-emitter"; import { DefaultEventsMap } from "@socket.io/component-emitter";
export type VoiceChangerWorkletListener = { export type VoiceChangerWorkletListener = {
notifyVolume: (vol: number) => void notifyVolume: (vol: number) => void
notifyOutputRecordData: (data: Float32Array[]) => void notifyOutputRecordData: (data: Float32Array[]) => void
notifySendBufferingTime: (time: number) => void
notifyResponseTime: (time: number) => void
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
} }
export class VoiceChangerWorkletNode extends AudioWorkletNode { export class VoiceChangerWorkletNode extends AudioWorkletNode {
private listener: VoiceChangerWorkletListener private listener: VoiceChangerWorkletListener
private setting: WorkletNodeSetting = DefaultWorkletNodeSetting
private requestChunks: ArrayBuffer[] = [] private requestChunks: ArrayBuffer[] = []
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
// performance monitor
private bufferStart = 0;
constructor(context: AudioContext, listener: VoiceChangerWorkletListener) { constructor(context: AudioContext, listener: VoiceChangerWorkletListener) {
super(context, "voice-changer-worklet-processor"); super(context, "voice-changer-worklet-processor");
this.port.onmessage = this.handleMessage.bind(this); this.port.onmessage = this.handleMessage.bind(this);
@ -20,7 +28,56 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
console.log(`[worklet_node][voice-changer-worklet-processor] created.`); console.log(`[worklet_node][voice-changer-worklet-processor] created.`);
} }
postReceivedVoice = (data: ArrayBuffer) => { // 設定
updateSetting = (setting: WorkletNodeSetting) => {
console.log(`[WorkletNode] Updating WorkletNode Setting,`, this.setting, setting)
let recreateSocketIoRequired = false
if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
recreateSocketIoRequired = true
}
this.setting = setting
if (recreateSocketIoRequired) {
this.createSocketIO()
}
}
getSettings = (): WorkletNodeSetting => {
return this.setting
}
getSocketId = () => {
return this.socket?.id
}
// 処理
private createSocketIO = () => {
if (this.socket) {
this.socket.close()
}
if (this.setting.protocol === "sio") {
this.socket = io(this.setting.serverUrl + "/test");
this.socket.on('connect_error', (err) => {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
})
this.socket.on('connect', () => {
console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
console.log(`[SIO] ${this.socket?.id}`)
});
this.socket.on('response', (response: any[]) => {
const cur = Date.now()
const responseTime = cur - response[0]
const result = response[1] as ArrayBuffer
if (result.byteLength < 128 * 2) {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
} else {
this.postReceivedVoice(response[1])
this.listener.notifyResponseTime(responseTime)
}
});
}
}
private postReceivedVoice = (data: ArrayBuffer) => {
const req: VoiceChangerWorkletProcessorRequest = { const req: VoiceChangerWorkletProcessorRequest = {
requestType: "voice", requestType: "voice",
voice: data, voice: data,
@ -31,37 +88,33 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.port.postMessage(req) this.port.postMessage(req)
} }
private _averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
private createSocketIO = () => { if (originalSampleRate == destinationSamplerate) {
if (this.socket) { return buffer;
this.socket.close()
} }
// if (this.setting.protocol === "sio") { if (destinationSamplerate > originalSampleRate) {
// this.socket = io(this.setting.serverUrl + "/test"); throw "downsampling rate show be smaller than original sample rate";
this.socket = io("/test"); }
this.socket.on('connect_error', (err) => { const sampleRateRatio = originalSampleRate / destinationSamplerate;
console.log("connect exception !!!!!") const newLength = Math.round(buffer.length / sampleRateRatio);
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`) const result = new Float32Array(newLength);
}) let offsetResult = 0;
this.socket.on('connect', () => { let offsetBuffer = 0;
// console.log(`[SIO] sonnect to ${this.setting.serverUrl}`) while (offsetResult < result.length) {
console.log(`[SIO] ${this.socket?.id}`) var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
}); // Use average value of skipped samples
this.socket.on('response', (response: any[]) => { var accum = 0, count = 0;
const cur = Date.now() for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
const responseTime = cur - response[0] accum += buffer[i];
const result = response[1] as ArrayBuffer count++;
if (result.byteLength < 128 * 2) {
console.log("tooshort!!")
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
} else {
console.log("response!!!")
this.postReceivedVoice(response[1])
// this.callbacks.onVoiceReceived(response[1])
// this.audioStreamerListeners.notifyResponseTime(responseTime)
} }
}); result[offsetResult] = accum / count;
// } // Or you can simply get rid of the skipped samples:
// result[offsetResult] = buffer[nextOffsetBuffer];
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
return result;
} }
handleMessage(event: any) { handleMessage(event: any) {
@ -74,21 +127,42 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
const inputData = event.data.inputData as Float32Array const inputData = event.data.inputData as Float32Array
// console.log("receive input data", inputData) // console.log("receive input data", inputData)
const arrayBuffer = new ArrayBuffer(inputData.length * 2) // ダウンサンプリング
let downsampledBuffer: Float32Array | null = null
if (this.setting.sendingSampleRate == 48000) {
downsampledBuffer = inputData
} else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
//////// (Kind 1) 間引き //////////
// bufferSize個のデータ（48Khz）が入ってくる。
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。
downsampledBuffer = new Float32Array(inputData.length / 2);
for (let i = 0; i < inputData.length; i++) {
if (i % 2 == 0) {
downsampledBuffer[i / 2] = inputData[i]
}
}
} else {
//////// (Kind 2) 平均 //////////
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
downsampledBuffer = this._averageDownsampleBuffer(inputData, 48000, this.setting.sendingSampleRate)
}
// Float to Int16
const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2)
const dataView = new DataView(arrayBuffer); const dataView = new DataView(arrayBuffer);
for (let i = 0; i < inputData.length; i++) { for (let i = 0; i < downsampledBuffer.length; i++) {
let s = Math.max(-1, Math.min(1, inputData[i])); let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
s = s < 0 ? s * 0x8000 : s * 0x7FFF s = s < 0 ? s * 0x8000 : s * 0x7FFF
dataView.setInt16(i * 2, s, true); dataView.setInt16(i * 2, s, true);
} }
// バッファリング
this.requestChunks.push(arrayBuffer) this.requestChunks.push(arrayBuffer)
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。 //// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
if (this.requestChunks.length < 32) { if (this.requestChunks.length < this.setting.inputChunkNum) {
return return
} }
console.log("sending...")
// リクエスト用の入れ物を作成 // リクエスト用の入れ物を作成
const windowByteLength = this.requestChunks.reduce((prev, cur) => { const windowByteLength = this.requestChunks.reduce((prev, cur) => {
@ -104,10 +178,10 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.sendBuffer(newBuffer) this.sendBuffer(newBuffer)
console.log("sended...")
this.requestChunks = [] this.requestChunks = []
this.listener.notifySendBufferingTime(Date.now() - this.bufferStart)
this.bufferStart = Date.now()
} else { } else {
console.warn(`[worklet_node][voice-changer-worklet-processor] unknown response ${event.data.responseType}`, event.data) console.warn(`[worklet_node][voice-changer-worklet-processor] unknown response ${event.data.responseType}`, event.data)
@ -118,28 +192,28 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
private sendBuffer = async (newBuffer: Uint8Array) => { private sendBuffer = async (newBuffer: Uint8Array) => {
const timestamp = Date.now() const timestamp = Date.now()
// if (this.setting.protocol === "sio") { if (this.setting.protocol === "sio") {
if (!this.socket) { if (!this.socket) {
console.warn(`sio is not initialized`) console.warn(`sio is not initialized`)
return return
} }
// console.log("emit!") // console.log("emit!")
this.socket.emit('request_message', [ this.socket.emit('request_message', [
timestamp, timestamp,
newBuffer.buffer]); newBuffer.buffer]);
// } else { } else {
// const res = await postVoice( const res = await postVoice(
// this.setting.serverUrl + "/test", this.setting.serverUrl + "/test",
// timestamp, timestamp,
// newBuffer.buffer) newBuffer.buffer)
// if (res.byteLength < 128 * 2) { if (res.byteLength < 128 * 2) {
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`) this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
// } else { } else {
// this.callbacks.onVoiceReceived(res) this.postReceivedVoice(res)
// this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp) this.listener.notifyResponseTime(Date.now() - timestamp)
// } }
// } }
} }
@ -154,7 +228,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.port.postMessage(req) this.port.postMessage(req)
} }
startOutputRecordingWorklet = () => { startRecording = () => {
const req: VoiceChangerWorkletProcessorRequest = { const req: VoiceChangerWorkletProcessorRequest = {
requestType: "startRecording", requestType: "startRecording",
voice: new ArrayBuffer(1), voice: new ArrayBuffer(1),
@ -165,7 +239,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.port.postMessage(req) this.port.postMessage(req)
} }
stopOutputRecordingWorklet = () => { stopRecording = () => {
const req: VoiceChangerWorkletProcessorRequest = { const req: VoiceChangerWorkletProcessorRequest = {
requestType: "stopRecording", requestType: "stopRecording",
voice: new ArrayBuffer(1), voice: new ArrayBuffer(1),
@ -175,4 +249,35 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
} }
this.port.postMessage(req) this.port.postMessage(req)
} }
}
export const postVoice = async (
url: string,
timestamp: number,
buffer: ArrayBuffer) => {
const obj = {
timestamp,
buffer: Buffer.from(buffer).toString('base64')
};
const body = JSON.stringify(obj);
const res = await fetch(`${url}`, {
method: "POST",
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: body
})
const receivedJson = await res.json()
const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
const buf = Buffer.from(changedVoiceBase64, "base64")
const ab = new ArrayBuffer(buf.length);
const view = new Uint8Array(ab);
for (let i = 0; i < buf.length; ++i) {
view[i] = buf[i];
}
return ab
} }

8
client/lib/src/const.ts
View File

@ -132,7 +132,7 @@ export const DefaultWorkletSetting: WorkletSetting = {
volTrancateLength: 32 volTrancateLength: 32
} }
/////////////////////// ///////////////////////
// Audio Streamerセッティング // Worklet Nodeセッティング
/////////////////////// ///////////////////////
export const Protocol = { export const Protocol = {
"sio": "sio", "sio": "sio",
@ -153,14 +153,14 @@ export const DownSamplingMode = {
export type DownSamplingMode = typeof DownSamplingMode[keyof typeof DownSamplingMode] export type DownSamplingMode = typeof DownSamplingMode[keyof typeof DownSamplingMode]
export type AudioStreamerSetting = { export type WorkletNodeSetting = {
serverUrl: string, serverUrl: string,
protocol: Protocol, protocol: Protocol,
sendingSampleRate: SendingSampleRate, sendingSampleRate: SendingSampleRate,
inputChunkNum: number, inputChunkNum: number,
downSamplingMode: DownSamplingMode, downSamplingMode: DownSamplingMode,
} }
export const DefaultAudioStreamerSetting: AudioStreamerSetting = { export const DefaultWorkletNodeSetting: WorkletNodeSetting = {
serverUrl: "", serverUrl: "",
protocol: "sio", protocol: "sio",
sendingSampleRate: 48000, sendingSampleRate: 48000,
@ -265,7 +265,7 @@ export const INDEXEDDB_DB_APP_NAME = "INDEXEDDB_KEY_VOICE_CHANGER"
export const INDEXEDDB_DB_NAME = "INDEXEDDB_KEY_VOICE_CHANGER_DB" export const INDEXEDDB_DB_NAME = "INDEXEDDB_KEY_VOICE_CHANGER_DB"
export const INDEXEDDB_KEY_CLIENT = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_CLIENT" export const INDEXEDDB_KEY_CLIENT = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_CLIENT"
export const INDEXEDDB_KEY_SERVER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_SERVER" export const INDEXEDDB_KEY_SERVER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_SERVER"
export const INDEXEDDB_KEY_STREAMER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_STREAMER" export const INDEXEDDB_KEY_WORKLETNODE = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_WORKLETNODE"
export const INDEXEDDB_KEY_MODEL_DATA = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_MODEL_DATA" export const INDEXEDDB_KEY_MODEL_DATA = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_MODEL_DATA"
export const INDEXEDDB_KEY_WORKLET = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_WORKLET" export const INDEXEDDB_KEY_WORKLET = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_WORKLET"

74
client/lib/src/hooks/useAudioStreamerSetting.ts
View File

@ -1,74 +0,0 @@
import { useState, useMemo, useEffect } from "react"
import { INDEXEDDB_KEY_STREAMER, AudioStreamerSetting, DefaultAudioStreamerSetting } from "../const"
import { VoiceChangerClient } from "../VoiceChangerClient"
import { useIndexedDB } from "./useIndexedDB"
export type UseAudioStreamerSettingProps = {
voiceChangerClient: VoiceChangerClient | null
}
export type AudioStreamerSettingState = {
audioStreamerSetting: AudioStreamerSetting;
clearSetting: () => Promise<void>
updateAudioStreamerSetting: (setting: AudioStreamerSetting) => void
}
export const useAudioStreamerSetting = (props: UseAudioStreamerSettingProps): AudioStreamerSettingState => {
const [audioStreamerSetting, _setAudioStreamerSetting] = useState<AudioStreamerSetting>(DefaultAudioStreamerSetting)
const { setItem, getItem, removeItem } = useIndexedDB()
// 初期化 その１ DBから取得
useEffect(() => {
const loadCache = async () => {
const setting = await getItem(INDEXEDDB_KEY_STREAMER) as AudioStreamerSetting
if (setting) {
_setAudioStreamerSetting(setting)
}
}
loadCache()
}, [])
// 初期化 その２ クライアントに設定
useEffect(() => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setServerUrl(audioStreamerSetting.serverUrl)
props.voiceChangerClient.updateAudioStreamerSetting(audioStreamerSetting)
}, [props.voiceChangerClient])
const clearSetting = async () => {
await removeItem(INDEXEDDB_KEY_STREAMER)
}
//////////////
// 設定
/////////////
const updateAudioStreamerSetting = useMemo(() => {
return (_audioStreamerSetting: AudioStreamerSetting) => {
if (!props.voiceChangerClient) return
for (let k in _audioStreamerSetting) {
const cur_v = audioStreamerSetting[k]
const new_v = _audioStreamerSetting[k]
if (cur_v != new_v) {
_setAudioStreamerSetting(_audioStreamerSetting)
setItem(INDEXEDDB_KEY_STREAMER, _audioStreamerSetting)
props.voiceChangerClient.updateAudioStreamerSetting(_audioStreamerSetting)
break
}
}
}
}, [props.voiceChangerClient, audioStreamerSetting])
return {
audioStreamerSetting,
clearSetting,
updateAudioStreamerSetting,
}
}

14
client/lib/src/hooks/useClient.ts
View File

@ -1,9 +1,8 @@
import { useEffect, useMemo, useRef, useState } from "react" import { useEffect, useMemo, useRef, useState } from "react"
import { AudioStreamer } from "../AudioStreamer"
import { VoiceChangerClient } from "../VoiceChangerClient" import { VoiceChangerClient } from "../VoiceChangerClient"
import { AudioStreamerSettingState, useAudioStreamerSetting } from "./useAudioStreamerSetting"
import { ClientSettingState, useClientSetting } from "./useClientSetting" import { ClientSettingState, useClientSetting } from "./useClientSetting"
import { ServerSettingState, useServerSetting } from "./useServerSetting" import { ServerSettingState, useServerSetting } from "./useServerSetting"
import { useWorkletNodeSetting, WorkletNodeSettingState } from "./useWorkletNodeSetting"
import { useWorkletSetting, WorkletSettingState } from "./useWorkletSetting" import { useWorkletSetting, WorkletSettingState } from "./useWorkletSetting"
export type UseClientProps = { export type UseClientProps = {
@ -15,7 +14,7 @@ export type ClientState = {
// 各種設定I/Fへの参照 // 各種設定I/Fへの参照
workletSetting: WorkletSettingState workletSetting: WorkletSettingState
clientSetting: ClientSettingState clientSetting: ClientSettingState
streamerSetting: AudioStreamerSettingState workletNodeSetting: WorkletNodeSettingState
serverSetting: ServerSettingState serverSetting: ServerSettingState
// モニタリングデータ // モニタリングデータ
@ -48,7 +47,7 @@ export const useClient = (props: UseClientProps): ClientState => {
// (1-2) 各種設定I/F // (1-2) 各種設定I/F
const clientSetting = useClientSetting({ voiceChangerClient, audioContext: props.audioContext }) const clientSetting = useClientSetting({ voiceChangerClient, audioContext: props.audioContext })
const streamerSetting = useAudioStreamerSetting({ voiceChangerClient }) const workletNodeSetting = useWorkletNodeSetting({ voiceChangerClient })
const workletSetting = useWorkletSetting({ voiceChangerClient }) const workletSetting = useWorkletSetting({ voiceChangerClient })
const serverSetting = useServerSetting({ voiceChangerClient }) const serverSetting = useServerSetting({ voiceChangerClient })
@ -83,8 +82,7 @@ export const useClient = (props: UseClientProps): ClientState => {
errorCountRef.current = 0 errorCountRef.current = 0
} }
} }
} },
}, {
notifyVolume: (vol: number) => { notifyVolume: (vol: number) => {
setVolume(vol) setVolume(vol)
}, },
@ -119,7 +117,7 @@ export const useClient = (props: UseClientProps): ClientState => {
const clearSetting = async () => { const clearSetting = async () => {
await clientSetting.clearSetting() await clientSetting.clearSetting()
await streamerSetting.clearSetting() await workletNodeSetting.clearSetting()
await workletSetting.clearSetting() await workletSetting.clearSetting()
await serverSetting.clearSetting() await serverSetting.clearSetting()
} }
@ -127,7 +125,7 @@ export const useClient = (props: UseClientProps): ClientState => {
return { return {
// 各種設定I/Fへの参照 // 各種設定I/Fへの参照
clientSetting, clientSetting,
streamerSetting, workletNodeSetting,
workletSetting, workletSetting,
serverSetting, serverSetting,

73
client/lib/src/hooks/useWorkletNodeSetting.ts Normal file
View File

@ -0,0 +1,73 @@
import { useState, useMemo, useEffect } from "react"
import { DefaultWorkletNodeSetting, INDEXEDDB_KEY_WORKLETNODE, WorkletNodeSetting } from "../const"
import { VoiceChangerClient } from "../VoiceChangerClient"
import { useIndexedDB } from "./useIndexedDB"
export type UseWorkletNodeSettingProps = {
voiceChangerClient: VoiceChangerClient | null
}
export type WorkletNodeSettingState = {
workletNodeSetting: WorkletNodeSetting;
clearSetting: () => Promise<void>
updateWorkletNodeSetting: (setting: WorkletNodeSetting) => void
}
export const useWorkletNodeSetting = (props: UseWorkletNodeSettingProps): WorkletNodeSettingState => {
const [workletNodeSetting, _setWorkletNodeSetting] = useState<WorkletNodeSetting>(DefaultWorkletNodeSetting)
const { setItem, getItem, removeItem } = useIndexedDB()
// 初期化 その１ DBから取得
useEffect(() => {
const loadCache = async () => {
const setting = await getItem(INDEXEDDB_KEY_WORKLETNODE) as WorkletNodeSetting
if (setting) {
_setWorkletNodeSetting(setting)
}
}
loadCache()
}, [])
// 初期化 その２ クライアントに設定
useEffect(() => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setServerUrl(workletNodeSetting.serverUrl)
props.voiceChangerClient.updateWorkletNodeSetting(workletNodeSetting)
}, [props.voiceChangerClient])
const clearSetting = async () => {
await removeItem(INDEXEDDB_KEY_WORKLETNODE)
}
//////////////
// 設定
/////////////
const updateWorkletNodeSetting = useMemo(() => {
return (_workletNodeSetting: WorkletNodeSetting) => {
if (!props.voiceChangerClient) return
for (let k in _workletNodeSetting) {
const cur_v = workletNodeSetting[k]
const new_v = _workletNodeSetting[k]
if (cur_v != new_v) {
_setWorkletNodeSetting(_workletNodeSetting)
setItem(INDEXEDDB_KEY_WORKLETNODE, _workletNodeSetting)
props.voiceChangerClient.updateWorkletNodeSetting(_workletNodeSetting)
break
}
}
}
}, [props.voiceChangerClient, workletNodeSetting])
return {
workletNodeSetting,
clearSetting,
updateWorkletNodeSetting,
}
}

32
client/lib/src/hooks/useWorkletSetting.ts
View File

@ -11,8 +11,8 @@ export type WorkletSettingState = {
setting: WorkletSetting; setting: WorkletSetting;
clearSetting: () => Promise<void> clearSetting: () => Promise<void>
setSetting: (setting: WorkletSetting) => void; setSetting: (setting: WorkletSetting) => void;
startOutputRecording: () => void // startOutputRecording: () => void
stopOutputRecording: () => Promise<void> // stopOutputRecording: () => Promise<void>
} }
export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSettingState => { export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSettingState => {
@ -68,26 +68,26 @@ export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSetting
await removeItem(INDEXEDDB_KEY_WORKLET) await removeItem(INDEXEDDB_KEY_WORKLET)
} }
const startOutputRecording = useMemo(() => { // const startOutputRecording = useMemo(() => {
return () => { // return () => {
if (!props.voiceChangerClient) return // if (!props.voiceChangerClient) return
props.voiceChangerClient.startOutputRecordingWorklet() // props.voiceChangerClient.startOutputRecordingWorklet()
} // }
}, [props.voiceChangerClient]) // }, [props.voiceChangerClient])
const stopOutputRecording = useMemo(() => { // const stopOutputRecording = useMemo(() => {
return async () => { // return async () => {
if (!props.voiceChangerClient) return // if (!props.voiceChangerClient) return
props.voiceChangerClient.stopOutputRecordingWorklet() // props.voiceChangerClient.stopOutputRecordingWorklet()
} // }
}, [props.voiceChangerClient]) // }, [props.voiceChangerClient])
return { return {
setting, setting,
clearSetting, clearSetting,
setSetting, setSetting,
startOutputRecording, // startOutputRecording,
stopOutputRecording // stopOutputRecording
} }
} }

19
client/lib/worklet/src/voice-changer-worklet-processor.ts
View File

@ -9,7 +9,6 @@ export type RequestType = typeof RequestType[keyof typeof RequestType]
export const ResponseType = { export const ResponseType = {
"volume": "volume", "volume": "volume",
"recordData": "recordData",
"inputData": "inputData" "inputData": "inputData"
} as const } as const
export type ResponseType = typeof ResponseType[keyof typeof ResponseType] export type ResponseType = typeof ResponseType[keyof typeof ResponseType]
@ -43,7 +42,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
private isRecording = false private isRecording = false
playBuffer: Float32Array[] = [] playBuffer: Float32Array[] = []
recordingBuffer: Float32Array[] = []
/** /**
* @constructor * @constructor
*/ */
@ -75,7 +73,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
return return
} }
this.isRecording = true this.isRecording = true
this.recordingBuffer = []
return return
} else if (request.requestType === "stopRecording") { } else if (request.requestType === "stopRecording") {
if (!this.isRecording) { if (!this.isRecording) {
@ -83,13 +80,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
return return
} }
this.isRecording = false this.isRecording = false
const recordResponse: VoiceChangerWorkletProcessorResponse = {
responseType: ResponseType.recordData,
recordData: this.recordingBuffer
}
this.port.postMessage(recordResponse);
this.recordingBuffer = []
return return
} }
@ -126,9 +116,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
f32Block![frameIndexInBlock + 1] = (currentFrame + nextFrame) / 2 f32Block![frameIndexInBlock + 1] = (currentFrame + nextFrame) / 2
if (f32Block!.length === frameIndexInBlock + 2) { if (f32Block!.length === frameIndexInBlock + 2) {
this.playBuffer.push(f32Block!) this.playBuffer.push(f32Block!)
if (this.isRecording) {
this.recordingBuffer.push(f32Block!)
}
} }
} }
} }
@ -148,8 +135,10 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
return true; return true;
} }
if (_inputs.length > 0 && _inputs[0].length > 0) { if (this.isRecording) {
this.pushData(_inputs[0][0]) if (_inputs.length > 0 && _inputs[0].length > 0) {
this.pushData(_inputs[0][0])
}
} }
if (this.playBuffer.length === 0) { if (this.playBuffer.length === 0) {