1berry/wokada-voice-changer
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EXP. remove microphone stream 2

Browse Source
This commit is contained in:
wataru 2023-02-20 02:21:51 +09:00
parent de0be188ff
commit bde893b665
12 changed files with 518 additions and 478 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/demo/dist/index.js vendored
View File

File diff suppressed because one or more lines are too long

8
client/demo/src/106_convert_setting.tsx
View File

@ -25,18 +25,18 @@ export const useConvertSetting = (): ConvertSettingState => {
<div className="body-row split-3-2-1-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">Input Chunk Num(128sample/chunk)</div>
<div className="body-input-container">
<input type="number" min={1} max={256} step={1} value={appState.streamerSetting.audioStreamerSetting.inputChunkNum} onChange={(e) => {
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, inputChunkNum: Number(e.target.value) })
<input type="number" min={1} max={256} step={1} value={appState.workletNodeSetting.workletNodeSetting.inputChunkNum} onChange={(e) => {
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, inputChunkNum: Number(e.target.value) })
}} />
</div>
<div className="body-item-text">
<div>buff: {(appState.streamerSetting.audioStreamerSetting.inputChunkNum * 128 * 1000 / 24000).toFixed(1)}ms</div>
<div>buff: {(appState.workletNodeSetting.workletNodeSetting.inputChunkNum * 128 * 1000 / 48000).toFixed(1)}ms</div>
</div>
<div className="body-item-text"></div>
</div>
)
}, [appState.streamerSetting.audioStreamerSetting.inputChunkNum, appState.streamerSetting.updateAudioStreamerSetting])
}, [appState.workletNodeSetting.workletNodeSetting.inputChunkNum, appState.workletNodeSetting.updateWorkletNodeSetting])
const gpuRow = useMemo(() => {
return (

22
client/demo/src/107_advanced_setting.tsx
View File

@ -30,24 +30,24 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">MMVC Server</div>
<div className="body-input-container">
<input type="text" defaultValue={appState.streamerSetting.audioStreamerSetting.serverUrl} id="mmvc-server-url" className="body-item-input" />
<input type="text" defaultValue={appState.workletNodeSetting.workletNodeSetting.serverUrl} id="mmvc-server-url" className="body-item-input" />
</div>
<div className="body-button-container">
<div className="body-button" onClick={onSetServerClicked}>set</div>
</div>
</div>
)
}, [appState.streamerSetting.audioStreamerSetting.serverUrl, appState.clientSetting.setServerUrl])
}, [appState.workletNodeSetting.workletNodeSetting.serverUrl, appState.clientSetting.setServerUrl])
const protocolRow = useMemo(() => {
const onProtocolChanged = async (val: Protocol) => {
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, protocol: val })
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, protocol: val })
}
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Protocol</div>
<div className="body-select-container">
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.protocol} onChange={(e) => {
<select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.protocol} onChange={(e) => {
onProtocolChanged(e.target.value as
Protocol)
}}>
@ -60,7 +60,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
</div>
</div>
)
}, [appState.streamerSetting.audioStreamerSetting.protocol, appState.streamerSetting.updateAudioStreamerSetting])
}, [appState.workletNodeSetting.workletNodeSetting.protocol, appState.workletNodeSetting.updateWorkletNodeSetting])
const sampleRateRow = useMemo(() => {
@ -87,8 +87,8 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Sending Sample Rate</div>
<div className="body-select-container">
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.sendingSampleRate} onChange={(e) => {
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, sendingSampleRate: Number(e.target.value) as InputSampleRate })
<select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.sendingSampleRate} onChange={(e) => {
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, sendingSampleRate: Number(e.target.value) as InputSampleRate })
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, inputSampleRate: Number(e.target.value) as InputSampleRate })
}}>
{
@ -100,7 +100,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
</div>
</div>
)
}, [appState.streamerSetting.audioStreamerSetting.sendingSampleRate, appState.streamerSetting.updateAudioStreamerSetting, appState.serverSetting.updateServerSettings])
}, [appState.workletNodeSetting.workletNodeSetting.sendingSampleRate, appState.workletNodeSetting.updateWorkletNodeSetting, appState.serverSetting.updateServerSettings])
const bufferSizeRow = useMemo(() => {
return (
@ -174,8 +174,8 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1 ">DownSamplingMode</div>
<div className="body-select-container">
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.downSamplingMode} onChange={(e) => {
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, downSamplingMode: e.target.value as DownSamplingMode })
<select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.downSamplingMode} onChange={(e) => {
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, downSamplingMode: e.target.value as DownSamplingMode })
}}>
{
Object.values(DownSamplingMode).map(x => {
@ -186,7 +186,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
</div>
</div>
)
}, [appState.streamerSetting.audioStreamerSetting.downSamplingMode, appState.streamerSetting.updateAudioStreamerSetting])
}, [appState.workletNodeSetting.workletNodeSetting.downSamplingMode, appState.workletNodeSetting.updateWorkletNodeSetting])
const workletSettingRow = useMemo(() => {

424
client/lib/src/AudioStreamer.ts
View File

@ -1,243 +1,243 @@
import { io, Socket } from "socket.io-client";
import { DefaultEventsMap } from "@socket.io/component-emitter";
import { Duplex, DuplexOptions } from "readable-stream";
import { AudioStreamerSetting, DefaultAudioStreamerSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
// import { io, Socket } from "socket.io-client";
// import { DefaultEventsMap } from "@socket.io/component-emitter";
// import { Duplex, DuplexOptions } from "readable-stream";
// import { AudioStreamerSetting, DefaultAudioStreamerSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
export type Callbacks = {
onVoiceReceived: (data: ArrayBuffer) => void
}
export type AudioStreamerListeners = {
notifySendBufferingTime: (time: number) => void
notifyResponseTime: (time: number) => void
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
}
// export type Callbacks = {
// onVoiceReceived: (data: ArrayBuffer) => void
// }
// export type AudioStreamerListeners = {
// notifySendBufferingTime: (time: number) => void
// notifyResponseTime: (time: number) => void
// notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
// }
export class AudioStreamer extends Duplex {
private setting: AudioStreamerSetting = DefaultAudioStreamerSetting
// export class AudioStreamer extends Duplex {
// private setting: AudioStreamerSetting = DefaultAudioStreamerSetting
private callbacks: Callbacks
private audioStreamerListeners: AudioStreamerListeners
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
private requestChunks: ArrayBuffer[] = []
// private callbacks: Callbacks
// private audioStreamerListeners: AudioStreamerListeners
// private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
// private requestChunks: ArrayBuffer[] = []
// performance monitor
private bufferStart = 0;
// // performance monitor
// private bufferStart = 0;
constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
super(options);
this.callbacks = callbacks
this.audioStreamerListeners = audioStreamerListeners
this.createSocketIO()
}
// constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
// super(options);
// this.callbacks = callbacks
// this.audioStreamerListeners = audioStreamerListeners
// this.createSocketIO()
// }
private createSocketIO = () => {
if (this.socket) {
this.socket.close()
}
if (this.setting.protocol === "sio") {
this.socket = io(this.setting.serverUrl + "/test");
this.socket.on('connect_error', (err) => {
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
})
this.socket.on('connect', () => {
console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
console.log(`[SIO] ${this.socket?.id}`)
});
this.socket.on('response', (response: any[]) => {
const cur = Date.now()
const responseTime = cur - response[0]
const result = response[1] as ArrayBuffer
if (result.byteLength < 128 * 2) {
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
} else {
this.callbacks.onVoiceReceived(response[1])
this.audioStreamerListeners.notifyResponseTime(responseTime)
}
});
}
}
// private createSocketIO = () => {
// if (this.socket) {
// this.socket.close()
// }
// if (this.setting.protocol === "sio") {
// this.socket = io(this.setting.serverUrl + "/test");
// this.socket.on('connect_error', (err) => {
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
// })
// this.socket.on('connect', () => {
// console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
// console.log(`[SIO] ${this.socket?.id}`)
// });
// this.socket.on('response', (response: any[]) => {
// const cur = Date.now()
// const responseTime = cur - response[0]
// const result = response[1] as ArrayBuffer
// if (result.byteLength < 128 * 2) {
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
// } else {
// this.callbacks.onVoiceReceived(response[1])
// this.audioStreamerListeners.notifyResponseTime(responseTime)
// }
// });
// }
// }
// Option Change
updateSetting = (setting: AudioStreamerSetting) => {
console.log(`[AudioStreamer] Updating AudioStreamer Setting,`, this.setting, setting)
let recreateSocketIoRequired = false
if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
recreateSocketIoRequired = true
}
this.setting = setting
if (recreateSocketIoRequired) {
this.createSocketIO()
}
}
// // Option Change
// updateSetting = (setting: AudioStreamerSetting) => {
// console.log(`[AudioStreamer] Updating AudioStreamer Setting,`, this.setting, setting)
// let recreateSocketIoRequired = false
// if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
// recreateSocketIoRequired = true
// }
// this.setting = setting
// if (recreateSocketIoRequired) {
// this.createSocketIO()
// }
// }
getSettings = (): AudioStreamerSetting => {
return this.setting
}
// getSettings = (): AudioStreamerSetting => {
// return this.setting
// }
getSocketId = () => {
return this.socket?.id
}
// getSocketId = () => {
// return this.socket?.id
// }
// Main Process
//// Pipe from mic stream
_write = (chunk: AudioBuffer, _encoding: any, callback: any) => {
const buffer = chunk.getChannelData(0);
this._write_realtime(buffer)
callback();
}
// // Main Process
// //// Pipe from mic stream
// _write = (chunk: AudioBuffer, _encoding: any, callback: any) => {
// const buffer = chunk.getChannelData(0);
// this._write_realtime(buffer)
// callback();
// }
_averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
if (originalSampleRate == destinationSamplerate) {
return buffer;
}
if (destinationSamplerate > originalSampleRate) {
throw "downsampling rate show be smaller than original sample rate";
}
const sampleRateRatio = originalSampleRate / destinationSamplerate;
const newLength = Math.round(buffer.length / sampleRateRatio);
const result = new Float32Array(newLength);
let offsetResult = 0;
let offsetBuffer = 0;
while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
// Use average value of skipped samples
var accum = 0, count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
accum += buffer[i];
count++;
}
result[offsetResult] = accum / count;
// Or you can simply get rid of the skipped samples:
// result[offsetResult] = buffer[nextOffsetBuffer];
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
return result;
}
// _averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
// if (originalSampleRate == destinationSamplerate) {
// return buffer;
// }
// if (destinationSamplerate > originalSampleRate) {
// throw "downsampling rate show be smaller than original sample rate";
// }
// const sampleRateRatio = originalSampleRate / destinationSamplerate;
// const newLength = Math.round(buffer.length / sampleRateRatio);
// const result = new Float32Array(newLength);
// let offsetResult = 0;
// let offsetBuffer = 0;
// while (offsetResult < result.length) {
// var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
// // Use average value of skipped samples
// var accum = 0, count = 0;
// for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
// accum += buffer[i];
// count++;
// }
// result[offsetResult] = accum / count;
// // Or you can simply get rid of the skipped samples:
// // result[offsetResult] = buffer[nextOffsetBuffer];
// offsetResult++;
// offsetBuffer = nextOffsetBuffer;
// }
// return result;
// }
private _write_realtime = async (buffer: Float32Array) => {
// private _write_realtime = async (buffer: Float32Array) => {
let downsampledBuffer: Float32Array | null = null
if (this.setting.sendingSampleRate == 48000) {
downsampledBuffer = buffer
} else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
//////// (Kind 1) 間引き //////////
// bufferSize個のデータ（48Khz）が入ってくる。
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。
downsampledBuffer = new Float32Array(buffer.length / 2);
for (let i = 0; i < buffer.length; i++) {
if (i % 2 == 0) {
downsampledBuffer[i / 2] = buffer[i]
}
}
} else {
//////// (Kind 2) 平均 //////////
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, this.setting.sendingSampleRate)
}
// let downsampledBuffer: Float32Array | null = null
// if (this.setting.sendingSampleRate == 48000) {
// downsampledBuffer = buffer
// } else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
// //////// (Kind 1) 間引き //////////
// // bufferSize個のデータ（48Khz）が入ってくる。
// //// 48000Hz で入ってくるので間引いて24000Hzに変換する。
// downsampledBuffer = new Float32Array(buffer.length / 2);
// for (let i = 0; i < buffer.length; i++) {
// if (i % 2 == 0) {
// downsampledBuffer[i / 2] = buffer[i]
// }
// }
// } else {
// //////// (Kind 2) 平均 //////////
// // downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, this.setting.sendingSampleRate)
// }
// Float to signed16
const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2)
const dataView = new DataView(arrayBuffer);
for (let i = 0; i < downsampledBuffer.length; i++) {
let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
s = s < 0 ? s * 0x8000 : s * 0x7FFF
dataView.setInt16(i * 2, s, true);
}
// // Float to signed16
// const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2)
// const dataView = new DataView(arrayBuffer);
// for (let i = 0; i < downsampledBuffer.length; i++) {
// let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
// s = s < 0 ? s * 0x8000 : s * 0x7FFF
// dataView.setInt16(i * 2, s, true);
// }
// 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理
// const chunkByteSize = 256 // (const.ts ★1)
// const chunkByteSize = 256 * 2 // (const.ts ★1)
const chunkByteSize = (256 * 2) * (this.setting.sendingSampleRate / 48000) // (const.ts ★1)
for (let i = 0; i < arrayBuffer.byteLength / chunkByteSize; i++) {
const ab = arrayBuffer.slice(i * chunkByteSize, (i + 1) * chunkByteSize)
this.requestChunks.push(ab)
}
// // 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理
// // const chunkByteSize = 256 // (const.ts ★1)
// // const chunkByteSize = 256 * 2 // (const.ts ★1)
// const chunkByteSize = (256 * 2) * (this.setting.sendingSampleRate / 48000) // (const.ts ★1)
// for (let i = 0; i < arrayBuffer.byteLength / chunkByteSize; i++) {
// const ab = arrayBuffer.slice(i * chunkByteSize, (i + 1) * chunkByteSize)
// this.requestChunks.push(ab)
// }
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
if (this.requestChunks.length < this.setting.inputChunkNum) {
return
}
// //// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
// if (this.requestChunks.length < this.setting.inputChunkNum) {
// return
// }
// リクエスト用の入れ物を作成
const windowByteLength = this.requestChunks.reduce((prev, cur) => {
return prev + cur.byteLength
}, 0)
const newBuffer = new Uint8Array(windowByteLength);
// // リクエスト用の入れ物を作成
// const windowByteLength = this.requestChunks.reduce((prev, cur) => {
// return prev + cur.byteLength
// }, 0)
// const newBuffer = new Uint8Array(windowByteLength);
// リクエストのデータをセット
this.requestChunks.reduce((prev, cur) => {
newBuffer.set(new Uint8Array(cur), prev)
return prev + cur.byteLength
}, 0)
// // リクエストのデータをセット
// this.requestChunks.reduce((prev, cur) => {
// newBuffer.set(new Uint8Array(cur), prev)
// return prev + cur.byteLength
// }, 0)
// console.log("send buff length", newBuffer.length)
// // console.log("send buff length", newBuffer.length)
this.sendBuffer(newBuffer)
this.requestChunks = []
// this.sendBuffer(newBuffer)
// this.requestChunks = []
this.audioStreamerListeners.notifySendBufferingTime(Date.now() - this.bufferStart)
this.bufferStart = Date.now()
}
// this.audioStreamerListeners.notifySendBufferingTime(Date.now() - this.bufferStart)
// this.bufferStart = Date.now()
// }
private sendBuffer = async (newBuffer: Uint8Array) => {
const timestamp = Date.now()
if (this.setting.protocol === "sio") {
if (!this.socket) {
console.warn(`sio is not initialized`)
return
}
// console.log("emit!")
this.socket.emit('request_message', [
timestamp,
newBuffer.buffer]);
} else {
const res = await postVoice(
this.setting.serverUrl + "/test",
timestamp,
newBuffer.buffer)
// private sendBuffer = async (newBuffer: Uint8Array) => {
// const timestamp = Date.now()
// if (this.setting.protocol === "sio") {
// if (!this.socket) {
// console.warn(`sio is not initialized`)
// return
// }
// // console.log("emit!")
// this.socket.emit('request_message', [
// timestamp,
// newBuffer.buffer]);
// } else {
// const res = await postVoice(
// this.setting.serverUrl + "/test",
// timestamp,
// newBuffer.buffer)
if (res.byteLength < 128 * 2) {
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
} else {
this.callbacks.onVoiceReceived(res)
this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
}
}
}
}
// if (res.byteLength < 128 * 2) {
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
// } else {
// this.callbacks.onVoiceReceived(res)
// this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
// }
// }
// }
// }
export const postVoice = async (
url: string,
timestamp: number,
buffer: ArrayBuffer) => {
const obj = {
timestamp,
buffer: Buffer.from(buffer).toString('base64')
};
const body = JSON.stringify(obj);
// export const postVoice = async (
// url: string,
// timestamp: number,
// buffer: ArrayBuffer) => {
// const obj = {
// timestamp,
// buffer: Buffer.from(buffer).toString('base64')
// };
// const body = JSON.stringify(obj);
const res = await fetch(`${url}`, {
method: "POST",
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: body
})
// const res = await fetch(`${url}`, {
// method: "POST",
// headers: {
// 'Accept': 'application/json',
// 'Content-Type': 'application/json'
// },
// body: body
// })
const receivedJson = await res.json()
const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
const buf = Buffer.from(changedVoiceBase64, "base64")
const ab = new ArrayBuffer(buf.length);
const view = new Uint8Array(ab);
for (let i = 0; i < buf.length; ++i) {
view[i] = buf[i];
}
return ab
}
// const receivedJson = await res.json()
// const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
// const buf = Buffer.from(changedVoiceBase64, "base64")
// const ab = new ArrayBuffer(buf.length);
// const view = new Uint8Array(ab);
// for (let i = 0; i < buf.length; ++i) {
// view[i] = buf[i];
// }
// return ab
// }

89
client/lib/src/VoiceChangerClient.ts
View File

@ -3,14 +3,12 @@ import { VoiceChangerWorkletNode, VoiceChangerWorkletListener } from "./VoiceCha
import workerjs from "raw-loader!../worklet/dist/index.js";
import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
import { createDummyMediaStream, validateUrl } from "./util";
import { AudioStreamerSetting, DefaultVoiceChangerClientSetting, ServerSettingKey, VoiceChangerClientSetting, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletSetting } from "./const";
import MicrophoneStream from "microphone-stream";
import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer";
import { DefaultVoiceChangerClientSetting, ServerSettingKey, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "./const";
import { ServerConfigurator } from "./ServerConfigurator";
// オーディオデータの流れ
// input node(mic or MediaStream) -> [vf node] -> microphne stream -> audio streamer ->
// sio/rest server -> audio streamer-> vc node -> output node
// input node(mic or MediaStream) -> [vf node] -> [vc node] ->
// sio/rest server -> [vc node] -> output node
import { BlockingQueue } from "./utils/BlockingQueue";
@ -23,11 +21,8 @@ export class VoiceChangerClient {
private currentMediaStream: MediaStream | null = null
private currentMediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null
private outputNodeFromVF: MediaStreamAudioDestinationNode | null = null
private inputGainNode: GainNode | null = null
private outputGainNode: GainNode | null = null
private micStream: MicrophoneStream | null = null
private audioStreamer!: AudioStreamer
private vcNode!: VoiceChangerWorkletNode
private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode
@ -41,13 +36,7 @@ export class VoiceChangerClient {
private sem = new BlockingQueue<number>();
private callbacks: Callbacks = {
onVoiceReceived: (data: ArrayBuffer): void => {
this.vcNode.postReceivedVoice(data)
}
}
constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners, voiceChangerWorkletListener: VoiceChangerWorkletListener) {
constructor(ctx: AudioContext, vfEnable: boolean, voiceChangerWorkletListener: VoiceChangerWorkletListener) {
this.sem.enqueue(0);
this.configurator = new ServerConfigurator()
this.ctx = ctx
@ -62,14 +51,11 @@ export class VoiceChangerClient {
this.outputGainNode.gain.value = this.setting.outputGain
this.vcNode.connect(this.outputGainNode) // vc node -> output node
this.outputGainNode.connect(this.currentMediaStreamAudioDestinationNode)
// (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる)
this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, })
if (this.vfEnable) {
this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' })
const dummyMediaStream = createDummyMediaStream(this.ctx)
this.currentDevice = (await this.vf.createTransformDevice(dummyMediaStream)) || null;
this.outputNodeFromVF = this.ctx.createMediaStreamDestination();
}
resolve()
})
@ -94,7 +80,6 @@ export class VoiceChangerClient {
// オペレーション
/////////////////////////////////////////////////////
/// Operations ///
// setup = async (input: string | MediaStream | null, bufferSize: BufferSize, echoCancel: boolean = true, noiseSuppression: boolean = true, noiseSuppression2: boolean = false) => {
setup = async () => {
const lockNum = await this.lock()
@ -115,9 +100,7 @@ export class VoiceChangerClient {
//// Input デバイスがnullの時はmicStreamを止めてリターン
if (!this.setting.audioInput) {
console.log(`Input Setup=> client mic is disabled.`)
if (this.micStream) {
this.micStream.pauseRecording()
}
this.vcNode.stopRecording()
await this.unlock(lockNum)
return
}
@ -143,17 +126,6 @@ export class VoiceChangerClient {
this.currentMediaStream = this.setting.audioInput
}
// create mic stream
if (this.micStream) {
this.micStream.unpipe()
this.micStream.destroy()
this.micStream = null
}
this.micStream = new MicrophoneStream({
objectMode: true,
bufferSize: this.setting.bufferSize,
context: this.ctx
})
// connect nodes.
this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream)
this.inputGainNode = this.ctx.createGain()
@ -163,11 +135,8 @@ export class VoiceChangerClient {
this.currentDevice.chooseNewInnerDevice(this.currentMediaStream)
const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node
this.inputGainNode.connect(voiceFocusNode.start) // input node -> vf node
voiceFocusNode.end.connect(this.outputNodeFromVF!)
// this.micStream.setStream(this.outputNodeFromVF!.stream) // vf node -> mic stream
voiceFocusNode.end.connect(this.vcNode)
} else {
// const inputDestinationNodeForMicStream = this.ctx.createMediaStreamDestination()
// this.inputGainNode.connect(inputDestinationNodeForMicStream)
console.log("input___ media stream", this.currentMediaStream)
this.currentMediaStream.getTracks().forEach(x => {
console.log("input___ media stream set", x.getSettings())
@ -177,17 +146,6 @@ export class VoiceChangerClient {
console.log("input___ media node", this.currentMediaStreamAudioSourceNode)
console.log("input___ gain node", this.inputGainNode.channelCount, this.inputGainNode)
this.inputGainNode.connect(this.vcNode)
// this.micStream.setStream(inputDestinationNodeForMicStream.stream) // input device -> mic stream
}
this.micStream.pipe(this.audioStreamer) // mic stream -> audio streamer
if (!this._isVoiceChanging) {
this.micStream.pauseRecording()
} else {
this.micStream.playRecording()
}
console.log("Input Setup=> success")
await this.unlock(lockNum)
@ -197,18 +155,14 @@ export class VoiceChangerClient {
}
start = () => {
if (!this.micStream) {
throw `Exception:${VOICE_CHANGER_CLIENT_EXCEPTION.ERR_MIC_STREAM_NOT_INITIALIZED}`
return
}
this.micStream.playRecording()
this.vcNode.startRecording()
this._isVoiceChanging = true
}
stop = () => {
if (!this.micStream) { return }
this.micStream.pauseRecording()
this.vcNode.stopRecording()
this._isVoiceChanging = false
}
get isVoiceChanging(): boolean {
return this._isVoiceChanging
}
@ -231,7 +185,7 @@ export class VoiceChangerClient {
}
}
}
this.audioStreamer.updateSetting({ ...this.audioStreamer.getSettings(), serverUrl: url })
this.vcNode.updateSetting({ ...this.vcNode.getSettings(), serverUrl: url })
this.configurator.setServerUrl(url)
}
@ -260,9 +214,6 @@ export class VoiceChangerClient {
if (reconstructInputRequired) {
this.setup()
}
}
setInputGain = (val: number) => {
@ -301,17 +252,17 @@ export class VoiceChangerClient {
configureWorklet = (setting: WorkletSetting) => {
this.vcNode.configure(setting)
}
startOutputRecordingWorklet = () => {
this.vcNode.startOutputRecordingWorklet()
startRecording = () => {
this.vcNode.startRecording()
}
stopOutputRecordingWorklet = () => {
this.vcNode.stopOutputRecordingWorklet()
stopRecording = () => {
this.vcNode.stopRecording()
}
//## Audio Streamer ##//
updateAudioStreamerSetting = (setting: AudioStreamerSetting) => {
this.audioStreamer.updateSetting(setting)
//## Worklet Node ##//
updateWorkletNodeSetting = (setting: WorkletNodeSetting) => {
this.vcNode.updateSetting(setting)
}
@ -320,16 +271,14 @@ export class VoiceChangerClient {
/////////////////////////////////////////////////////
// Information
getClientSettings = () => {
return this.audioStreamer.getSettings()
return this.vcNode.getSettings()
}
getServerSettings = () => {
return this.configurator.getSettings()
}
getSocketId = () => {
return this.audioStreamer.getSocketId()
return this.vcNode.getSocketId()
}
}

227
client/lib/src/VoiceChangerWorkletNode.ts
View File

@ -1,17 +1,25 @@
import { VoiceChangerWorkletProcessorRequest } from "./@types/voice-changer-worklet-processor";
import { WorkletSetting } from "./const";
import { DefaultWorkletNodeSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletNodeSetting, WorkletSetting } from "./const";
import { io, Socket } from "socket.io-client";
import { DefaultEventsMap } from "@socket.io/component-emitter";
export type VoiceChangerWorkletListener = {
notifyVolume: (vol: number) => void
notifyOutputRecordData: (data: Float32Array[]) => void
notifySendBufferingTime: (time: number) => void
notifyResponseTime: (time: number) => void
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
}
export class VoiceChangerWorkletNode extends AudioWorkletNode {
private listener: VoiceChangerWorkletListener
private setting: WorkletNodeSetting = DefaultWorkletNodeSetting
private requestChunks: ArrayBuffer[] = []
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
// performance monitor
private bufferStart = 0;
constructor(context: AudioContext, listener: VoiceChangerWorkletListener) {
super(context, "voice-changer-worklet-processor");
this.port.onmessage = this.handleMessage.bind(this);
@ -20,7 +28,56 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
console.log(`[worklet_node][voice-changer-worklet-processor] created.`);
}
postReceivedVoice = (data: ArrayBuffer) => {
// 設定
updateSetting = (setting: WorkletNodeSetting) => {
console.log(`[WorkletNode] Updating WorkletNode Setting,`, this.setting, setting)
let recreateSocketIoRequired = false
if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
recreateSocketIoRequired = true
}
this.setting = setting
if (recreateSocketIoRequired) {
this.createSocketIO()
}
}
getSettings = (): WorkletNodeSetting => {
return this.setting
}
getSocketId = () => {
return this.socket?.id
}
// 処理
private createSocketIO = () => {
if (this.socket) {
this.socket.close()
}
if (this.setting.protocol === "sio") {
this.socket = io(this.setting.serverUrl + "/test");
this.socket.on('connect_error', (err) => {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
})
this.socket.on('connect', () => {
console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
console.log(`[SIO] ${this.socket?.id}`)
});
this.socket.on('response', (response: any[]) => {
const cur = Date.now()
const responseTime = cur - response[0]
const result = response[1] as ArrayBuffer
if (result.byteLength < 128 * 2) {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
} else {
this.postReceivedVoice(response[1])
this.listener.notifyResponseTime(responseTime)
}
});
}
}
private postReceivedVoice = (data: ArrayBuffer) => {
const req: VoiceChangerWorkletProcessorRequest = {
requestType: "voice",
voice: data,
@ -31,37 +88,33 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.port.postMessage(req)
}
private createSocketIO = () => {
if (this.socket) {
this.socket.close()
private _averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
if (originalSampleRate == destinationSamplerate) {
return buffer;
}
// if (this.setting.protocol === "sio") {
// this.socket = io(this.setting.serverUrl + "/test");
this.socket = io("/test");
this.socket.on('connect_error', (err) => {
console.log("connect exception !!!!!")
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
})
this.socket.on('connect', () => {
// console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
console.log(`[SIO] ${this.socket?.id}`)
});
this.socket.on('response', (response: any[]) => {
const cur = Date.now()
const responseTime = cur - response[0]
const result = response[1] as ArrayBuffer
if (result.byteLength < 128 * 2) {
console.log("tooshort!!")
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
} else {
console.log("response!!!")
this.postReceivedVoice(response[1])
// this.callbacks.onVoiceReceived(response[1])
// this.audioStreamerListeners.notifyResponseTime(responseTime)
if (destinationSamplerate > originalSampleRate) {
throw "downsampling rate show be smaller than original sample rate";
}
const sampleRateRatio = originalSampleRate / destinationSamplerate;
const newLength = Math.round(buffer.length / sampleRateRatio);
const result = new Float32Array(newLength);
let offsetResult = 0;
let offsetBuffer = 0;
while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
// Use average value of skipped samples
var accum = 0, count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
accum += buffer[i];
count++;
}
});
// }
result[offsetResult] = accum / count;
// Or you can simply get rid of the skipped samples:
// result[offsetResult] = buffer[nextOffsetBuffer];
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
return result;
}
handleMessage(event: any) {
@ -74,21 +127,42 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
const inputData = event.data.inputData as Float32Array
// console.log("receive input data", inputData)
const arrayBuffer = new ArrayBuffer(inputData.length * 2)
// ダウンサンプリング
let downsampledBuffer: Float32Array | null = null
if (this.setting.sendingSampleRate == 48000) {
downsampledBuffer = inputData
} else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
//////// (Kind 1) 間引き //////////
// bufferSize個のデータ（48Khz）が入ってくる。
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。
downsampledBuffer = new Float32Array(inputData.length / 2);
for (let i = 0; i < inputData.length; i++) {
if (i % 2 == 0) {
downsampledBuffer[i / 2] = inputData[i]
}
}
} else {
//////// (Kind 2) 平均 //////////
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
downsampledBuffer = this._averageDownsampleBuffer(inputData, 48000, this.setting.sendingSampleRate)
}
// Float to Int16
const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2)
const dataView = new DataView(arrayBuffer);
for (let i = 0; i < inputData.length; i++) {
let s = Math.max(-1, Math.min(1, inputData[i]));
for (let i = 0; i < downsampledBuffer.length; i++) {
let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
s = s < 0 ? s * 0x8000 : s * 0x7FFF
dataView.setInt16(i * 2, s, true);
}
// バッファリング
this.requestChunks.push(arrayBuffer)
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
if (this.requestChunks.length < 32) {
if (this.requestChunks.length < this.setting.inputChunkNum) {
return
}
console.log("sending...")
// リクエスト用の入れ物を作成
const windowByteLength = this.requestChunks.reduce((prev, cur) => {
@ -104,10 +178,10 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.sendBuffer(newBuffer)
console.log("sended...")
this.requestChunks = []
this.listener.notifySendBufferingTime(Date.now() - this.bufferStart)
this.bufferStart = Date.now()
} else {
console.warn(`[worklet_node][voice-changer-worklet-processor] unknown response ${event.data.responseType}`, event.data)
@ -118,28 +192,28 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
private sendBuffer = async (newBuffer: Uint8Array) => {
const timestamp = Date.now()
// if (this.setting.protocol === "sio") {
if (!this.socket) {
console.warn(`sio is not initialized`)
return
}
// console.log("emit!")
this.socket.emit('request_message', [
timestamp,
newBuffer.buffer]);
// } else {
// const res = await postVoice(
// this.setting.serverUrl + "/test",
// timestamp,
// newBuffer.buffer)
if (this.setting.protocol === "sio") {
if (!this.socket) {
console.warn(`sio is not initialized`)
return
}
// console.log("emit!")
this.socket.emit('request_message', [
timestamp,
newBuffer.buffer]);
} else {
const res = await postVoice(
this.setting.serverUrl + "/test",
timestamp,
newBuffer.buffer)
// if (res.byteLength < 128 * 2) {
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
// } else {
// this.callbacks.onVoiceReceived(res)
// this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
// }
// }
if (res.byteLength < 128 * 2) {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
} else {
this.postReceivedVoice(res)
this.listener.notifyResponseTime(Date.now() - timestamp)
}
}
}
@ -154,7 +228,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.port.postMessage(req)
}
startOutputRecordingWorklet = () => {
startRecording = () => {
const req: VoiceChangerWorkletProcessorRequest = {
requestType: "startRecording",
voice: new ArrayBuffer(1),
@ -165,7 +239,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.port.postMessage(req)
}
stopOutputRecordingWorklet = () => {
stopRecording = () => {
const req: VoiceChangerWorkletProcessorRequest = {
requestType: "stopRecording",
voice: new ArrayBuffer(1),
@ -175,4 +249,35 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
}
this.port.postMessage(req)
}
}
export const postVoice = async (
url: string,
timestamp: number,
buffer: ArrayBuffer) => {
const obj = {
timestamp,
buffer: Buffer.from(buffer).toString('base64')
};
const body = JSON.stringify(obj);
const res = await fetch(`${url}`, {
method: "POST",
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: body
})
const receivedJson = await res.json()
const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
const buf = Buffer.from(changedVoiceBase64, "base64")
const ab = new ArrayBuffer(buf.length);
const view = new Uint8Array(ab);
for (let i = 0; i < buf.length; ++i) {
view[i] = buf[i];
}
return ab
}

8
client/lib/src/const.ts
View File

@ -132,7 +132,7 @@ export const DefaultWorkletSetting: WorkletSetting = {
volTrancateLength: 32
}
///////////////////////
// Audio Streamerセッティング
// Worklet Nodeセッティング
///////////////////////
export const Protocol = {
"sio": "sio",
@ -153,14 +153,14 @@ export const DownSamplingMode = {
export type DownSamplingMode = typeof DownSamplingMode[keyof typeof DownSamplingMode]
export type AudioStreamerSetting = {
export type WorkletNodeSetting = {
serverUrl: string,
protocol: Protocol,
sendingSampleRate: SendingSampleRate,
inputChunkNum: number,
downSamplingMode: DownSamplingMode,
}
export const DefaultAudioStreamerSetting: AudioStreamerSetting = {
export const DefaultWorkletNodeSetting: WorkletNodeSetting = {
serverUrl: "",
protocol: "sio",
sendingSampleRate: 48000,
@ -265,7 +265,7 @@ export const INDEXEDDB_DB_APP_NAME = "INDEXEDDB_KEY_VOICE_CHANGER"
export const INDEXEDDB_DB_NAME = "INDEXEDDB_KEY_VOICE_CHANGER_DB"
export const INDEXEDDB_KEY_CLIENT = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_CLIENT"
export const INDEXEDDB_KEY_SERVER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_SERVER"
export const INDEXEDDB_KEY_STREAMER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_STREAMER"
export const INDEXEDDB_KEY_WORKLETNODE = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_WORKLETNODE"
export const INDEXEDDB_KEY_MODEL_DATA = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_MODEL_DATA"
export const INDEXEDDB_KEY_WORKLET = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_WORKLET"

74
client/lib/src/hooks/useAudioStreamerSetting.ts
View File

@ -1,74 +0,0 @@
import { useState, useMemo, useEffect } from "react"
import { INDEXEDDB_KEY_STREAMER, AudioStreamerSetting, DefaultAudioStreamerSetting } from "../const"
import { VoiceChangerClient } from "../VoiceChangerClient"
import { useIndexedDB } from "./useIndexedDB"
export type UseAudioStreamerSettingProps = {
voiceChangerClient: VoiceChangerClient | null
}
export type AudioStreamerSettingState = {
audioStreamerSetting: AudioStreamerSetting;
clearSetting: () => Promise<void>
updateAudioStreamerSetting: (setting: AudioStreamerSetting) => void
}
export const useAudioStreamerSetting = (props: UseAudioStreamerSettingProps): AudioStreamerSettingState => {
const [audioStreamerSetting, _setAudioStreamerSetting] = useState<AudioStreamerSetting>(DefaultAudioStreamerSetting)
const { setItem, getItem, removeItem } = useIndexedDB()
// 初期化 その１ DBから取得
useEffect(() => {
const loadCache = async () => {
const setting = await getItem(INDEXEDDB_KEY_STREAMER) as AudioStreamerSetting
if (setting) {
_setAudioStreamerSetting(setting)
}
}
loadCache()
}, [])
// 初期化 その２ クライアントに設定
useEffect(() => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setServerUrl(audioStreamerSetting.serverUrl)
props.voiceChangerClient.updateAudioStreamerSetting(audioStreamerSetting)
}, [props.voiceChangerClient])
const clearSetting = async () => {
await removeItem(INDEXEDDB_KEY_STREAMER)
}
//////////////
// 設定
/////////////
const updateAudioStreamerSetting = useMemo(() => {
return (_audioStreamerSetting: AudioStreamerSetting) => {
if (!props.voiceChangerClient) return
for (let k in _audioStreamerSetting) {
const cur_v = audioStreamerSetting[k]
const new_v = _audioStreamerSetting[k]
if (cur_v != new_v) {
_setAudioStreamerSetting(_audioStreamerSetting)
setItem(INDEXEDDB_KEY_STREAMER, _audioStreamerSetting)
props.voiceChangerClient.updateAudioStreamerSetting(_audioStreamerSetting)
break
}
}
}
}, [props.voiceChangerClient, audioStreamerSetting])
return {
audioStreamerSetting,
clearSetting,
updateAudioStreamerSetting,
}
}

14
client/lib/src/hooks/useClient.ts
View File

@ -1,9 +1,8 @@
import { useEffect, useMemo, useRef, useState } from "react"
import { AudioStreamer } from "../AudioStreamer"
import { VoiceChangerClient } from "../VoiceChangerClient"
import { AudioStreamerSettingState, useAudioStreamerSetting } from "./useAudioStreamerSetting"
import { ClientSettingState, useClientSetting } from "./useClientSetting"
import { ServerSettingState, useServerSetting } from "./useServerSetting"
import { useWorkletNodeSetting, WorkletNodeSettingState } from "./useWorkletNodeSetting"
import { useWorkletSetting, WorkletSettingState } from "./useWorkletSetting"
export type UseClientProps = {
@ -15,7 +14,7 @@ export type ClientState = {
// 各種設定I/Fへの参照
workletSetting: WorkletSettingState
clientSetting: ClientSettingState
streamerSetting: AudioStreamerSettingState
workletNodeSetting: WorkletNodeSettingState
serverSetting: ServerSettingState
// モニタリングデータ
@ -48,7 +47,7 @@ export const useClient = (props: UseClientProps): ClientState => {
// (1-2) 各種設定I/F
const clientSetting = useClientSetting({ voiceChangerClient, audioContext: props.audioContext })
const streamerSetting = useAudioStreamerSetting({ voiceChangerClient })
const workletNodeSetting = useWorkletNodeSetting({ voiceChangerClient })
const workletSetting = useWorkletSetting({ voiceChangerClient })
const serverSetting = useServerSetting({ voiceChangerClient })
@ -83,8 +82,7 @@ export const useClient = (props: UseClientProps): ClientState => {
errorCountRef.current = 0
}
}
}
}, {
},
notifyVolume: (vol: number) => {
setVolume(vol)
},
@ -119,7 +117,7 @@ export const useClient = (props: UseClientProps): ClientState => {
const clearSetting = async () => {
await clientSetting.clearSetting()
await streamerSetting.clearSetting()
await workletNodeSetting.clearSetting()
await workletSetting.clearSetting()
await serverSetting.clearSetting()
}
@ -127,7 +125,7 @@ export const useClient = (props: UseClientProps): ClientState => {
return {
// 各種設定I/Fへの参照
clientSetting,
streamerSetting,
workletNodeSetting,
workletSetting,
serverSetting,

73
client/lib/src/hooks/useWorkletNodeSetting.ts Normal file
View File

@ -0,0 +1,73 @@
import { useState, useMemo, useEffect } from "react"
import { DefaultWorkletNodeSetting, INDEXEDDB_KEY_WORKLETNODE, WorkletNodeSetting } from "../const"
import { VoiceChangerClient } from "../VoiceChangerClient"
import { useIndexedDB } from "./useIndexedDB"
export type UseWorkletNodeSettingProps = {
voiceChangerClient: VoiceChangerClient | null
}
export type WorkletNodeSettingState = {
workletNodeSetting: WorkletNodeSetting;
clearSetting: () => Promise<void>
updateWorkletNodeSetting: (setting: WorkletNodeSetting) => void
}
export const useWorkletNodeSetting = (props: UseWorkletNodeSettingProps): WorkletNodeSettingState => {
const [workletNodeSetting, _setWorkletNodeSetting] = useState<WorkletNodeSetting>(DefaultWorkletNodeSetting)
const { setItem, getItem, removeItem } = useIndexedDB()
// 初期化 その１ DBから取得
useEffect(() => {
const loadCache = async () => {
const setting = await getItem(INDEXEDDB_KEY_WORKLETNODE) as WorkletNodeSetting
if (setting) {
_setWorkletNodeSetting(setting)
}
}
loadCache()
}, [])
// 初期化 その２ クライアントに設定
useEffect(() => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setServerUrl(workletNodeSetting.serverUrl)
props.voiceChangerClient.updateWorkletNodeSetting(workletNodeSetting)
}, [props.voiceChangerClient])
const clearSetting = async () => {
await removeItem(INDEXEDDB_KEY_WORKLETNODE)
}
//////////////
// 設定
/////////////
const updateWorkletNodeSetting = useMemo(() => {
return (_workletNodeSetting: WorkletNodeSetting) => {
if (!props.voiceChangerClient) return
for (let k in _workletNodeSetting) {
const cur_v = workletNodeSetting[k]
const new_v = _workletNodeSetting[k]
if (cur_v != new_v) {
_setWorkletNodeSetting(_workletNodeSetting)
setItem(INDEXEDDB_KEY_WORKLETNODE, _workletNodeSetting)
props.voiceChangerClient.updateWorkletNodeSetting(_workletNodeSetting)
break
}
}
}
}, [props.voiceChangerClient, workletNodeSetting])
return {
workletNodeSetting,
clearSetting,
updateWorkletNodeSetting,
}
}

32
client/lib/src/hooks/useWorkletSetting.ts
View File

@ -11,8 +11,8 @@ export type WorkletSettingState = {
setting: WorkletSetting;
clearSetting: () => Promise<void>
setSetting: (setting: WorkletSetting) => void;
startOutputRecording: () => void
stopOutputRecording: () => Promise<void>
// startOutputRecording: () => void
// stopOutputRecording: () => Promise<void>
}
export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSettingState => {
@ -68,26 +68,26 @@ export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSetting
await removeItem(INDEXEDDB_KEY_WORKLET)
}
const startOutputRecording = useMemo(() => {
return () => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.startOutputRecordingWorklet()
}
}, [props.voiceChangerClient])
// const startOutputRecording = useMemo(() => {
// return () => {
// if (!props.voiceChangerClient) return
// props.voiceChangerClient.startOutputRecordingWorklet()
// }
// }, [props.voiceChangerClient])
const stopOutputRecording = useMemo(() => {
return async () => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.stopOutputRecordingWorklet()
}
}, [props.voiceChangerClient])
// const stopOutputRecording = useMemo(() => {
// return async () => {
// if (!props.voiceChangerClient) return
// props.voiceChangerClient.stopOutputRecordingWorklet()
// }
// }, [props.voiceChangerClient])
return {
setting,
clearSetting,
setSetting,
startOutputRecording,
stopOutputRecording
// startOutputRecording,
// stopOutputRecording
}
}

19
client/lib/worklet/src/voice-changer-worklet-processor.ts
View File

@ -9,7 +9,6 @@ export type RequestType = typeof RequestType[keyof typeof RequestType]
export const ResponseType = {
"volume": "volume",
"recordData": "recordData",
"inputData": "inputData"
} as const
export type ResponseType = typeof ResponseType[keyof typeof ResponseType]
@ -43,7 +42,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
private isRecording = false
playBuffer: Float32Array[] = []
recordingBuffer: Float32Array[] = []
/**
* @constructor
*/
@ -75,7 +73,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
return
}
this.isRecording = true
this.recordingBuffer = []
return
} else if (request.requestType === "stopRecording") {
if (!this.isRecording) {
@ -83,13 +80,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
return
}
this.isRecording = false
const recordResponse: VoiceChangerWorkletProcessorResponse = {
responseType: ResponseType.recordData,
recordData: this.recordingBuffer
}
this.port.postMessage(recordResponse);
this.recordingBuffer = []
return
}
@ -126,9 +116,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
f32Block![frameIndexInBlock + 1] = (currentFrame + nextFrame) / 2
if (f32Block!.length === frameIndexInBlock + 2) {
this.playBuffer.push(f32Block!)
if (this.isRecording) {
this.recordingBuffer.push(f32Block!)
}
}
}
}
@ -148,8 +135,10 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
return true;
}
if (_inputs.length > 0 && _inputs[0].length > 0) {
this.pushData(_inputs[0][0])
if (this.isRecording) {
if (_inputs.length > 0 && _inputs[0].length > 0) {
this.pushData(_inputs[0][0])
}
}
if (this.playBuffer.length === 0) {