EXP. remove microphone stream 2
This commit is contained in:
wataru
parent
de0be188ff
commit
bde893b665
6
client/demo/dist/index.js
vendored
6
client/demo/dist/index.js
vendored
File diff suppressed because one or more lines are too long
@ -25,18 +25,18 @@ export const useConvertSetting = (): ConvertSettingState => {
|
||||
<div className="body-row split-3-2-1-4 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Input Chunk Num(128sample/chunk)</div>
|
||||
<div className="body-input-container">
|
||||
<input type="number" min={1} max={256} step={1} value={appState.streamerSetting.audioStreamerSetting.inputChunkNum} onChange={(e) => {
|
||||
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, inputChunkNum: Number(e.target.value) })
|
||||
<input type="number" min={1} max={256} step={1} value={appState.workletNodeSetting.workletNodeSetting.inputChunkNum} onChange={(e) => {
|
||||
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, inputChunkNum: Number(e.target.value) })
|
||||
}} />
|
||||
</div>
|
||||
<div className="body-item-text">
|
||||
<div>buff: {(appState.streamerSetting.audioStreamerSetting.inputChunkNum * 128 * 1000 / 24000).toFixed(1)}ms</div>
|
||||
<div>buff: {(appState.workletNodeSetting.workletNodeSetting.inputChunkNum * 128 * 1000 / 48000).toFixed(1)}ms</div>
|
||||
</div>
|
||||
<div className="body-item-text"></div>
|
||||
|
||||
</div>
|
||||
)
|
||||
}, [appState.streamerSetting.audioStreamerSetting.inputChunkNum, appState.streamerSetting.updateAudioStreamerSetting])
|
||||
}, [appState.workletNodeSetting.workletNodeSetting.inputChunkNum, appState.workletNodeSetting.updateWorkletNodeSetting])
|
||||
|
||||
const gpuRow = useMemo(() => {
|
||||
return (
|
||||
|
||||
@ -30,24 +30,24 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
<div className="body-row split-3-3-4 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">MMVC Server</div>
|
||||
<div className="body-input-container">
|
||||
<input type="text" defaultValue={appState.streamerSetting.audioStreamerSetting.serverUrl} id="mmvc-server-url" className="body-item-input" />
|
||||
<input type="text" defaultValue={appState.workletNodeSetting.workletNodeSetting.serverUrl} id="mmvc-server-url" className="body-item-input" />
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
<div className="body-button" onClick={onSetServerClicked}>set</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.streamerSetting.audioStreamerSetting.serverUrl, appState.clientSetting.setServerUrl])
|
||||
}, [appState.workletNodeSetting.workletNodeSetting.serverUrl, appState.clientSetting.setServerUrl])
|
||||
|
||||
const protocolRow = useMemo(() => {
|
||||
const onProtocolChanged = async (val: Protocol) => {
|
||||
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, protocol: val })
|
||||
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, protocol: val })
|
||||
}
|
||||
return (
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Protocol</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.protocol} onChange={(e) => {
|
||||
<select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.protocol} onChange={(e) => {
|
||||
onProtocolChanged(e.target.value as
|
||||
Protocol)
|
||||
}}>
|
||||
@ -60,7 +60,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.streamerSetting.audioStreamerSetting.protocol, appState.streamerSetting.updateAudioStreamerSetting])
|
||||
}, [appState.workletNodeSetting.workletNodeSetting.protocol, appState.workletNodeSetting.updateWorkletNodeSetting])
|
||||
|
||||
|
||||
const sampleRateRow = useMemo(() => {
|
||||
@ -87,8 +87,8 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Sending Sample Rate</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.sendingSampleRate} onChange={(e) => {
|
||||
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, sendingSampleRate: Number(e.target.value) as InputSampleRate })
|
||||
<select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.sendingSampleRate} onChange={(e) => {
|
||||
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, sendingSampleRate: Number(e.target.value) as InputSampleRate })
|
||||
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, inputSampleRate: Number(e.target.value) as InputSampleRate })
|
||||
}}>
|
||||
{
|
||||
@ -100,7 +100,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.streamerSetting.audioStreamerSetting.sendingSampleRate, appState.streamerSetting.updateAudioStreamerSetting, appState.serverSetting.updateServerSettings])
|
||||
}, [appState.workletNodeSetting.workletNodeSetting.sendingSampleRate, appState.workletNodeSetting.updateWorkletNodeSetting, appState.serverSetting.updateServerSettings])
|
||||
|
||||
const bufferSizeRow = useMemo(() => {
|
||||
return (
|
||||
@ -174,8 +174,8 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1 ">DownSamplingMode</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.downSamplingMode} onChange={(e) => {
|
||||
appState.streamerSetting.updateAudioStreamerSetting({ ...appState.streamerSetting.audioStreamerSetting, downSamplingMode: e.target.value as DownSamplingMode })
|
||||
<select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.downSamplingMode} onChange={(e) => {
|
||||
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, downSamplingMode: e.target.value as DownSamplingMode })
|
||||
}}>
|
||||
{
|
||||
Object.values(DownSamplingMode).map(x => {
|
||||
@ -186,7 +186,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.streamerSetting.audioStreamerSetting.downSamplingMode, appState.streamerSetting.updateAudioStreamerSetting])
|
||||
}, [appState.workletNodeSetting.workletNodeSetting.downSamplingMode, appState.workletNodeSetting.updateWorkletNodeSetting])
|
||||
|
||||
|
||||
const workletSettingRow = useMemo(() => {
|
||||
|
||||
@ -1,243 +1,243 @@
|
||||
import { io, Socket } from "socket.io-client";
|
||||
import { DefaultEventsMap } from "@socket.io/component-emitter";
|
||||
import { Duplex, DuplexOptions } from "readable-stream";
|
||||
import { AudioStreamerSetting, DefaultAudioStreamerSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
|
||||
// import { io, Socket } from "socket.io-client";
|
||||
// import { DefaultEventsMap } from "@socket.io/component-emitter";
|
||||
// import { Duplex, DuplexOptions } from "readable-stream";
|
||||
// import { AudioStreamerSetting, DefaultAudioStreamerSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
|
||||
|
||||
|
||||
export type Callbacks = {
|
||||
onVoiceReceived: (data: ArrayBuffer) => void
|
||||
}
|
||||
export type AudioStreamerListeners = {
|
||||
notifySendBufferingTime: (time: number) => void
|
||||
notifyResponseTime: (time: number) => void
|
||||
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
|
||||
}
|
||||
// export type Callbacks = {
|
||||
// onVoiceReceived: (data: ArrayBuffer) => void
|
||||
// }
|
||||
// export type AudioStreamerListeners = {
|
||||
// notifySendBufferingTime: (time: number) => void
|
||||
// notifyResponseTime: (time: number) => void
|
||||
// notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
|
||||
// }
|
||||
|
||||
export class AudioStreamer extends Duplex {
|
||||
private setting: AudioStreamerSetting = DefaultAudioStreamerSetting
|
||||
// export class AudioStreamer extends Duplex {
|
||||
// private setting: AudioStreamerSetting = DefaultAudioStreamerSetting
|
||||
|
||||
private callbacks: Callbacks
|
||||
private audioStreamerListeners: AudioStreamerListeners
|
||||
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
|
||||
private requestChunks: ArrayBuffer[] = []
|
||||
// private callbacks: Callbacks
|
||||
// private audioStreamerListeners: AudioStreamerListeners
|
||||
// private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
|
||||
// private requestChunks: ArrayBuffer[] = []
|
||||
|
||||
// performance monitor
|
||||
private bufferStart = 0;
|
||||
// // performance monitor
|
||||
// private bufferStart = 0;
|
||||
|
||||
constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
|
||||
super(options);
|
||||
this.callbacks = callbacks
|
||||
this.audioStreamerListeners = audioStreamerListeners
|
||||
this.createSocketIO()
|
||||
}
|
||||
// constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
|
||||
// super(options);
|
||||
// this.callbacks = callbacks
|
||||
// this.audioStreamerListeners = audioStreamerListeners
|
||||
// this.createSocketIO()
|
||||
// }
|
||||
|
||||
private createSocketIO = () => {
|
||||
if (this.socket) {
|
||||
this.socket.close()
|
||||
}
|
||||
if (this.setting.protocol === "sio") {
|
||||
this.socket = io(this.setting.serverUrl + "/test");
|
||||
this.socket.on('connect_error', (err) => {
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
|
||||
})
|
||||
this.socket.on('connect', () => {
|
||||
console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
|
||||
console.log(`[SIO] ${this.socket?.id}`)
|
||||
});
|
||||
this.socket.on('response', (response: any[]) => {
|
||||
const cur = Date.now()
|
||||
const responseTime = cur - response[0]
|
||||
const result = response[1] as ArrayBuffer
|
||||
if (result.byteLength < 128 * 2) {
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
|
||||
} else {
|
||||
this.callbacks.onVoiceReceived(response[1])
|
||||
this.audioStreamerListeners.notifyResponseTime(responseTime)
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
// private createSocketIO = () => {
|
||||
// if (this.socket) {
|
||||
// this.socket.close()
|
||||
// }
|
||||
// if (this.setting.protocol === "sio") {
|
||||
// this.socket = io(this.setting.serverUrl + "/test");
|
||||
// this.socket.on('connect_error', (err) => {
|
||||
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
|
||||
// })
|
||||
// this.socket.on('connect', () => {
|
||||
// console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
|
||||
// console.log(`[SIO] ${this.socket?.id}`)
|
||||
// });
|
||||
// this.socket.on('response', (response: any[]) => {
|
||||
// const cur = Date.now()
|
||||
// const responseTime = cur - response[0]
|
||||
// const result = response[1] as ArrayBuffer
|
||||
// if (result.byteLength < 128 * 2) {
|
||||
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
|
||||
// } else {
|
||||
// this.callbacks.onVoiceReceived(response[1])
|
||||
// this.audioStreamerListeners.notifyResponseTime(responseTime)
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
// Option Change
|
||||
updateSetting = (setting: AudioStreamerSetting) => {
|
||||
console.log(`[AudioStreamer] Updating AudioStreamer Setting,`, this.setting, setting)
|
||||
let recreateSocketIoRequired = false
|
||||
if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
|
||||
recreateSocketIoRequired = true
|
||||
}
|
||||
this.setting = setting
|
||||
if (recreateSocketIoRequired) {
|
||||
this.createSocketIO()
|
||||
}
|
||||
}
|
||||
// // Option Change
|
||||
// updateSetting = (setting: AudioStreamerSetting) => {
|
||||
// console.log(`[AudioStreamer] Updating AudioStreamer Setting,`, this.setting, setting)
|
||||
// let recreateSocketIoRequired = false
|
||||
// if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
|
||||
// recreateSocketIoRequired = true
|
||||
// }
|
||||
// this.setting = setting
|
||||
// if (recreateSocketIoRequired) {
|
||||
// this.createSocketIO()
|
||||
// }
|
||||
// }
|
||||
|
||||
getSettings = (): AudioStreamerSetting => {
|
||||
return this.setting
|
||||
}
|
||||
// getSettings = (): AudioStreamerSetting => {
|
||||
// return this.setting
|
||||
// }
|
||||
|
||||
getSocketId = () => {
|
||||
return this.socket?.id
|
||||
}
|
||||
// getSocketId = () => {
|
||||
// return this.socket?.id
|
||||
// }
|
||||
|
||||
// Main Process
|
||||
//// Pipe from mic stream
|
||||
_write = (chunk: AudioBuffer, _encoding: any, callback: any) => {
|
||||
const buffer = chunk.getChannelData(0);
|
||||
this._write_realtime(buffer)
|
||||
callback();
|
||||
}
|
||||
// // Main Process
|
||||
// //// Pipe from mic stream
|
||||
// _write = (chunk: AudioBuffer, _encoding: any, callback: any) => {
|
||||
// const buffer = chunk.getChannelData(0);
|
||||
// this._write_realtime(buffer)
|
||||
// callback();
|
||||
// }
|
||||
|
||||
_averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
|
||||
if (originalSampleRate == destinationSamplerate) {
|
||||
return buffer;
|
||||
}
|
||||
if (destinationSamplerate > originalSampleRate) {
|
||||
throw "downsampling rate show be smaller than original sample rate";
|
||||
}
|
||||
const sampleRateRatio = originalSampleRate / destinationSamplerate;
|
||||
const newLength = Math.round(buffer.length / sampleRateRatio);
|
||||
const result = new Float32Array(newLength);
|
||||
let offsetResult = 0;
|
||||
let offsetBuffer = 0;
|
||||
while (offsetResult < result.length) {
|
||||
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
|
||||
// Use average value of skipped samples
|
||||
var accum = 0, count = 0;
|
||||
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
|
||||
accum += buffer[i];
|
||||
count++;
|
||||
}
|
||||
result[offsetResult] = accum / count;
|
||||
// Or you can simply get rid of the skipped samples:
|
||||
// result[offsetResult] = buffer[nextOffsetBuffer];
|
||||
offsetResult++;
|
||||
offsetBuffer = nextOffsetBuffer;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
// _averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
|
||||
// if (originalSampleRate == destinationSamplerate) {
|
||||
// return buffer;
|
||||
// }
|
||||
// if (destinationSamplerate > originalSampleRate) {
|
||||
// throw "downsampling rate show be smaller than original sample rate";
|
||||
// }
|
||||
// const sampleRateRatio = originalSampleRate / destinationSamplerate;
|
||||
// const newLength = Math.round(buffer.length / sampleRateRatio);
|
||||
// const result = new Float32Array(newLength);
|
||||
// let offsetResult = 0;
|
||||
// let offsetBuffer = 0;
|
||||
// while (offsetResult < result.length) {
|
||||
// var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
|
||||
// // Use average value of skipped samples
|
||||
// var accum = 0, count = 0;
|
||||
// for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
|
||||
// accum += buffer[i];
|
||||
// count++;
|
||||
// }
|
||||
// result[offsetResult] = accum / count;
|
||||
// // Or you can simply get rid of the skipped samples:
|
||||
// // result[offsetResult] = buffer[nextOffsetBuffer];
|
||||
// offsetResult++;
|
||||
// offsetBuffer = nextOffsetBuffer;
|
||||
// }
|
||||
// return result;
|
||||
// }
|
||||
|
||||
|
||||
private _write_realtime = async (buffer: Float32Array) => {
|
||||
// private _write_realtime = async (buffer: Float32Array) => {
|
||||
|
||||
let downsampledBuffer: Float32Array | null = null
|
||||
if (this.setting.sendingSampleRate == 48000) {
|
||||
downsampledBuffer = buffer
|
||||
} else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
|
||||
//////// (Kind 1) 間引き //////////
|
||||
// bufferSize個のデータ（48Khz）が入ってくる。
|
||||
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。
|
||||
downsampledBuffer = new Float32Array(buffer.length / 2);
|
||||
for (let i = 0; i < buffer.length; i++) {
|
||||
if (i % 2 == 0) {
|
||||
downsampledBuffer[i / 2] = buffer[i]
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//////// (Kind 2) 平均 //////////
|
||||
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
|
||||
downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, this.setting.sendingSampleRate)
|
||||
}
|
||||
// let downsampledBuffer: Float32Array | null = null
|
||||
// if (this.setting.sendingSampleRate == 48000) {
|
||||
// downsampledBuffer = buffer
|
||||
// } else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
|
||||
// //////// (Kind 1) 間引き //////////
|
||||
// // bufferSize個のデータ（48Khz）が入ってくる。
|
||||
// //// 48000Hz で入ってくるので間引いて24000Hzに変換する。
|
||||
// downsampledBuffer = new Float32Array(buffer.length / 2);
|
||||
// for (let i = 0; i < buffer.length; i++) {
|
||||
// if (i % 2 == 0) {
|
||||
// downsampledBuffer[i / 2] = buffer[i]
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// //////// (Kind 2) 平均 //////////
|
||||
// // downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
|
||||
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, this.setting.sendingSampleRate)
|
||||
// }
|
||||
|
||||
// Float to signed16
|
||||
const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2)
|
||||
const dataView = new DataView(arrayBuffer);
|
||||
for (let i = 0; i < downsampledBuffer.length; i++) {
|
||||
let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
|
||||
s = s < 0 ? s * 0x8000 : s * 0x7FFF
|
||||
dataView.setInt16(i * 2, s, true);
|
||||
}
|
||||
// // Float to signed16
|
||||
// const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2)
|
||||
// const dataView = new DataView(arrayBuffer);
|
||||
// for (let i = 0; i < downsampledBuffer.length; i++) {
|
||||
// let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
|
||||
// s = s < 0 ? s * 0x8000 : s * 0x7FFF
|
||||
// dataView.setInt16(i * 2, s, true);
|
||||
// }
|
||||
|
||||
|
||||
// 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理
|
||||
// const chunkByteSize = 256 // (const.ts ★1)
|
||||
// const chunkByteSize = 256 * 2 // (const.ts ★1)
|
||||
const chunkByteSize = (256 * 2) * (this.setting.sendingSampleRate / 48000) // (const.ts ★1)
|
||||
for (let i = 0; i < arrayBuffer.byteLength / chunkByteSize; i++) {
|
||||
const ab = arrayBuffer.slice(i * chunkByteSize, (i + 1) * chunkByteSize)
|
||||
this.requestChunks.push(ab)
|
||||
}
|
||||
// // 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理
|
||||
// // const chunkByteSize = 256 // (const.ts ★1)
|
||||
// // const chunkByteSize = 256 * 2 // (const.ts ★1)
|
||||
// const chunkByteSize = (256 * 2) * (this.setting.sendingSampleRate / 48000) // (const.ts ★1)
|
||||
// for (let i = 0; i < arrayBuffer.byteLength / chunkByteSize; i++) {
|
||||
// const ab = arrayBuffer.slice(i * chunkByteSize, (i + 1) * chunkByteSize)
|
||||
// this.requestChunks.push(ab)
|
||||
// }
|
||||
|
||||
|
||||
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
|
||||
if (this.requestChunks.length < this.setting.inputChunkNum) {
|
||||
return
|
||||
}
|
||||
// //// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
|
||||
// if (this.requestChunks.length < this.setting.inputChunkNum) {
|
||||
// return
|
||||
// }
|
||||
|
||||
// リクエスト用の入れ物を作成
|
||||
const windowByteLength = this.requestChunks.reduce((prev, cur) => {
|
||||
return prev + cur.byteLength
|
||||
}, 0)
|
||||
const newBuffer = new Uint8Array(windowByteLength);
|
||||
// // リクエスト用の入れ物を作成
|
||||
// const windowByteLength = this.requestChunks.reduce((prev, cur) => {
|
||||
// return prev + cur.byteLength
|
||||
// }, 0)
|
||||
// const newBuffer = new Uint8Array(windowByteLength);
|
||||
|
||||
// リクエストのデータをセット
|
||||
this.requestChunks.reduce((prev, cur) => {
|
||||
newBuffer.set(new Uint8Array(cur), prev)
|
||||
return prev + cur.byteLength
|
||||
}, 0)
|
||||
// // リクエストのデータをセット
|
||||
// this.requestChunks.reduce((prev, cur) => {
|
||||
// newBuffer.set(new Uint8Array(cur), prev)
|
||||
// return prev + cur.byteLength
|
||||
// }, 0)
|
||||
|
||||
// console.log("send buff length", newBuffer.length)
|
||||
// // console.log("send buff length", newBuffer.length)
|
||||
|
||||
this.sendBuffer(newBuffer)
|
||||
this.requestChunks = []
|
||||
// this.sendBuffer(newBuffer)
|
||||
// this.requestChunks = []
|
||||
|
||||
this.audioStreamerListeners.notifySendBufferingTime(Date.now() - this.bufferStart)
|
||||
this.bufferStart = Date.now()
|
||||
}
|
||||
// this.audioStreamerListeners.notifySendBufferingTime(Date.now() - this.bufferStart)
|
||||
// this.bufferStart = Date.now()
|
||||
// }
|
||||
|
||||
private sendBuffer = async (newBuffer: Uint8Array) => {
|
||||
const timestamp = Date.now()
|
||||
if (this.setting.protocol === "sio") {
|
||||
if (!this.socket) {
|
||||
console.warn(`sio is not initialized`)
|
||||
return
|
||||
}
|
||||
// console.log("emit!")
|
||||
this.socket.emit('request_message', [
|
||||
timestamp,
|
||||
newBuffer.buffer]);
|
||||
} else {
|
||||
const res = await postVoice(
|
||||
this.setting.serverUrl + "/test",
|
||||
timestamp,
|
||||
newBuffer.buffer)
|
||||
// private sendBuffer = async (newBuffer: Uint8Array) => {
|
||||
// const timestamp = Date.now()
|
||||
// if (this.setting.protocol === "sio") {
|
||||
// if (!this.socket) {
|
||||
// console.warn(`sio is not initialized`)
|
||||
// return
|
||||
// }
|
||||
// // console.log("emit!")
|
||||
// this.socket.emit('request_message', [
|
||||
// timestamp,
|
||||
// newBuffer.buffer]);
|
||||
// } else {
|
||||
// const res = await postVoice(
|
||||
// this.setting.serverUrl + "/test",
|
||||
// timestamp,
|
||||
// newBuffer.buffer)
|
||||
|
||||
if (res.byteLength < 128 * 2) {
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
|
||||
} else {
|
||||
this.callbacks.onVoiceReceived(res)
|
||||
this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// if (res.byteLength < 128 * 2) {
|
||||
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
|
||||
// } else {
|
||||
// this.callbacks.onVoiceReceived(res)
|
||||
// this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
export const postVoice = async (
|
||||
url: string,
|
||||
timestamp: number,
|
||||
buffer: ArrayBuffer) => {
|
||||
const obj = {
|
||||
timestamp,
|
||||
buffer: Buffer.from(buffer).toString('base64')
|
||||
};
|
||||
const body = JSON.stringify(obj);
|
||||
// export const postVoice = async (
|
||||
// url: string,
|
||||
// timestamp: number,
|
||||
// buffer: ArrayBuffer) => {
|
||||
// const obj = {
|
||||
// timestamp,
|
||||
// buffer: Buffer.from(buffer).toString('base64')
|
||||
// };
|
||||
// const body = JSON.stringify(obj);
|
||||
|
||||
const res = await fetch(`${url}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: body
|
||||
})
|
||||
// const res = await fetch(`${url}`, {
|
||||
// method: "POST",
|
||||
// headers: {
|
||||
// 'Accept': 'application/json',
|
||||
// 'Content-Type': 'application/json'
|
||||
// },
|
||||
// body: body
|
||||
// })
|
||||
|
||||
const receivedJson = await res.json()
|
||||
const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
|
||||
const buf = Buffer.from(changedVoiceBase64, "base64")
|
||||
const ab = new ArrayBuffer(buf.length);
|
||||
const view = new Uint8Array(ab);
|
||||
for (let i = 0; i < buf.length; ++i) {
|
||||
view[i] = buf[i];
|
||||
}
|
||||
return ab
|
||||
}
|
||||
// const receivedJson = await res.json()
|
||||
// const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
|
||||
// const buf = Buffer.from(changedVoiceBase64, "base64")
|
||||
// const ab = new ArrayBuffer(buf.length);
|
||||
// const view = new Uint8Array(ab);
|
||||
// for (let i = 0; i < buf.length; ++i) {
|
||||
// view[i] = buf[i];
|
||||
// }
|
||||
// return ab
|
||||
// }
|
||||
@ -3,14 +3,12 @@ import { VoiceChangerWorkletNode, VoiceChangerWorkletListener } from "./VoiceCha
|
||||
import workerjs from "raw-loader!../worklet/dist/index.js";
|
||||
import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
|
||||
import { createDummyMediaStream, validateUrl } from "./util";
|
||||
import { AudioStreamerSetting, DefaultVoiceChangerClientSetting, ServerSettingKey, VoiceChangerClientSetting, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletSetting } from "./const";
|
||||
import MicrophoneStream from "microphone-stream";
|
||||
import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer";
|
||||
import { DefaultVoiceChangerClientSetting, ServerSettingKey, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "./const";
|
||||
import { ServerConfigurator } from "./ServerConfigurator";
|
||||
|
||||
// オーディオデータの流れ
|
||||
// input node(mic or MediaStream) -> [vf node] -> microphne stream -> audio streamer ->
|
||||
// sio/rest server -> audio streamer-> vc node -> output node
|
||||
// input node(mic or MediaStream) -> [vf node] -> [vc node] ->
|
||||
// sio/rest server -> [vc node] -> output node
|
||||
|
||||
import { BlockingQueue } from "./utils/BlockingQueue";
|
||||
|
||||
@ -23,11 +21,8 @@ export class VoiceChangerClient {
|
||||
|
||||
private currentMediaStream: MediaStream | null = null
|
||||
private currentMediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null
|
||||
private outputNodeFromVF: MediaStreamAudioDestinationNode | null = null
|
||||
private inputGainNode: GainNode | null = null
|
||||
private outputGainNode: GainNode | null = null
|
||||
private micStream: MicrophoneStream | null = null
|
||||
private audioStreamer!: AudioStreamer
|
||||
private vcNode!: VoiceChangerWorkletNode
|
||||
private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode
|
||||
|
||||
@ -41,13 +36,7 @@ export class VoiceChangerClient {
|
||||
|
||||
private sem = new BlockingQueue<number>();
|
||||
|
||||
private callbacks: Callbacks = {
|
||||
onVoiceReceived: (data: ArrayBuffer): void => {
|
||||
this.vcNode.postReceivedVoice(data)
|
||||
}
|
||||
}
|
||||
|
||||
constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners, voiceChangerWorkletListener: VoiceChangerWorkletListener) {
|
||||
constructor(ctx: AudioContext, vfEnable: boolean, voiceChangerWorkletListener: VoiceChangerWorkletListener) {
|
||||
this.sem.enqueue(0);
|
||||
this.configurator = new ServerConfigurator()
|
||||
this.ctx = ctx
|
||||
@ -62,14 +51,11 @@ export class VoiceChangerClient {
|
||||
this.outputGainNode.gain.value = this.setting.outputGain
|
||||
this.vcNode.connect(this.outputGainNode) // vc node -> output node
|
||||
this.outputGainNode.connect(this.currentMediaStreamAudioDestinationNode)
|
||||
// (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる)
|
||||
this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, })
|
||||
|
||||
if (this.vfEnable) {
|
||||
this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' })
|
||||
const dummyMediaStream = createDummyMediaStream(this.ctx)
|
||||
this.currentDevice = (await this.vf.createTransformDevice(dummyMediaStream)) || null;
|
||||
this.outputNodeFromVF = this.ctx.createMediaStreamDestination();
|
||||
}
|
||||
resolve()
|
||||
})
|
||||
@ -94,7 +80,6 @@ export class VoiceChangerClient {
|
||||
// オペレーション
|
||||
/////////////////////////////////////////////////////
|
||||
/// Operations ///
|
||||
// setup = async (input: string | MediaStream | null, bufferSize: BufferSize, echoCancel: boolean = true, noiseSuppression: boolean = true, noiseSuppression2: boolean = false) => {
|
||||
setup = async () => {
|
||||
const lockNum = await this.lock()
|
||||
|
||||
@ -115,9 +100,7 @@ export class VoiceChangerClient {
|
||||
//// Input デバイスがnullの時はmicStreamを止めてリターン
|
||||
if (!this.setting.audioInput) {
|
||||
console.log(`Input Setup=> client mic is disabled.`)
|
||||
if (this.micStream) {
|
||||
this.micStream.pauseRecording()
|
||||
}
|
||||
this.vcNode.stopRecording()
|
||||
await this.unlock(lockNum)
|
||||
return
|
||||
}
|
||||
@ -143,17 +126,6 @@ export class VoiceChangerClient {
|
||||
this.currentMediaStream = this.setting.audioInput
|
||||
}
|
||||
|
||||
// create mic stream
|
||||
if (this.micStream) {
|
||||
this.micStream.unpipe()
|
||||
this.micStream.destroy()
|
||||
this.micStream = null
|
||||
}
|
||||
this.micStream = new MicrophoneStream({
|
||||
objectMode: true,
|
||||
bufferSize: this.setting.bufferSize,
|
||||
context: this.ctx
|
||||
})
|
||||
// connect nodes.
|
||||
this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream)
|
||||
this.inputGainNode = this.ctx.createGain()
|
||||
@ -163,11 +135,8 @@ export class VoiceChangerClient {
|
||||
this.currentDevice.chooseNewInnerDevice(this.currentMediaStream)
|
||||
const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node
|
||||
this.inputGainNode.connect(voiceFocusNode.start) // input node -> vf node
|
||||
voiceFocusNode.end.connect(this.outputNodeFromVF!)
|
||||
// this.micStream.setStream(this.outputNodeFromVF!.stream) // vf node -> mic stream
|
||||
voiceFocusNode.end.connect(this.vcNode)
|
||||
} else {
|
||||
// const inputDestinationNodeForMicStream = this.ctx.createMediaStreamDestination()
|
||||
// this.inputGainNode.connect(inputDestinationNodeForMicStream)
|
||||
console.log("input___ media stream", this.currentMediaStream)
|
||||
this.currentMediaStream.getTracks().forEach(x => {
|
||||
console.log("input___ media stream set", x.getSettings())
|
||||
@ -177,17 +146,6 @@ export class VoiceChangerClient {
|
||||
console.log("input___ media node", this.currentMediaStreamAudioSourceNode)
|
||||
console.log("input___ gain node", this.inputGainNode.channelCount, this.inputGainNode)
|
||||
this.inputGainNode.connect(this.vcNode)
|
||||
|
||||
|
||||
|
||||
|
||||
// this.micStream.setStream(inputDestinationNodeForMicStream.stream) // input device -> mic stream
|
||||
}
|
||||
this.micStream.pipe(this.audioStreamer) // mic stream -> audio streamer
|
||||
if (!this._isVoiceChanging) {
|
||||
this.micStream.pauseRecording()
|
||||
} else {
|
||||
this.micStream.playRecording()
|
||||
}
|
||||
console.log("Input Setup=> success")
|
||||
await this.unlock(lockNum)
|
||||
@ -197,18 +155,14 @@ export class VoiceChangerClient {
|
||||
}
|
||||
|
||||
start = () => {
|
||||
if (!this.micStream) {
|
||||
throw `Exception:${VOICE_CHANGER_CLIENT_EXCEPTION.ERR_MIC_STREAM_NOT_INITIALIZED}`
|
||||
return
|
||||
}
|
||||
this.micStream.playRecording()
|
||||
this.vcNode.startRecording()
|
||||
this._isVoiceChanging = true
|
||||
}
|
||||
stop = () => {
|
||||
if (!this.micStream) { return }
|
||||
this.micStream.pauseRecording()
|
||||
this.vcNode.stopRecording()
|
||||
this._isVoiceChanging = false
|
||||
}
|
||||
|
||||
get isVoiceChanging(): boolean {
|
||||
return this._isVoiceChanging
|
||||
}
|
||||
@ -231,7 +185,7 @@ export class VoiceChangerClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
this.audioStreamer.updateSetting({ ...this.audioStreamer.getSettings(), serverUrl: url })
|
||||
this.vcNode.updateSetting({ ...this.vcNode.getSettings(), serverUrl: url })
|
||||
this.configurator.setServerUrl(url)
|
||||
}
|
||||
|
||||
@ -260,9 +214,6 @@ export class VoiceChangerClient {
|
||||
if (reconstructInputRequired) {
|
||||
this.setup()
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
setInputGain = (val: number) => {
|
||||
@ -301,17 +252,17 @@ export class VoiceChangerClient {
|
||||
configureWorklet = (setting: WorkletSetting) => {
|
||||
this.vcNode.configure(setting)
|
||||
}
|
||||
startOutputRecordingWorklet = () => {
|
||||
this.vcNode.startOutputRecordingWorklet()
|
||||
startRecording = () => {
|
||||
this.vcNode.startRecording()
|
||||
}
|
||||
stopOutputRecordingWorklet = () => {
|
||||
this.vcNode.stopOutputRecordingWorklet()
|
||||
stopRecording = () => {
|
||||
this.vcNode.stopRecording()
|
||||
}
|
||||
|
||||
|
||||
//## Audio Streamer ##//
|
||||
updateAudioStreamerSetting = (setting: AudioStreamerSetting) => {
|
||||
this.audioStreamer.updateSetting(setting)
|
||||
//## Worklet Node ##//
|
||||
updateWorkletNodeSetting = (setting: WorkletNodeSetting) => {
|
||||
this.vcNode.updateSetting(setting)
|
||||
}
|
||||
|
||||
|
||||
@ -320,16 +271,14 @@ export class VoiceChangerClient {
|
||||
/////////////////////////////////////////////////////
|
||||
// Information
|
||||
getClientSettings = () => {
|
||||
return this.audioStreamer.getSettings()
|
||||
return this.vcNode.getSettings()
|
||||
}
|
||||
getServerSettings = () => {
|
||||
return this.configurator.getSettings()
|
||||
}
|
||||
|
||||
|
||||
getSocketId = () => {
|
||||
return this.audioStreamer.getSocketId()
|
||||
return this.vcNode.getSocketId()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,17 +1,25 @@
|
||||
import { VoiceChangerWorkletProcessorRequest } from "./@types/voice-changer-worklet-processor";
|
||||
import { WorkletSetting } from "./const";
|
||||
import { DefaultWorkletNodeSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletNodeSetting, WorkletSetting } from "./const";
|
||||
import { io, Socket } from "socket.io-client";
|
||||
import { DefaultEventsMap } from "@socket.io/component-emitter";
|
||||
|
||||
export type VoiceChangerWorkletListener = {
|
||||
notifyVolume: (vol: number) => void
|
||||
notifyOutputRecordData: (data: Float32Array[]) => void
|
||||
notifySendBufferingTime: (time: number) => void
|
||||
notifyResponseTime: (time: number) => void
|
||||
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
|
||||
}
|
||||
|
||||
export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
private listener: VoiceChangerWorkletListener
|
||||
|
||||
private setting: WorkletNodeSetting = DefaultWorkletNodeSetting
|
||||
private requestChunks: ArrayBuffer[] = []
|
||||
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
|
||||
// performance monitor
|
||||
private bufferStart = 0;
|
||||
|
||||
constructor(context: AudioContext, listener: VoiceChangerWorkletListener) {
|
||||
super(context, "voice-changer-worklet-processor");
|
||||
this.port.onmessage = this.handleMessage.bind(this);
|
||||
@ -20,7 +28,56 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
console.log(`[worklet_node][voice-changer-worklet-processor] created.`);
|
||||
}
|
||||
|
||||
postReceivedVoice = (data: ArrayBuffer) => {
|
||||
// 設定
|
||||
updateSetting = (setting: WorkletNodeSetting) => {
|
||||
console.log(`[WorkletNode] Updating WorkletNode Setting,`, this.setting, setting)
|
||||
let recreateSocketIoRequired = false
|
||||
if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
|
||||
recreateSocketIoRequired = true
|
||||
}
|
||||
this.setting = setting
|
||||
if (recreateSocketIoRequired) {
|
||||
this.createSocketIO()
|
||||
}
|
||||
}
|
||||
|
||||
getSettings = (): WorkletNodeSetting => {
|
||||
return this.setting
|
||||
}
|
||||
|
||||
getSocketId = () => {
|
||||
return this.socket?.id
|
||||
}
|
||||
|
||||
// 処理
|
||||
private createSocketIO = () => {
|
||||
if (this.socket) {
|
||||
this.socket.close()
|
||||
}
|
||||
if (this.setting.protocol === "sio") {
|
||||
this.socket = io(this.setting.serverUrl + "/test");
|
||||
this.socket.on('connect_error', (err) => {
|
||||
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
|
||||
})
|
||||
this.socket.on('connect', () => {
|
||||
console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
|
||||
console.log(`[SIO] ${this.socket?.id}`)
|
||||
});
|
||||
this.socket.on('response', (response: any[]) => {
|
||||
const cur = Date.now()
|
||||
const responseTime = cur - response[0]
|
||||
const result = response[1] as ArrayBuffer
|
||||
if (result.byteLength < 128 * 2) {
|
||||
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
|
||||
} else {
|
||||
this.postReceivedVoice(response[1])
|
||||
this.listener.notifyResponseTime(responseTime)
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private postReceivedVoice = (data: ArrayBuffer) => {
|
||||
const req: VoiceChangerWorkletProcessorRequest = {
|
||||
requestType: "voice",
|
||||
voice: data,
|
||||
@ -31,37 +88,33 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
this.port.postMessage(req)
|
||||
}
|
||||
|
||||
|
||||
private createSocketIO = () => {
|
||||
if (this.socket) {
|
||||
this.socket.close()
|
||||
private _averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
|
||||
if (originalSampleRate == destinationSamplerate) {
|
||||
return buffer;
|
||||
}
|
||||
// if (this.setting.protocol === "sio") {
|
||||
// this.socket = io(this.setting.serverUrl + "/test");
|
||||
this.socket = io("/test");
|
||||
this.socket.on('connect_error', (err) => {
|
||||
console.log("connect exception !!!!!")
|
||||
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
|
||||
})
|
||||
this.socket.on('connect', () => {
|
||||
// console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
|
||||
console.log(`[SIO] ${this.socket?.id}`)
|
||||
});
|
||||
this.socket.on('response', (response: any[]) => {
|
||||
const cur = Date.now()
|
||||
const responseTime = cur - response[0]
|
||||
const result = response[1] as ArrayBuffer
|
||||
if (result.byteLength < 128 * 2) {
|
||||
console.log("tooshort!!")
|
||||
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
|
||||
} else {
|
||||
console.log("response!!!")
|
||||
this.postReceivedVoice(response[1])
|
||||
// this.callbacks.onVoiceReceived(response[1])
|
||||
// this.audioStreamerListeners.notifyResponseTime(responseTime)
|
||||
if (destinationSamplerate > originalSampleRate) {
|
||||
throw "downsampling rate show be smaller than original sample rate";
|
||||
}
|
||||
const sampleRateRatio = originalSampleRate / destinationSamplerate;
|
||||
const newLength = Math.round(buffer.length / sampleRateRatio);
|
||||
const result = new Float32Array(newLength);
|
||||
let offsetResult = 0;
|
||||
let offsetBuffer = 0;
|
||||
while (offsetResult < result.length) {
|
||||
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
|
||||
// Use average value of skipped samples
|
||||
var accum = 0, count = 0;
|
||||
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
|
||||
accum += buffer[i];
|
||||
count++;
|
||||
}
|
||||
});
|
||||
// }
|
||||
result[offsetResult] = accum / count;
|
||||
// Or you can simply get rid of the skipped samples:
|
||||
// result[offsetResult] = buffer[nextOffsetBuffer];
|
||||
offsetResult++;
|
||||
offsetBuffer = nextOffsetBuffer;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
handleMessage(event: any) {
|
||||
@ -74,21 +127,42 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
const inputData = event.data.inputData as Float32Array
|
||||
// console.log("receive input data", inputData)
|
||||
|
||||
const arrayBuffer = new ArrayBuffer(inputData.length * 2)
|
||||
// ダウンサンプリング
|
||||
let downsampledBuffer: Float32Array | null = null
|
||||
if (this.setting.sendingSampleRate == 48000) {
|
||||
downsampledBuffer = inputData
|
||||
} else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
|
||||
//////// (Kind 1) 間引き //////////
|
||||
// bufferSize個のデータ（48Khz）が入ってくる。
|
||||
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。
|
||||
downsampledBuffer = new Float32Array(inputData.length / 2);
|
||||
for (let i = 0; i < inputData.length; i++) {
|
||||
if (i % 2 == 0) {
|
||||
downsampledBuffer[i / 2] = inputData[i]
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//////// (Kind 2) 平均 //////////
|
||||
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
|
||||
downsampledBuffer = this._averageDownsampleBuffer(inputData, 48000, this.setting.sendingSampleRate)
|
||||
}
|
||||
|
||||
// Float to Int16
|
||||
const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2)
|
||||
const dataView = new DataView(arrayBuffer);
|
||||
for (let i = 0; i < inputData.length; i++) {
|
||||
let s = Math.max(-1, Math.min(1, inputData[i]));
|
||||
for (let i = 0; i < downsampledBuffer.length; i++) {
|
||||
let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
|
||||
s = s < 0 ? s * 0x8000 : s * 0x7FFF
|
||||
dataView.setInt16(i * 2, s, true);
|
||||
}
|
||||
|
||||
// バッファリング
|
||||
this.requestChunks.push(arrayBuffer)
|
||||
|
||||
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
|
||||
if (this.requestChunks.length < 32) {
|
||||
if (this.requestChunks.length < this.setting.inputChunkNum) {
|
||||
return
|
||||
}
|
||||
console.log("sending...")
|
||||
|
||||
// リクエスト用の入れ物を作成
|
||||
const windowByteLength = this.requestChunks.reduce((prev, cur) => {
|
||||
@ -104,10 +178,10 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
|
||||
|
||||
this.sendBuffer(newBuffer)
|
||||
console.log("sended...")
|
||||
this.requestChunks = []
|
||||
|
||||
|
||||
this.listener.notifySendBufferingTime(Date.now() - this.bufferStart)
|
||||
this.bufferStart = Date.now()
|
||||
|
||||
} else {
|
||||
console.warn(`[worklet_node][voice-changer-worklet-processor] unknown response ${event.data.responseType}`, event.data)
|
||||
@ -118,28 +192,28 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
|
||||
private sendBuffer = async (newBuffer: Uint8Array) => {
|
||||
const timestamp = Date.now()
|
||||
// if (this.setting.protocol === "sio") {
|
||||
if (!this.socket) {
|
||||
console.warn(`sio is not initialized`)
|
||||
return
|
||||
}
|
||||
// console.log("emit!")
|
||||
this.socket.emit('request_message', [
|
||||
timestamp,
|
||||
newBuffer.buffer]);
|
||||
// } else {
|
||||
// const res = await postVoice(
|
||||
// this.setting.serverUrl + "/test",
|
||||
// timestamp,
|
||||
// newBuffer.buffer)
|
||||
if (this.setting.protocol === "sio") {
|
||||
if (!this.socket) {
|
||||
console.warn(`sio is not initialized`)
|
||||
return
|
||||
}
|
||||
// console.log("emit!")
|
||||
this.socket.emit('request_message', [
|
||||
timestamp,
|
||||
newBuffer.buffer]);
|
||||
} else {
|
||||
const res = await postVoice(
|
||||
this.setting.serverUrl + "/test",
|
||||
timestamp,
|
||||
newBuffer.buffer)
|
||||
|
||||
// if (res.byteLength < 128 * 2) {
|
||||
// this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
|
||||
// } else {
|
||||
// this.callbacks.onVoiceReceived(res)
|
||||
// this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
|
||||
// }
|
||||
// }
|
||||
if (res.byteLength < 128 * 2) {
|
||||
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
|
||||
} else {
|
||||
this.postReceivedVoice(res)
|
||||
this.listener.notifyResponseTime(Date.now() - timestamp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -154,7 +228,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
this.port.postMessage(req)
|
||||
}
|
||||
|
||||
startOutputRecordingWorklet = () => {
|
||||
startRecording = () => {
|
||||
const req: VoiceChangerWorkletProcessorRequest = {
|
||||
requestType: "startRecording",
|
||||
voice: new ArrayBuffer(1),
|
||||
@ -165,7 +239,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
this.port.postMessage(req)
|
||||
|
||||
}
|
||||
stopOutputRecordingWorklet = () => {
|
||||
stopRecording = () => {
|
||||
const req: VoiceChangerWorkletProcessorRequest = {
|
||||
requestType: "stopRecording",
|
||||
voice: new ArrayBuffer(1),
|
||||
@ -176,3 +250,34 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
this.port.postMessage(req)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export const postVoice = async (
|
||||
url: string,
|
||||
timestamp: number,
|
||||
buffer: ArrayBuffer) => {
|
||||
const obj = {
|
||||
timestamp,
|
||||
buffer: Buffer.from(buffer).toString('base64')
|
||||
};
|
||||
const body = JSON.stringify(obj);
|
||||
|
||||
const res = await fetch(`${url}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: body
|
||||
})
|
||||
|
||||
const receivedJson = await res.json()
|
||||
const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
|
||||
const buf = Buffer.from(changedVoiceBase64, "base64")
|
||||
const ab = new ArrayBuffer(buf.length);
|
||||
const view = new Uint8Array(ab);
|
||||
for (let i = 0; i < buf.length; ++i) {
|
||||
view[i] = buf[i];
|
||||
}
|
||||
return ab
|
||||
}
|
||||
@ -132,7 +132,7 @@ export const DefaultWorkletSetting: WorkletSetting = {
|
||||
volTrancateLength: 32
|
||||
}
|
||||
///////////////////////
|
||||
// Audio Streamerセッティング
|
||||
// Worklet Nodeセッティング
|
||||
///////////////////////
|
||||
export const Protocol = {
|
||||
"sio": "sio",
|
||||
@ -153,14 +153,14 @@ export const DownSamplingMode = {
|
||||
export type DownSamplingMode = typeof DownSamplingMode[keyof typeof DownSamplingMode]
|
||||
|
||||
|
||||
export type AudioStreamerSetting = {
|
||||
export type WorkletNodeSetting = {
|
||||
serverUrl: string,
|
||||
protocol: Protocol,
|
||||
sendingSampleRate: SendingSampleRate,
|
||||
inputChunkNum: number,
|
||||
downSamplingMode: DownSamplingMode,
|
||||
}
|
||||
export const DefaultAudioStreamerSetting: AudioStreamerSetting = {
|
||||
export const DefaultWorkletNodeSetting: WorkletNodeSetting = {
|
||||
serverUrl: "",
|
||||
protocol: "sio",
|
||||
sendingSampleRate: 48000,
|
||||
@ -265,7 +265,7 @@ export const INDEXEDDB_DB_APP_NAME = "INDEXEDDB_KEY_VOICE_CHANGER"
|
||||
export const INDEXEDDB_DB_NAME = "INDEXEDDB_KEY_VOICE_CHANGER_DB"
|
||||
export const INDEXEDDB_KEY_CLIENT = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_CLIENT"
|
||||
export const INDEXEDDB_KEY_SERVER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_SERVER"
|
||||
export const INDEXEDDB_KEY_STREAMER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_STREAMER"
|
||||
export const INDEXEDDB_KEY_WORKLETNODE = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_WORKLETNODE"
|
||||
export const INDEXEDDB_KEY_MODEL_DATA = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_MODEL_DATA"
|
||||
export const INDEXEDDB_KEY_WORKLET = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_WORKLET"
|
||||
|
||||
|
||||
@ -1,74 +0,0 @@
|
||||
import { useState, useMemo, useEffect } from "react"
|
||||
|
||||
import { INDEXEDDB_KEY_STREAMER, AudioStreamerSetting, DefaultAudioStreamerSetting } from "../const"
|
||||
import { VoiceChangerClient } from "../VoiceChangerClient"
|
||||
import { useIndexedDB } from "./useIndexedDB"
|
||||
|
||||
export type UseAudioStreamerSettingProps = {
|
||||
voiceChangerClient: VoiceChangerClient | null
|
||||
}
|
||||
|
||||
export type AudioStreamerSettingState = {
|
||||
audioStreamerSetting: AudioStreamerSetting;
|
||||
clearSetting: () => Promise<void>
|
||||
updateAudioStreamerSetting: (setting: AudioStreamerSetting) => void
|
||||
|
||||
}
|
||||
|
||||
export const useAudioStreamerSetting = (props: UseAudioStreamerSettingProps): AudioStreamerSettingState => {
|
||||
const [audioStreamerSetting, _setAudioStreamerSetting] = useState<AudioStreamerSetting>(DefaultAudioStreamerSetting)
|
||||
const { setItem, getItem, removeItem } = useIndexedDB()
|
||||
|
||||
// 初期化 その１ DBから取得
|
||||
useEffect(() => {
|
||||
const loadCache = async () => {
|
||||
const setting = await getItem(INDEXEDDB_KEY_STREAMER) as AudioStreamerSetting
|
||||
if (setting) {
|
||||
_setAudioStreamerSetting(setting)
|
||||
}
|
||||
}
|
||||
loadCache()
|
||||
}, [])
|
||||
|
||||
// 初期化 その２ クライアントに設定
|
||||
useEffect(() => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setServerUrl(audioStreamerSetting.serverUrl)
|
||||
props.voiceChangerClient.updateAudioStreamerSetting(audioStreamerSetting)
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
|
||||
|
||||
const clearSetting = async () => {
|
||||
await removeItem(INDEXEDDB_KEY_STREAMER)
|
||||
}
|
||||
|
||||
//////////////
|
||||
// 設定
|
||||
/////////////
|
||||
|
||||
|
||||
const updateAudioStreamerSetting = useMemo(() => {
|
||||
return (_audioStreamerSetting: AudioStreamerSetting) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
for (let k in _audioStreamerSetting) {
|
||||
const cur_v = audioStreamerSetting[k]
|
||||
const new_v = _audioStreamerSetting[k]
|
||||
if (cur_v != new_v) {
|
||||
_setAudioStreamerSetting(_audioStreamerSetting)
|
||||
setItem(INDEXEDDB_KEY_STREAMER, _audioStreamerSetting)
|
||||
props.voiceChangerClient.updateAudioStreamerSetting(_audioStreamerSetting)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [props.voiceChangerClient, audioStreamerSetting])
|
||||
|
||||
|
||||
return {
|
||||
audioStreamerSetting,
|
||||
clearSetting,
|
||||
updateAudioStreamerSetting,
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,9 +1,8 @@
|
||||
import { useEffect, useMemo, useRef, useState } from "react"
|
||||
import { AudioStreamer } from "../AudioStreamer"
|
||||
import { VoiceChangerClient } from "../VoiceChangerClient"
|
||||
import { AudioStreamerSettingState, useAudioStreamerSetting } from "./useAudioStreamerSetting"
|
||||
import { ClientSettingState, useClientSetting } from "./useClientSetting"
|
||||
import { ServerSettingState, useServerSetting } from "./useServerSetting"
|
||||
import { useWorkletNodeSetting, WorkletNodeSettingState } from "./useWorkletNodeSetting"
|
||||
import { useWorkletSetting, WorkletSettingState } from "./useWorkletSetting"
|
||||
|
||||
export type UseClientProps = {
|
||||
@ -15,7 +14,7 @@ export type ClientState = {
|
||||
// 各種設定I/Fへの参照
|
||||
workletSetting: WorkletSettingState
|
||||
clientSetting: ClientSettingState
|
||||
streamerSetting: AudioStreamerSettingState
|
||||
workletNodeSetting: WorkletNodeSettingState
|
||||
serverSetting: ServerSettingState
|
||||
|
||||
// モニタリングデータ
|
||||
@ -48,7 +47,7 @@ export const useClient = (props: UseClientProps): ClientState => {
|
||||
|
||||
// (1-2) 各種設定I/F
|
||||
const clientSetting = useClientSetting({ voiceChangerClient, audioContext: props.audioContext })
|
||||
const streamerSetting = useAudioStreamerSetting({ voiceChangerClient })
|
||||
const workletNodeSetting = useWorkletNodeSetting({ voiceChangerClient })
|
||||
const workletSetting = useWorkletSetting({ voiceChangerClient })
|
||||
const serverSetting = useServerSetting({ voiceChangerClient })
|
||||
|
||||
@ -83,8 +82,7 @@ export const useClient = (props: UseClientProps): ClientState => {
|
||||
errorCountRef.current = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}, {
|
||||
},
|
||||
notifyVolume: (vol: number) => {
|
||||
setVolume(vol)
|
||||
},
|
||||
@ -119,7 +117,7 @@ export const useClient = (props: UseClientProps): ClientState => {
|
||||
|
||||
const clearSetting = async () => {
|
||||
await clientSetting.clearSetting()
|
||||
await streamerSetting.clearSetting()
|
||||
await workletNodeSetting.clearSetting()
|
||||
await workletSetting.clearSetting()
|
||||
await serverSetting.clearSetting()
|
||||
}
|
||||
@ -127,7 +125,7 @@ export const useClient = (props: UseClientProps): ClientState => {
|
||||
return {
|
||||
// 各種設定I/Fへの参照
|
||||
clientSetting,
|
||||
streamerSetting,
|
||||
workletNodeSetting,
|
||||
workletSetting,
|
||||
serverSetting,
|
||||
|
||||
|
||||
73
client/lib/src/hooks/useWorkletNodeSetting.ts
Normal file
73
client/lib/src/hooks/useWorkletNodeSetting.ts
Normal file
@ -0,0 +1,73 @@
|
||||
import { useState, useMemo, useEffect } from "react"
|
||||
|
||||
import { DefaultWorkletNodeSetting, INDEXEDDB_KEY_WORKLETNODE, WorkletNodeSetting } from "../const"
|
||||
import { VoiceChangerClient } from "../VoiceChangerClient"
|
||||
import { useIndexedDB } from "./useIndexedDB"
|
||||
|
||||
export type UseWorkletNodeSettingProps = {
|
||||
voiceChangerClient: VoiceChangerClient | null
|
||||
}
|
||||
|
||||
export type WorkletNodeSettingState = {
|
||||
workletNodeSetting: WorkletNodeSetting;
|
||||
clearSetting: () => Promise<void>
|
||||
updateWorkletNodeSetting: (setting: WorkletNodeSetting) => void
|
||||
|
||||
}
|
||||
|
||||
export const useWorkletNodeSetting = (props: UseWorkletNodeSettingProps): WorkletNodeSettingState => {
|
||||
const [workletNodeSetting, _setWorkletNodeSetting] = useState<WorkletNodeSetting>(DefaultWorkletNodeSetting)
|
||||
const { setItem, getItem, removeItem } = useIndexedDB()
|
||||
|
||||
// 初期化 その１ DBから取得
|
||||
useEffect(() => {
|
||||
const loadCache = async () => {
|
||||
const setting = await getItem(INDEXEDDB_KEY_WORKLETNODE) as WorkletNodeSetting
|
||||
if (setting) {
|
||||
_setWorkletNodeSetting(setting)
|
||||
}
|
||||
}
|
||||
loadCache()
|
||||
}, [])
|
||||
|
||||
// 初期化 その２ クライアントに設定
|
||||
useEffect(() => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setServerUrl(workletNodeSetting.serverUrl)
|
||||
props.voiceChangerClient.updateWorkletNodeSetting(workletNodeSetting)
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
|
||||
|
||||
const clearSetting = async () => {
|
||||
await removeItem(INDEXEDDB_KEY_WORKLETNODE)
|
||||
}
|
||||
|
||||
//////////////
|
||||
// 設定
|
||||
/////////////
|
||||
|
||||
const updateWorkletNodeSetting = useMemo(() => {
|
||||
return (_workletNodeSetting: WorkletNodeSetting) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
for (let k in _workletNodeSetting) {
|
||||
const cur_v = workletNodeSetting[k]
|
||||
const new_v = _workletNodeSetting[k]
|
||||
if (cur_v != new_v) {
|
||||
_setWorkletNodeSetting(_workletNodeSetting)
|
||||
setItem(INDEXEDDB_KEY_WORKLETNODE, _workletNodeSetting)
|
||||
props.voiceChangerClient.updateWorkletNodeSetting(_workletNodeSetting)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}, [props.voiceChangerClient, workletNodeSetting])
|
||||
|
||||
|
||||
return {
|
||||
workletNodeSetting,
|
||||
clearSetting,
|
||||
updateWorkletNodeSetting,
|
||||
|
||||
}
|
||||
}
|
||||
@ -11,8 +11,8 @@ export type WorkletSettingState = {
|
||||
setting: WorkletSetting;
|
||||
clearSetting: () => Promise<void>
|
||||
setSetting: (setting: WorkletSetting) => void;
|
||||
startOutputRecording: () => void
|
||||
stopOutputRecording: () => Promise<void>
|
||||
// startOutputRecording: () => void
|
||||
// stopOutputRecording: () => Promise<void>
|
||||
}
|
||||
|
||||
export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSettingState => {
|
||||
@ -68,26 +68,26 @@ export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSetting
|
||||
await removeItem(INDEXEDDB_KEY_WORKLET)
|
||||
}
|
||||
|
||||
const startOutputRecording = useMemo(() => {
|
||||
return () => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.startOutputRecordingWorklet()
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
// const startOutputRecording = useMemo(() => {
|
||||
// return () => {
|
||||
// if (!props.voiceChangerClient) return
|
||||
// props.voiceChangerClient.startOutputRecordingWorklet()
|
||||
// }
|
||||
// }, [props.voiceChangerClient])
|
||||
|
||||
const stopOutputRecording = useMemo(() => {
|
||||
return async () => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.stopOutputRecordingWorklet()
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
// const stopOutputRecording = useMemo(() => {
|
||||
// return async () => {
|
||||
// if (!props.voiceChangerClient) return
|
||||
// props.voiceChangerClient.stopOutputRecordingWorklet()
|
||||
// }
|
||||
// }, [props.voiceChangerClient])
|
||||
|
||||
|
||||
return {
|
||||
setting,
|
||||
clearSetting,
|
||||
setSetting,
|
||||
startOutputRecording,
|
||||
stopOutputRecording
|
||||
// startOutputRecording,
|
||||
// stopOutputRecording
|
||||
}
|
||||
}
|
||||
@ -9,7 +9,6 @@ export type RequestType = typeof RequestType[keyof typeof RequestType]
|
||||
|
||||
export const ResponseType = {
|
||||
"volume": "volume",
|
||||
"recordData": "recordData",
|
||||
"inputData": "inputData"
|
||||
} as const
|
||||
export type ResponseType = typeof ResponseType[keyof typeof ResponseType]
|
||||
@ -43,7 +42,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
|
||||
private isRecording = false
|
||||
|
||||
playBuffer: Float32Array[] = []
|
||||
recordingBuffer: Float32Array[] = []
|
||||
/**
|
||||
* @constructor
|
||||
*/
|
||||
@ -75,7 +73,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
|
||||
return
|
||||
}
|
||||
this.isRecording = true
|
||||
this.recordingBuffer = []
|
||||
return
|
||||
} else if (request.requestType === "stopRecording") {
|
||||
if (!this.isRecording) {
|
||||
@ -83,13 +80,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
|
||||
return
|
||||
}
|
||||
this.isRecording = false
|
||||
const recordResponse: VoiceChangerWorkletProcessorResponse = {
|
||||
responseType: ResponseType.recordData,
|
||||
recordData: this.recordingBuffer
|
||||
|
||||
}
|
||||
this.port.postMessage(recordResponse);
|
||||
this.recordingBuffer = []
|
||||
return
|
||||
}
|
||||
|
||||
@ -126,9 +116,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
|
||||
f32Block![frameIndexInBlock + 1] = (currentFrame + nextFrame) / 2
|
||||
if (f32Block!.length === frameIndexInBlock + 2) {
|
||||
this.playBuffer.push(f32Block!)
|
||||
if (this.isRecording) {
|
||||
this.recordingBuffer.push(f32Block!)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -148,8 +135,10 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (_inputs.length > 0 && _inputs[0].length > 0) {
|
||||
this.pushData(_inputs[0][0])
|
||||
if (this.isRecording) {
|
||||
if (_inputs.length > 0 && _inputs[0].length > 0) {
|
||||
this.pushData(_inputs[0][0])
|
||||
}
|
||||
}
|
||||
|
||||
if (this.playBuffer.length === 0) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user