277 lines
10 KiB
TypeScript
277 lines
10 KiB
TypeScript
import { io, Socket } from "socket.io-client";
|
||
import { DefaultEventsMap } from "@socket.io/component-emitter";
|
||
import { Duplex, DuplexOptions } from "readable-stream";
|
||
import { DefaultVoiceChangerRequestParamas, MajarModeTypes, VoiceChangerMode, VoiceChangerRequestParamas } from "./const";
|
||
|
||
export type Callbacks = {
|
||
onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer) => void
|
||
}
|
||
export type AudioStreamerListeners = {
|
||
notifySendBufferingTime: (time: number) => void
|
||
notifyResponseTime: (time: number) => void
|
||
notifyException: (message: string) => void
|
||
}
|
||
export class AudioStreamer extends Duplex {
|
||
private callbacks: Callbacks
|
||
private audioStreamerListeners: AudioStreamerListeners
|
||
private majarMode: MajarModeTypes
|
||
private serverUrl = ""
|
||
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
|
||
private voiceChangerMode: VoiceChangerMode = "realtime"
|
||
private requestParamas: VoiceChangerRequestParamas = DefaultVoiceChangerRequestParamas
|
||
private chunkNum = 8
|
||
private requestChunks: ArrayBuffer[] = []
|
||
private recordChunks: ArrayBuffer[] = []
|
||
private isRecording = false
|
||
|
||
// performance monitor
|
||
private bufferStart = 0;
|
||
|
||
constructor(majarMode: MajarModeTypes, callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
|
||
super(options);
|
||
this.majarMode = majarMode
|
||
this.callbacks = callbacks
|
||
this.audioStreamerListeners = audioStreamerListeners
|
||
}
|
||
|
||
private createSocketIO = () => {
|
||
if (this.socket) {
|
||
this.socket.close()
|
||
}
|
||
if (this.majarMode === "sio") {
|
||
this.socket = io(this.serverUrl);
|
||
this.socket.on('connect', () => console.log(`[SIO] sonnect to ${this.serverUrl}`));
|
||
this.socket.on('response', (response: any[]) => {
|
||
const cur = Date.now()
|
||
const responseTime = cur - response[0]
|
||
const result = response[1] as ArrayBuffer
|
||
if (result.byteLength < 128 * 2) {
|
||
this.audioStreamerListeners.notifyException(`[SIO] recevied data is too short ${result.byteLength}`)
|
||
} else {
|
||
this.audioStreamerListeners.notifyException(``)
|
||
this.callbacks.onVoiceReceived(this.voiceChangerMode, response[1])
|
||
this.audioStreamerListeners.notifyResponseTime(responseTime)
|
||
}
|
||
});
|
||
}
|
||
}
|
||
|
||
// Option Change
|
||
setServerUrl = (serverUrl: string, mode: MajarModeTypes) => {
|
||
this.serverUrl = serverUrl
|
||
this.majarMode = mode
|
||
window.open(serverUrl, '_blank')
|
||
console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.majarMode}`)
|
||
|
||
this.createSocketIO()// mode check is done in the method.
|
||
}
|
||
|
||
setRequestParams = (val: VoiceChangerRequestParamas) => {
|
||
this.requestParamas = val
|
||
}
|
||
|
||
setChunkNum = (num: number) => {
|
||
this.chunkNum = num
|
||
}
|
||
|
||
setVoiceChangerMode = (val: VoiceChangerMode) => {
|
||
this.voiceChangerMode = val
|
||
}
|
||
|
||
|
||
// Main Process
|
||
//// Pipe from mic stream
|
||
_write = (chunk: AudioBuffer, _encoding: any, callback: any) => {
|
||
const buffer = chunk.getChannelData(0);
|
||
// console.log("SAMPLERATE:", chunk.sampleRate, chunk.numberOfChannels, chunk.length, buffer)
|
||
if (this.voiceChangerMode === "realtime") {
|
||
this._write_realtime(buffer)
|
||
} else {
|
||
this._write_record(buffer)
|
||
}
|
||
callback();
|
||
}
|
||
|
||
private _write_realtime = (buffer: Float32Array) => {
|
||
// bufferSize個のデータ（48Khz）が入ってくる。
|
||
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。
|
||
//// バイトサイズは周波数変換で(x1/2), 16bit(2byte)で(x2)
|
||
const arrayBuffer = new ArrayBuffer((buffer.length / 2) * 2)
|
||
const dataView = new DataView(arrayBuffer);
|
||
|
||
for (let i = 0; i < buffer.length; i++) {
|
||
if (i % 2 == 0) {
|
||
let s = Math.max(-1, Math.min(1, buffer[i]));
|
||
s = s < 0 ? s * 0x8000 : s * 0x7FFF
|
||
// ２分の１個目で２バイトずつ進むので((i/2)*2)
|
||
dataView.setInt16((i / 2) * 2, s, true);
|
||
}
|
||
}
|
||
// 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理
|
||
const chunkByteSize = 256 // (const.ts ★1)
|
||
for (let i = 0; i < arrayBuffer.byteLength / chunkByteSize; i++) {
|
||
const ab = arrayBuffer.slice(i * chunkByteSize, (i + 1) * chunkByteSize)
|
||
this.requestChunks.push(ab)
|
||
}
|
||
|
||
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
|
||
if (this.requestChunks.length < this.chunkNum) {
|
||
return
|
||
}
|
||
|
||
// リクエスト用の入れ物を作成
|
||
const windowByteLength = this.requestChunks.reduce((prev, cur) => {
|
||
return prev + cur.byteLength
|
||
}, 0)
|
||
const newBuffer = new Uint8Array(windowByteLength);
|
||
|
||
// リクエストのデータをセット
|
||
this.requestChunks.reduce((prev, cur) => {
|
||
newBuffer.set(new Uint8Array(cur), prev)
|
||
return prev + cur.byteLength
|
||
}, 0)
|
||
|
||
console.log("send buff length", newBuffer.length)
|
||
|
||
this.sendBuffer(newBuffer)
|
||
this.requestChunks = []
|
||
|
||
this.audioStreamerListeners.notifySendBufferingTime(Date.now() - this.bufferStart)
|
||
this.bufferStart = Date.now()
|
||
}
|
||
|
||
|
||
private _write_record = (buffer: Float32Array) => {
|
||
if (!this.isRecording) { return }
|
||
// buffer(for48Khz)x16bit * chunksize / 2(for24Khz)
|
||
const sendBuffer = new ArrayBuffer(buffer.length * 2 / 2);
|
||
const sendDataView = new DataView(sendBuffer);
|
||
for (var i = 0; i < buffer.length; i++) {
|
||
if (i % 2 == 0) {
|
||
let s = Math.max(-1, Math.min(1, buffer[i]));
|
||
s = s < 0 ? s * 0x8000 : s * 0x7FFF
|
||
sendDataView.setInt16(i, s, true);
|
||
// if (i % 3000 === 0) {
|
||
// console.log("buffer_converting", s, buffer[i])
|
||
// }
|
||
}
|
||
}
|
||
this.recordChunks.push(sendBuffer)
|
||
}
|
||
|
||
// Near Realtime用のトリガ
|
||
sendRecordedData = () => {
|
||
const length = this.recordChunks.reduce((prev, cur) => {
|
||
return prev + cur.byteLength
|
||
}, 0)
|
||
const newBuffer = new Uint8Array(length);
|
||
this.recordChunks.reduce((prev, cur) => {
|
||
newBuffer.set(new Uint8Array(cur), prev)
|
||
return prev + cur.byteLength
|
||
}, 0)
|
||
|
||
this.sendBuffer(newBuffer)
|
||
}
|
||
|
||
startRecord = () => {
|
||
this.recordChunks = []
|
||
this.isRecording = true
|
||
}
|
||
|
||
stopRecord = () => {
|
||
this.isRecording = false
|
||
}
|
||
|
||
private sendBuffer = async (newBuffer: Uint8Array) => {
|
||
if (this.serverUrl.length == 0) {
|
||
console.error("no server url")
|
||
throw "no server url"
|
||
}
|
||
const timestamp = Date.now()
|
||
// console.log("REQUEST_MESSAGE:", [this.gpu, this.srcId, this.dstId, timestamp, newBuffer.buffer])
|
||
console.log("SERVER_URL", this.serverUrl, this.majarMode)
|
||
const convertChunkNum = this.voiceChangerMode === "realtime" ? this.requestParamas.convertChunkNum : 0
|
||
if (this.majarMode === "sio") {
|
||
if (!this.socket) {
|
||
console.warn(`sio is not initialized`)
|
||
return
|
||
}
|
||
console.log("emit!")
|
||
this.socket.emit('request_message', [
|
||
this.requestParamas.gpu,
|
||
this.requestParamas.srcId,
|
||
this.requestParamas.dstId,
|
||
timestamp,
|
||
convertChunkNum,
|
||
this.requestParamas.crossFadeLowerValue,
|
||
this.requestParamas.crossFadeOffsetRate,
|
||
this.requestParamas.crossFadeEndRate,
|
||
newBuffer.buffer]);
|
||
} else {
|
||
const res = await postVoice(
|
||
this.serverUrl,
|
||
this.requestParamas.gpu,
|
||
this.requestParamas.srcId,
|
||
this.requestParamas.dstId,
|
||
timestamp,
|
||
convertChunkNum,
|
||
this.requestParamas.crossFadeLowerValue,
|
||
this.requestParamas.crossFadeOffsetRate,
|
||
this.requestParamas.crossFadeEndRate,
|
||
newBuffer.buffer)
|
||
|
||
if (res.byteLength < 128 * 2) {
|
||
this.audioStreamerListeners.notifyException(`[REST] recevied data is too short ${res.byteLength}`)
|
||
} else {
|
||
this.audioStreamerListeners.notifyException(``)
|
||
this.callbacks.onVoiceReceived(this.voiceChangerMode, res)
|
||
this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
export const postVoice = async (
|
||
url: string,
|
||
gpu: number,
|
||
srcId: number,
|
||
dstId: number,
|
||
timestamp: number,
|
||
convertSize: number,
|
||
crossFadeLowerValue: number,
|
||
crossFadeOffsetRate: number,
|
||
crossFadeEndRate: number,
|
||
buffer: ArrayBuffer) => {
|
||
const obj = {
|
||
gpu,
|
||
srcId,
|
||
dstId,
|
||
timestamp,
|
||
convertSize,
|
||
crossFadeLowerValue,
|
||
crossFadeOffsetRate,
|
||
crossFadeEndRate,
|
||
buffer: Buffer.from(buffer).toString('base64')
|
||
};
|
||
const body = JSON.stringify(obj);
|
||
|
||
const res = await fetch(`${url}`, {
|
||
method: "POST",
|
||
headers: {
|
||
'Accept': 'application/json',
|
||
'Content-Type': 'application/json'
|
||
},
|
||
body: body
|
||
})
|
||
|
||
const receivedJson = await res.json()
|
||
const changedVoiceBase64 = receivedJson["changedVoiceBase64"]
|
||
const buf = Buffer.from(changedVoiceBase64, "base64")
|
||
const ab = new ArrayBuffer(buf.length);
|
||
// console.log("RECIV", buf.length)
|
||
const view = new Uint8Array(ab);
|
||
for (let i = 0; i < buf.length; ++i) {
|
||
view[i] = buf[i];
|
||
}
|
||
return ab
|
||
} |