sofya.transcription
Version:
a JavaScript library that provides a robust and flexible solution for real-time audio transcription. It is designed to transcribe audio streams and can be easily integrated into web applications.
276 lines • 10.1 kB
JavaScript
"use strict";
// import { MicVAD } from "@ricky0123/vad-web";
// import EventEmitter from "eventemitter3";
// import { ITranscriptionService } from "../interfaces/ITranscriptionService";
// interface ITranscriptionConfig {
// language: string;
// ws: WebSocket;
// }
// export class WhisperVadTranscriptionAdapter
// extends EventEmitter
// implements ITranscriptionService
// {
// private config: ITranscriptionConfig;
// private websocket: WebSocket | null = null;
// private audioContext: AudioContext | null = null;
// private mediaStream: MediaStream | null = null;
// private input: MediaStreamAudioSourceNode | undefined;
// private recordState: string = "PAUSED";
// private recordingNode: AudioWorkletNode | undefined = undefined;
// private speaking: boolean = false;
// private vad: MicVAD | null = null;
// private preSpeechBuffer: Float32Array[] = [];
// private maxPreSpeechFrames: number = 500;
// private isSpeechEventActive: boolean = false;
// constructor(config: ITranscriptionConfig) {
// super();
// this.config = config;
// this.websocket = config.ws;
// this.connectWebsocketHandler(config.ws);
// this.audioContext = new AudioContext({ sampleRate: 44100 });
// }
// private initializeVad = async (mediaStream: MediaStream) => {
// this.vad = await MicVAD.new({
// stream: mediaStream,
// preSpeechPadFrames: 10,
// ortConfig: (ort) => {
// ort.env.wasm.wasmPaths =
// "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.19.0/dist/";
// },
// workletURL:
// "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.18/dist/vad.worklet.bundle.min.js",
// modelURL:
// "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.18/dist/silero_vad.onnx",
// onSpeechStart: () => {
// this.speaking = true;
// this.isSpeechEventActive = false;
// },
// onSpeechEnd: () => {
// this.speaking = false;
// this.isSpeechEventActive = false;
// },
// onVADMisfire: () => {
// this.speaking = false;
// this.isSpeechEventActive = false;
// },
// });
// this.vad.start();
// };
// public async startTranscription(mediaStream: MediaStream): Promise<void> {
// if (!navigator.mediaDevices.getUserMedia) {
// throw new Error("getUserMedia not supported on your browser!");
// }
// this.mediaStream = mediaStream;
// try {
// this.input = this.audioContext?.createMediaStreamSource(this.mediaStream);
// this.recordingNode = await this.setupRecordingWorkletNode();
// this.initializeVad(mediaStream);
// this.recordingNode.port.onmessage = (event) => {
// const audioFrame = event.data;
// if (this.recordState !== "PAUSED" && !this.speaking) {
// this.preSpeechBuffer.push(audioFrame);
// if (this.preSpeechBuffer.length > this.maxPreSpeechFrames) {
// this.preSpeechBuffer.shift();
// }
// }
// if (this.recordState !== "PAUSED" && this.speaking) {
// if (!this.isSpeechEventActive) {
// this.isSpeechEventActive = true;
// if (
// Array.isArray(this.preSpeechBuffer) &&
// this.preSpeechBuffer.length
// ) {
// let audioToSend = [...this.preSpeechBuffer];
// if (audioFrame.length) {
// audioToSend.push(audioFrame);
// }
// const flattenedAudio = audioToSend.reduce(
// (acc: number[], frame: Float32Array) => {
// if (
// frame &&
// (Array.isArray(frame) || frame instanceof Float32Array)
// ) {
// return acc.concat(Array.from(frame));
// } else {
// return acc;
// }
// },
// []
// );
// this.postMessage(flattenedAudio);
// this.preSpeechBuffer = [];
// } else {
// if (audioFrame.length) {
// this.postMessage(audioFrame);
// }
// }
// } else {
// if (audioFrame.length) {
// this.postMessage(audioFrame);
// }
// }
// }
// };
// if (this.input) {
// this.input.connect(this.recordingNode);
// this.recordState = "RECORDING";
// } else {
// console.error("Unable to createMediaStreamSource");
// }
// } catch (error) {
// console.error("Erro no onmessage: ", error);
// }
// }
// async setupRecordingWorkletNode() {
// const workletCode = `
// class RealtimeAudioProcessor extends AudioWorkletProcessor {
// constructor(options) {
// super();
// }
// process(inputs, outputs, params) {
// // ASR and VAD models typically require a mono audio.
// this.port.postMessage(inputs[0][0]);
// return true;
// }
// }
// registerProcessor('realtime-audio-processor', RealtimeAudioProcessor);
// `;
// const blob = new Blob([workletCode], { type: "application/javascript" });
// const workletURL = URL.createObjectURL(blob);
// await this.audioContext!.audioWorklet.addModule(workletURL);
// return new AudioWorkletNode(this.audioContext!, "realtime-audio-processor");
// }
// private postMessage = (sampleData: any) => {
// const outputSampleRate = 16000;
// const decreaseResultBuffer = this.decreaseSampleRate(
// sampleData,
// 44100,
// outputSampleRate
// );
// const audioData = this.convertFloat32ToInt16(decreaseResultBuffer);
// if (this.websocket && this.websocket?.readyState === WebSocket.OPEN) {
// this.websocket.send(audioData);
// }
// };
// private connectWebsocketHandler = (ws: WebSocket) => {
// if (this.websocket && this.websocket?.readyState === WebSocket.OPEN) {
// this.sendAudioConfig(this.config?.language);
// this.websocket.onclose = (event) => {
// console.log("WebSocket connection closed", event);
// };
// this.websocket.onmessage = (event) => {
// const transcript_data = JSON.parse(event.data);
// if (!transcript_data.is_partial && transcript_data?.data?.text) {
// this.emit("recognized", transcript_data?.data?.text);
// }
// if (transcript_data.is_partial && transcript_data?.data?.text) {
// this.emit("recognizing", transcript_data?.data?.text);
// }
// };
// }
// };
// private languageSelector = (language: string) => {
// switch (language) {
// case "pt-BR":
// return "portuguese";
// case "en-US":
// return "english";
// case "es-ES":
// return "spanish";
// default:
// return "multilingual";
// }
// };
// private sendAudioConfig = (language = "multilingual") => {
// let processingArgs = {};
// const chunk_length_seconds = "3";
// const chunk_offset_seconds = "0.1";
// let selectedStrategy = "silence_at_end_of_chunk";
// if (selectedStrategy === "silence_at_end_of_chunk") {
// processingArgs = {
// chunk_length_seconds: parseFloat(chunk_length_seconds),
// chunk_offset_seconds: parseFloat(chunk_offset_seconds),
// };
// }
// const audioConfig = {
// type: "config",
// data: {
// sampleRate: 44100,
// channels: 1,
// language: this.languageSelector(language),
// processing_strategy: selectedStrategy,
// processing_args: processingArgs,
// },
// };
// if (this.websocket && this.websocket?.readyState === WebSocket.OPEN) {
// this.websocket.send(JSON.stringify(audioConfig));
// }
// };
// private decreaseSampleRate(
// buffer: any,
// inputSampleRate: any,
// outputSampleRate: any
// ) {
// if (inputSampleRate < outputSampleRate) {
// console.error("Sample rate too small.");
// return;
// } else if (inputSampleRate === outputSampleRate) {
// return;
// }
// let sampleRateRatio = inputSampleRate / outputSampleRate;
// let newLength = Math.ceil(buffer.length / sampleRateRatio);
// let result = new Float32Array(newLength);
// let offsetResult = 0;
// let offsetBuffer = 0;
// while (offsetResult < result.length) {
// let nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
// let accum = 0,
// count = 0;
// for (
// let i = offsetBuffer;
// i < nextOffsetBuffer && i < buffer.length;
// i++
// ) {
// accum += buffer[i];
// count++;
// }
// result[offsetResult] = accum / count;
// offsetResult++;
// offsetBuffer = nextOffsetBuffer;
// }
// return result;
// }
// private convertFloat32ToInt16(buffer: any) {
// let l = buffer.length;
// const buf = new Int16Array(l);
// while (l--) {
// buf[l] = Math.min(1, buffer[l]) * 0x7fff;
// }
// return buf.buffer;
// }
// public pauseTranscription() {
// this.recordState = "PAUSED";
// if (this.input) {
// this.input.disconnect();
// }
// }
// public resumeTranscription() {
// if (this.input && this.recordingNode) {
// this.input.connect(this.recordingNode);
// }
// this.recordState = "RECORDING";
// }
// public async stopTranscription(): Promise<void> {
// if (this.mediaStream) {
// this.mediaStream.getTracks().forEach((track) => track.stop());
// }
// if (this.audioContext) {
// await this.audioContext.close();
// this.audioContext = null;
// }
// if (this.websocket?.readyState === WebSocket.OPEN) {
// this.websocket.close();
// }
// }
// }
//# sourceMappingURL=WhisperVadTranscriptionAdapter.js.map