@fciannella/nvidia-asr-client
Version:
Minimal cross-platform wrapper around NVIDIA/Riva streaming ASR WebSocket API with optional client-side silence detection.
316 lines (315 loc) • 11.3 kB
JavaScript
// Browser-compatible EventEmitter implementation
class EventEmitter {
constructor() {
this.events = {};
}
on(event, listener) {
if (!this.events[event]) {
this.events[event] = [];
}
this.events[event].push(listener);
return this;
}
emit(event, ...args) {
const listeners = this.events[event];
if (!listeners || listeners.length === 0) {
return false;
}
listeners.forEach((listener) => {
listener(...args);
});
return true;
}
removeListener(event, listener) {
if (!this.events[event]) {
return this;
}
this.events[event] = this.events[event].filter((l) => l !== listener);
return this;
}
removeAllListeners(event) {
if (event) {
this.events[event] = [];
}
else {
this.events = {};
}
return this;
}
}
/**
* Minimal wrapper around NVIDIA/Riva streaming ASR WebSocket.
* Works in browsers with native WebSocket API.
*/
export class NvidiaAsrClient extends EventEmitter {
constructor(opts = {}) {
super();
this.ws = null;
this.lastActivity = Date.now();
this.transcript = '';
this._dbg = false;
if (!opts.websocketUrl) {
throw new Error('NvidiaAsrClient: `websocketUrl` is required – please provide your Riva endpoint.');
}
this.opts = {
websocketUrl: opts.websocketUrl,
languageCode: opts.languageCode ?? 'en-US',
silenceTimeout: opts.silenceTimeout,
closeOnSilence: opts.closeOnSilence !== false,
inputFormat: opts.inputFormat ?? 'f32',
inputSampleRate: opts.inputSampleRate ?? 16000,
targetSampleRate: opts.targetSampleRate ?? 16000,
};
}
/** Opens WebSocket and sends configuration packet. */
async connect() {
if (this.ws)
return;
return new Promise((resolve, reject) => {
this.ws = new WebSocket(this.opts.websocketUrl);
this.ws.onopen = () => {
const cfg = {
streaming_config: {
config: {
language_code: this.opts.languageCode,
max_alternatives: 1,
enable_automatic_punctuation: true,
},
interim_results: true,
},
};
console.log('Sending ASR config to server:', JSON.stringify(cfg));
this.ws.send(JSON.stringify(cfg));
this.setupSilenceWatcher();
resolve();
};
this.ws.onmessage = (event) => this.handleMessage(event.data.toString());
this.ws.onerror = (event) => {
const err = new Error('WebSocket error');
this.emit('error', err);
this.end();
reject(err);
};
this.ws.onclose = () => this.cleanup();
});
}
/**
* Send a chunk of PCM float32 samples. Provide sampleRate if not 16 kHz.
*/
write(chunk, sampleRate) {
if (!this.ws || this.ws.readyState !== WebSocket.OPEN)
return;
let floatData;
const inRate = sampleRate ?? this.opts.inputSampleRate;
switch (this.opts.inputFormat) {
case 'g711_ulaw':
floatData = decodeULaw(new Uint8Array(chunk instanceof ArrayBuffer ? chunk : chunk.buffer));
if (!this._dbg) {
this._dbg = true;
console.log('decoded', floatData.slice(0, 10));
}
break;
case 'pcm_s16': {
const src = chunk instanceof Int16Array ? chunk : new Int16Array(chunk instanceof ArrayBuffer ? chunk : chunk.buffer);
floatData = pcmS16ToF32(src);
break;
}
default:
floatData = chunk instanceof Float32Array ? chunk : new Float32Array(chunk instanceof ArrayBuffer ? chunk : chunk.buffer);
}
// resample if needed
let sendData = floatData;
let sendRate = this.opts.targetSampleRate;
if (inRate !== this.opts.targetSampleRate) {
sendData = linearResample(floatData, inRate, this.opts.targetSampleRate);
sendRate = this.opts.targetSampleRate;
}
const wav = encodeWav(sendData, sendRate);
const payload = JSON.stringify({ audio_content: arrayBufferToBase64(wav) });
this.ws.send(payload);
}
/**
* Signal end-of-audio but keep the socket open so Riva can flush any
* remaining hypotheses. The connection will be closed automatically
* by the silence timer or you can call `end()` manually afterwards.
*/
finish() {
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
try {
this.ws.send(JSON.stringify({ audio_content: '' }));
}
catch {
/* ignore */
}
}
}
/**
* Flushes EOS marker **and** closes the WebSocket immediately. In most
* cases you should prefer `finish()` + wait for a `final` or `silence`
* event so you don't lose the last transcript.
*/
end() {
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
// Send empty packet as EOS marker per Riva protocol
try {
this.ws.send(JSON.stringify({ audio_content: '' }));
}
catch {
/* ignore */
}
this.ws.close();
}
this.cleanup();
}
// ------------------------------------------------------------------
// Internal helpers
// ------------------------------------------------------------------
handleMessage(msgStr) {
try {
const msg = JSON.parse(msgStr);
if (!msg.results)
return;
msg.results.forEach((r) => {
if (!r.alternatives?.length)
return;
const text = r.alternatives[0].transcript;
if (r.is_final) {
// Server says this chunk is final, but from a UX standpoint we only
// consider the *whole* phrase final after the user has stopped
// speaking (silence timeout). Therefore we just append it to the
// buffer and treat it as another interim update.
this.transcript = this.transcript
? `${this.transcript} ${text}`
: text;
this.lastActivity = Date.now();
this.emit('partial', {
text: this.transcript.trim(),
serverFinal: true,
});
}
else {
const combined = this.transcript
? `${this.transcript} ${text}`
: text;
this.lastActivity = Date.now();
this.emit('partial', {
text: combined,
serverFinal: false,
});
}
});
}
catch (err) {
this.emit('error', err);
}
}
setupSilenceWatcher() {
if (!this.opts.silenceTimeout)
return;
this.silenceTimer = window.setInterval(() => {
const elapsed = Date.now() - this.lastActivity;
if (elapsed > (this.opts.silenceTimeout * 1000)) {
if (this.transcript.trim()) {
this.emit('final', {
text: this.transcript.trim(),
});
this.transcript = '';
}
this.emit('silence');
if (this.opts.closeOnSilence) {
this.end();
}
else {
this.lastActivity = Date.now();
}
}
}, 500);
}
clearSilenceWatcher() {
if (this.silenceTimer !== undefined)
window.clearInterval(this.silenceTimer);
this.silenceTimer = undefined;
}
cleanup() {
this.clearSilenceWatcher();
this.ws = null;
}
}
/* -------------------------------------------------------------- */
/* UTILITIES */
/* -------------------------------------------------------------- */
function encodeWav(samples, sampleRate) {
const buffer = new ArrayBuffer(44 + samples.length * 2);
const view = new DataView(buffer);
const writeStr = (o, s) => {
for (let i = 0; i < s.length; i++)
view.setUint8(o + i, s.charCodeAt(i));
};
// RIFF + WAVE header
writeStr(0, 'RIFF');
view.setUint32(4, 36 + samples.length * 2, true);
writeStr(8, 'WAVE');
// fmt sub-chunk
writeStr(12, 'fmt ');
view.setUint32(16, 16, true); // PCM header size
view.setUint16(20, 1, true); // audio format = PCM
view.setUint16(22, 1, true); // mono
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * 2, true); // byte rate (16-bit mono)
view.setUint16(32, 2, true); // block align
view.setUint16(34, 16, true); // bits per sample
// data sub-chunk
writeStr(36, 'data');
view.setUint32(40, samples.length * 2, true);
// PCM samples – float [-1,1] -> signed 16-bit
let offset = 44;
for (let i = 0; i < samples.length; i++) {
const s = Math.max(-1, Math.min(1, samples[i]));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
offset += 2;
}
return buffer;
}
// Browser-compatible base64 encoding
function arrayBufferToBase64(buffer) {
const bytes = new Uint8Array(buffer);
let binary = '';
for (let i = 0; i < bytes.byteLength; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
/* ---------------- decoding helpers ----------------- */
function pcmS16ToF32(src) {
const out = new Float32Array(src.length);
for (let i = 0; i < src.length; i++)
out[i] = src[i] / 0x8000;
return out;
}
const ULAW_TABLE = new Int16Array(256);
for (let i = 0; i < 256; i++) {
const u = ~i;
let t = ((u & 0xf) << 3) + 0x84;
t <<= (u & 0x70) >> 4;
ULAW_TABLE[i] = (u & 0x80) ? 0x84 - t : t - 0x84;
}
function decodeULaw(bytes) {
const out = new Float32Array(bytes.length);
for (let i = 0; i < bytes.length; i++)
out[i] = ULAW_TABLE[bytes[i]] / 0x8000;
return out;
}
function linearResample(src, inRate, outRate) {
if (inRate === outRate)
return src;
const factor = outRate / inRate;
const dst = new Float32Array(Math.floor(src.length * factor));
for (let i = 0; i < dst.length; i++) {
const s = i / factor;
const i0 = Math.floor(s);
const i1 = Math.min(i0 + 1, src.length - 1);
const frac = s - i0;
dst[i] = src[i0] * (1 - frac) + src[i1] * frac;
}
return dst;
}