assemblyai
Version:
The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.
1,157 lines (1,149 loc) • 70.4 kB
JavaScript
/**
* Thrown when `DualChannelCapture` is constructed in a non-browser environment
* (no `globalThis.AudioContext`). The helper is intentionally surfaced from the
* main entrypoint so the import path is uniform across runtimes; the runtime
* guard moves to construction time.
*/
class BrowserOnlyError extends Error {
constructor(message = "DualChannelCapture requires a browser environment (AudioContext is undefined).") {
super(message);
this.name = "BrowserOnlyError";
}
}
const { WritableStream } = typeof window !== "undefined"
? window
: typeof global !== "undefined"
? global
: globalThis;
const PolyfillWebSocket = WebSocket ?? global?.WebSocket ?? window?.WebSocket ?? self?.WebSocket;
const factory = (url, params) => {
if (params) {
return new PolyfillWebSocket(url, params);
}
return new PolyfillWebSocket(url);
};
const RealtimeErrorType = {
BadSampleRate: 4000,
AuthFailed: 4001,
/**
* @deprecated Use InsufficientFunds or FreeTierUser instead
*/
InsufficientFundsOrFreeAccount: 4002,
InsufficientFunds: 4002,
FreeTierUser: 4003,
NonexistentSessionId: 4004,
SessionExpired: 4008,
ClosedSession: 4010,
RateLimited: 4029,
UniqueSessionViolation: 4030,
SessionTimeout: 4031,
AudioTooShort: 4032,
AudioTooLong: 4033,
AudioTooSmallToTranscode: 4034,
/**
* @deprecated Don't use
*/
BadJson: 4100,
BadSchema: 4101,
TooManyStreams: 4102,
Reconnected: 4103,
/**
* @deprecated Don't use
*/
ReconnectAttemptsExhausted: 1013,
WordBoostParameterParsingFailed: 4104,
};
const RealtimeErrorMessages = {
[RealtimeErrorType.BadSampleRate]: "Sample rate must be a positive integer",
[RealtimeErrorType.AuthFailed]: "Not Authorized",
[RealtimeErrorType.InsufficientFunds]: "Insufficient funds",
[RealtimeErrorType.FreeTierUser]: "This feature is paid-only and requires you to add a credit card. Please visit https://app.assemblyai.com/ to add a credit card to your account.",
[RealtimeErrorType.NonexistentSessionId]: "Session ID does not exist",
[RealtimeErrorType.SessionExpired]: "Session has expired",
[RealtimeErrorType.ClosedSession]: "Session is closed",
[RealtimeErrorType.RateLimited]: "Rate limited",
[RealtimeErrorType.UniqueSessionViolation]: "Unique session violation",
[RealtimeErrorType.SessionTimeout]: "Session Timeout",
[RealtimeErrorType.AudioTooShort]: "Audio too short",
[RealtimeErrorType.AudioTooLong]: "Audio too long",
[RealtimeErrorType.AudioTooSmallToTranscode]: "Audio too small to transcode",
[RealtimeErrorType.BadJson]: "Bad JSON",
[RealtimeErrorType.BadSchema]: "Bad schema",
[RealtimeErrorType.TooManyStreams]: "Too many streams",
[RealtimeErrorType.Reconnected]: "This session has been reconnected. This WebSocket is no longer valid.",
[RealtimeErrorType.ReconnectAttemptsExhausted]: "Reconnect attempts exhausted",
[RealtimeErrorType.WordBoostParameterParsingFailed]: "Could not parse word boost parameter",
};
class RealtimeError extends Error {
}
const StreamingErrorType = {
BadSampleRate: 4000,
AuthFailed: 4001,
InsufficientFunds: 4002,
FreeTierUser: 4003,
NonexistentSessionId: 4004,
SessionExpired: 4008,
ClosedSession: 4010,
RateLimited: 4029,
UniqueSessionViolation: 4030,
SessionTimeout: 4031,
AudioTooShort: 4032,
AudioTooLong: 4033,
AudioTooSmallToTranscode: 4034,
BadSchema: 4101,
TooManyStreams: 4102,
Reconnected: 4103,
ServerError: 3005,
InputValidationError: 3006,
AudioChunkDurationViolation: 3007,
MaxSessionDurationExceeded: 3008,
ConcurrencyLimitExceeded: 3009,
};
const StreamingErrorMessages = {
[StreamingErrorType.ServerError]: "Server error",
[StreamingErrorType.InputValidationError]: "Input validation error",
[StreamingErrorType.AudioChunkDurationViolation]: "Audio chunk duration violation",
[StreamingErrorType.MaxSessionDurationExceeded]: "Session expired: maximum session duration exceeded",
[StreamingErrorType.ConcurrencyLimitExceeded]: "Too many concurrent sessions",
[StreamingErrorType.BadSampleRate]: "Sample rate must be a positive integer",
[StreamingErrorType.AuthFailed]: "Not Authorized",
[StreamingErrorType.InsufficientFunds]: "Insufficient funds",
[StreamingErrorType.FreeTierUser]: "This feature is paid-only and requires you to add a credit card. Please visit https://app.assemblyai.com/ to add a credit card to your account.",
[StreamingErrorType.NonexistentSessionId]: "Session ID does not exist",
[StreamingErrorType.SessionExpired]: "Session has expired",
[StreamingErrorType.ClosedSession]: "Session is closed",
[StreamingErrorType.RateLimited]: "Rate limited",
[StreamingErrorType.UniqueSessionViolation]: "Unique session violation",
[StreamingErrorType.SessionTimeout]: "Session Timeout",
[StreamingErrorType.AudioTooShort]: "Audio too short",
[StreamingErrorType.AudioTooLong]: "Audio too long",
[StreamingErrorType.AudioTooSmallToTranscode]: "Audio too small to transcode",
[StreamingErrorType.BadSchema]: "Bad schema",
[StreamingErrorType.TooManyStreams]: "Too many streams",
[StreamingErrorType.Reconnected]: "This session has been reconnected. This WebSocket is no longer valid.",
};
class StreamingError extends Error {
}
const defaultRealtimeUrl = "wss://api.assemblyai.com/v2/realtime/ws";
const forceEndOfUtteranceMessage = `{"force_end_utterance":true}`;
const terminateSessionMessage$1 = `{"terminate_session":true}`;
/**
* RealtimeTranscriber connects to the Streaming Speech-to-Text API and lets you transcribe audio in real-time.
*/
class RealtimeTranscriber {
/**
* Create a new RealtimeTranscriber.
* @param params - Parameters to configure the RealtimeTranscriber
*/
constructor(params) {
this.listeners = {};
this.realtimeUrl = params.realtimeUrl ?? defaultRealtimeUrl;
this.sampleRate = params.sampleRate ?? 16_000;
this.wordBoost = params.wordBoost;
this.encoding = params.encoding;
this.endUtteranceSilenceThreshold = params.endUtteranceSilenceThreshold;
this.disablePartialTranscripts = params.disablePartialTranscripts;
if ("token" in params && params.token)
this.token = params.token;
if ("apiKey" in params && params.apiKey)
this.apiKey = params.apiKey;
if (!(this.token || this.apiKey)) {
throw new Error("API key or temporary token is required.");
}
}
connectionUrl() {
const url = new URL(this.realtimeUrl);
if (url.protocol !== "wss:") {
throw new Error("Invalid protocol, must be wss");
}
const searchParams = new URLSearchParams();
if (this.token) {
searchParams.set("token", this.token);
}
searchParams.set("sample_rate", this.sampleRate.toString());
if (this.wordBoost && this.wordBoost.length > 0) {
searchParams.set("word_boost", JSON.stringify(this.wordBoost));
}
if (this.encoding) {
searchParams.set("encoding", this.encoding);
}
searchParams.set("enable_extra_session_information", "true");
if (this.disablePartialTranscripts) {
searchParams.set("disable_partial_transcripts", this.disablePartialTranscripts.toString());
}
url.search = searchParams.toString();
return url;
}
/**
* Add a listener for an event.
* @param event - The event to listen for.
* @param listener - The function to call when the event is emitted.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
on(event, listener) {
this.listeners[event] = listener;
}
/**
* Connect to the server and begin a new session.
* @returns A promise that resolves when the connection is established and the session begins.
*/
connect() {
return new Promise((resolve) => {
if (this.socket) {
throw new Error("Already connected");
}
const url = this.connectionUrl();
if (this.token) {
this.socket = factory(url.toString());
}
else {
{
console.warn(`API key authentication is not supported for the RealtimeTranscriber in browser environment. Use temporary token authentication instead.
Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/compat.md#browser-compatibility.`);
}
this.socket = factory(url.toString(), {
headers: { Authorization: this.apiKey },
});
}
this.socket.binaryType = "arraybuffer";
this.socket.onopen = () => {
if (this.endUtteranceSilenceThreshold === undefined ||
this.endUtteranceSilenceThreshold === null) {
return;
}
this.configureEndUtteranceSilenceThreshold(this.endUtteranceSilenceThreshold);
};
this.socket.onclose = ({ code, reason }) => {
if (!reason) {
if (code in RealtimeErrorMessages) {
reason = RealtimeErrorMessages[code];
}
}
this.listeners.close?.(code, reason);
};
this.socket.onerror = (event) => {
if (event.error)
this.listeners.error?.(event.error);
else
this.listeners.error?.(new Error(event.message));
};
this.socket.onmessage = ({ data }) => {
const message = JSON.parse(data.toString());
if ("error" in message) {
this.listeners.error?.(new RealtimeError(message.error));
return;
}
switch (message.message_type) {
case "SessionBegins": {
const openObject = {
sessionId: message.session_id,
expiresAt: new Date(message.expires_at),
};
resolve(openObject);
this.listeners.open?.(openObject);
break;
}
case "PartialTranscript": {
// message.created is actually a string when coming from the socket
message.created = new Date(message.created);
this.listeners.transcript?.(message);
this.listeners["transcript.partial"]?.(message);
break;
}
case "FinalTranscript": {
// message.created is actually a string when coming from the socket
message.created = new Date(message.created);
this.listeners.transcript?.(message);
this.listeners["transcript.final"]?.(message);
break;
}
case "SessionInformation": {
this.listeners.session_information?.(message);
break;
}
case "SessionTerminated": {
this.sessionTerminatedResolve?.();
break;
}
}
};
});
}
/**
* Send audio data to the server.
* @param audio - The audio data to send to the server.
*/
sendAudio(audio) {
this.send(audio);
}
/**
* Create a writable stream that can be used to send audio data to the server.
* @returns A writable stream that can be used to send audio data to the server.
*/
stream() {
return new WritableStream({
write: (chunk) => {
this.sendAudio(chunk);
},
});
}
/**
* Manually end an utterance
*/
forceEndUtterance() {
this.send(forceEndOfUtteranceMessage);
}
/**
* Configure the threshold for how long to wait before ending an utterance. Default is 700ms.
* @param threshold - The duration of the end utterance silence threshold in milliseconds.
* This value must be an integer between 0 and 20_000.
*/
configureEndUtteranceSilenceThreshold(threshold) {
this.send(`{"end_utterance_silence_threshold":${threshold}}`);
}
send(data) {
if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
throw new Error("Socket is not open for communication");
}
this.socket.send(data);
}
/**
* Close the connection to the server.
* @param waitForSessionTermination - If true, the method will wait for the session to be terminated before closing the connection.
* While waiting for the session to be terminated, you will receive the final transcript and session information.
*/
async close(waitForSessionTermination = true) {
if (this.socket) {
if (this.socket.readyState === this.socket.OPEN) {
if (waitForSessionTermination) {
const sessionTerminatedPromise = new Promise((resolve) => {
this.sessionTerminatedResolve = resolve;
});
this.socket.send(terminateSessionMessage$1);
await sessionTerminatedPromise;
}
else {
this.socket.send(terminateSessionMessage$1);
}
}
if (this.socket?.removeAllListeners)
this.socket.removeAllListeners();
this.socket.close();
}
this.listeners = {};
this.socket = undefined;
}
}
/**
* @deprecated Use RealtimeTranscriber instead
*/
class RealtimeService extends RealtimeTranscriber {
}
/**
* Energy-based VAD with adaptive noise-floor tracking and hangover. Pure JS,
* no dependencies. Suitable for the "which physical channel is speaking" task
* because the channels are already physically separated at capture — the harder
* problem (speech vs. non-speech in the wild) is one a customer can swap in a
* DNN VAD for via the `createVad` parameter.
*
* Tuning notes:
* - thresholdRatio below 2 will treat anything above noise as speech (too sensitive).
* - thresholdRatio above 6 will miss quiet utterance onsets/offsets.
* - noiseFloorAlpha above 0.1 makes the floor track quickly (good for non-stationary
* background) but risks slowly adapting *up* to a sustained low voice.
*/
class EnergyVad {
constructor(params = {}) {
this.hangoverRemaining = 0;
this.thresholdRatio = params.thresholdRatio ?? 3.0;
this.noiseFloorAlpha = params.noiseFloorAlpha ?? 0.05;
this.hangoverFrames = params.hangoverFrames ?? 10;
this.initialNoiseFloor = params.initialNoiseFloor ?? 1e-4;
this.noiseFloor = this.initialNoiseFloor;
}
process(frame) {
let sumSq = 0;
for (let i = 0; i < frame.length; i++) {
sumSq += frame[i] * frame[i];
}
const rms = frame.length > 0 ? Math.sqrt(sumSq / frame.length) : 0;
const threshold = this.noiseFloor * this.thresholdRatio;
let active = rms > threshold;
if (active) {
this.hangoverRemaining = this.hangoverFrames;
}
else if (this.hangoverRemaining > 0) {
this.hangoverRemaining--;
active = true;
// While in hangover, do not update noise floor — RMS may still reflect tail energy.
}
else {
this.noiseFloor =
this.noiseFloor * (1 - this.noiseFloorAlpha) +
rms * this.noiseFloorAlpha;
}
return { active, energy: rms };
}
reset() {
this.noiseFloor = this.initialNoiseFloor;
this.hangoverRemaining = 0;
}
}
/**
* Append-only ring buffer of VAD frames in stream-relative ms order.
* `pushFrame` is O(1) amortized; `framesInWindow` is O(n) over kept frames,
* which is fine for the per-word lookups we do (a 30 s window at 50 frames/s
* per channel × 2 channels = 3000 entries, scanned once per word).
*
* Runtime-agnostic — no DOM or Web Audio dependencies.
*/
class VadTimeline {
constructor(windowMs) {
this.windowMs = windowMs;
this.frames = [];
this.head = 0;
}
pushFrame(frame) {
this.frames.push(frame);
const cutoff = frame.ts - this.windowMs;
while (this.head < this.frames.length &&
this.frames[this.head].ts < cutoff) {
this.head++;
}
if (this.head > 1024 && this.head * 2 > this.frames.length) {
this.frames = this.frames.slice(this.head);
this.head = 0;
}
}
framesInWindow(startMs, endMs) {
const out = [];
for (let i = this.head; i < this.frames.length; i++) {
const f = this.frames[i];
if (f.ts < startMs)
continue;
if (f.ts > endMs)
break;
out.push(f);
}
return out;
}
clear() {
this.frames = [];
this.head = 0;
}
}
/**
* Sum per-channel active RMS over a window. Returns a Map from channel name
* to total score. Channels with zero score are omitted.
*/
function scoreChannels(frames) {
const scores = new Map();
for (const f of frames) {
if (!f.active)
continue;
scores.set(f.channel, (scores.get(f.channel) ?? 0) + f.rms);
}
return scores;
}
/**
* Decide which channel was dominant during a word's `[start, end]` window.
*
* - If no channel has any active VAD energy → `"unknown"`.
* - If the top channel beats the runner-up by at least `dominanceRatio` → top channel.
* - Else: top channel wins on absolute score; exact ties → `"unknown"`.
*/
function attributeWord(word, timeline, params) {
const scores = scoreChannels(timeline.framesInWindow(word.start, word.end));
if (scores.size === 0)
return "unknown";
const sorted = [...scores.entries()].sort((a, b) => b[1] - a[1]);
if (sorted.length === 1)
return sorted[0][0];
const [topName, topScore] = sorted[0];
const [runnerName, runnerScore] = sorted[1];
if (topScore >= params.dominanceRatio * runnerScore)
return topName;
if (topScore > runnerScore)
return topName;
if (runnerScore > topScore)
return runnerName;
return "unknown";
}
/**
* Duration-weighted majority of word channels. `"unknown"` if there are no
* words, every word resolved to `"unknown"`, or two channels tie exactly.
*/
function rollUpTurnChannel(words) {
const totals = new Map();
for (const w of words) {
if (!w.channel || w.channel === "unknown")
continue;
const dur = Math.max(0, w.end - w.start);
totals.set(w.channel, (totals.get(w.channel) ?? 0) + dur);
}
if (totals.size === 0)
return "unknown";
const sorted = [...totals.entries()].sort((a, b) => b[1] - a[1]);
if (sorted.length === 1)
return sorted[0][0];
const [topName, topMs] = sorted[0];
const [, runnerMs] = sorted[1];
if (topMs === runnerMs)
return "unknown";
return topName;
}
/**
* Mutate `turn` in place: write `turn.words[i].channel` for every word and set
* `turn.channel` to the duration-weighted rollup.
*
* Returns `void` because the transcriber owns the `TurnEvent` ref and forwards
* the same object to the customer listener — no need to allocate a copy.
*/
function attributeTurn(turn, timeline, params) {
for (const w of turn.words) {
w.channel = attributeWord(w, timeline, params);
}
turn.channel = rollUpTurnChannel(turn.words);
}
/**
* View any `AudioData` (ArrayBuffer / ArrayBufferView / typed array) as a
* little-endian Int16 sample sequence without copying. Callers must guarantee
* the underlying byte length is even.
*/
function toInt16View(audio) {
// AudioData is ArrayBufferLike per the public type, but in practice callers
// pass ArrayBuffer or a typed-array view. Handle both without copying.
if (audio instanceof Int16Array)
return audio;
if (ArrayBuffer.isView(audio)) {
const view = audio;
return new Int16Array(view.buffer, view.byteOffset, Math.floor(view.byteLength / 2));
}
return new Int16Array(audio);
}
const defaultStreamingUrl = "wss://streaming.assemblyai.com/v3/ws";
const terminateSessionMessage = `{"type":"Terminate"}`;
const DEFAULT_CONNECT_TIMEOUT_MS = 1000;
const DEFAULT_MAX_CONNECTION_RETRIES = 2;
const DEFAULT_CONNECTION_RETRY_DELAY_MS = 500;
/**
* Close/error codes that signal a permanent client-side problem (auth,
* billing, malformed config). A retry would hit the same failure, so the
* connection is never retried on these.
*/
const NON_RETRYABLE_CLOSE_CODES = new Set([
StreamingErrorType.BadSampleRate,
StreamingErrorType.AuthFailed,
StreamingErrorType.InsufficientFunds,
StreamingErrorType.FreeTierUser,
StreamingErrorType.BadSchema,
]);
function isRetryableCloseCode(code) {
return code !== 1000 && !NON_RETRYABLE_CLOSE_CODES.has(code);
}
/**
* Per-send chunk cap in milliseconds for the dual-channel mixer. The streaming
* server rejects audio messages longer than 1000 ms (`Input Duration Error`).
* If a backlog accumulates (e.g. when a browser tab is backgrounded and
* `setInterval` is throttled to ~1 Hz), `flushMix` loops and emits multiple
* sends each ≤ this cap until the buffers drain.
*/
const MAX_CHUNK_MS = 200;
/**
* Per-send minimum chunk size in milliseconds. The streaming server also
* rejects audio messages shorter than 50 ms with the same
* `Input Duration Error`, so the mixer waits until both per-channel buffers
* have at least this much accumulated before emitting. Final-flush (close
* path) bypasses this floor so the trailing partial buffer still gets sent.
*/
const MIN_CHUNK_MS = 50;
class StreamingTranscriber {
constructor(params) {
this.listeners = {};
// Dual-channel mode state (allocated only when params.channels is set).
this.isDualChannel = false;
this.vadFrameSamples = 0;
this.minChunkSamples = 0;
this.maxChunkSamples = 0;
this.params = {
...params,
websocketBaseUrl: params.websocketBaseUrl || defaultStreamingUrl,
};
if ("token" in params && params.token)
this.token = params.token;
if ("apiKey" in params && params.apiKey)
this.apiKey = params.apiKey;
if (!(this.token || this.apiKey)) {
throw new Error("API key or temporary token is required.");
}
const isSelfDescribing = params.encoding === "opus" ||
params.encoding === "ogg_opus" ||
params.encoding === "aac";
if (params.sampleRate === undefined &&
(!isSelfDescribing || params.channels)) {
throw new Error('`sampleRate` is required; it may only be omitted when `encoding` is "opus", "ogg_opus", or "aac" (these streams are self-describing) and dual-channel mode is not used.');
}
if (params.channels) {
if (params.channels.length !== 2) {
throw new Error("StreamingTranscriber.channels must have exactly 2 entries.");
}
const names = params.channels.map((c) => c.name);
if (new Set(names).size !== names.length) {
throw new Error("StreamingTranscriber.channels names must be unique.");
}
this.isDualChannel = true;
this.channelNames = names;
const att = params.channelAttribution ?? {};
this.attributionParams = {
dominanceRatio: att.dominanceRatio ?? 4,
timelineWindowMs: att.timelineWindowMs ?? 30_000,
createVad: att.createVad ?? (() => new EnergyVad()),
flushIntervalMs: att.flushIntervalMs ?? 50,
resolveUnknownChannelsMethod: att.resolveUnknownChannelsMethod ?? "window",
resolutionWindowWords: att.resolutionWindowWords ?? 2,
speakerHistoryMinRmsEvidence: att.speakerHistoryMinRmsEvidence ?? 0.5,
speakerHistoryDominanceRatio: att.speakerHistoryDominanceRatio ?? 3,
};
if (this.attributionParams.resolveUnknownChannelsMethod ===
"speaker-history") {
this.speakerHistory = new Map();
}
// 20 ms VAD frames at the transcriber's target sample rate. The
// constructor check above guarantees sampleRate in dual-channel mode.
const sampleRate = params.sampleRate;
this.vadFrameSamples = Math.max(1, Math.round(sampleRate * 0.02));
this.minChunkSamples = Math.max(1, Math.round(sampleRate * (MIN_CHUNK_MS / 1000)));
this.maxChunkSamples = Math.max(this.minChunkSamples, Math.round(sampleRate * (MAX_CHUNK_MS / 1000)));
this.channelBuffers = new Map(names.map((n) => [n, []]));
this.channelSamplesReceived = new Map(names.map((n) => [n, 0]));
this.channelVadFloatBuffers = new Map(names.map((n) => [n, new Float32Array(this.vadFrameSamples)]));
this.channelVadBufferIdx = new Map(names.map((n) => [n, 0]));
this.channelVads = new Map(names.map((n) => [n, this.attributionParams.createVad(n)]));
this.timeline = new VadTimeline(this.attributionParams.timelineWindowMs);
}
}
connectionUrl() {
const url = new URL(this.params.websocketBaseUrl ?? "");
if (url.protocol !== "wss:") {
throw new Error("Invalid protocol, must be wss");
}
const searchParams = new URLSearchParams();
if (this.token) {
searchParams.set("token", this.token);
}
if (this.params.sampleRate !== undefined) {
searchParams.set("sample_rate", this.params.sampleRate.toString());
}
if (this.params.endOfTurnConfidenceThreshold) {
searchParams.set("end_of_turn_confidence_threshold", this.params.endOfTurnConfidenceThreshold.toString());
}
if (this.params.minEndOfTurnSilenceWhenConfident !== undefined) {
if (this.params.minTurnSilence !== undefined) {
console.warn("[Deprecation Warning] Both `minEndOfTurnSilenceWhenConfident` and `minTurnSilence` are set. Using `minTurnSilence`; `minEndOfTurnSilenceWhenConfident` is deprecated.");
}
else {
console.warn("[Deprecation Warning] `minEndOfTurnSilenceWhenConfident` is deprecated and will be removed in a future release. Please use `minTurnSilence` instead.");
}
}
const effectiveMinTurnSilence = this.params.minTurnSilence ??
this.params.minEndOfTurnSilenceWhenConfident;
if (effectiveMinTurnSilence !== undefined) {
searchParams.set("min_turn_silence", effectiveMinTurnSilence.toString());
}
if (this.params.maxTurnSilence) {
searchParams.set("max_turn_silence", this.params.maxTurnSilence.toString());
}
if (this.params.vadThreshold !== undefined) {
searchParams.set("vad_threshold", this.params.vadThreshold.toString());
}
if (this.params.formatTurns) {
searchParams.set("format_turns", this.params.formatTurns.toString());
}
if (this.params.sessionHeartbeat !== undefined) {
searchParams.set("session_heartbeat", this.params.sessionHeartbeat.toString());
}
if (this.params.encoding) {
searchParams.set("encoding", this.params.encoding.toString());
}
if (this.params.keytermsPrompt) {
searchParams.set("keyterms_prompt", JSON.stringify(this.params.keytermsPrompt));
}
else if (this.params.keyterms) {
console.warn("[Deprecation Warning] `keyterms` is deprecated and will be removed in a future release. Please use `keytermsPrompt` instead.");
searchParams.set("keyterms_prompt", JSON.stringify(this.params.keyterms));
}
if (this.params.prompt) {
searchParams.set("prompt", this.params.prompt);
}
if (this.params.agentContext) {
searchParams.set("agent_context", this.params.agentContext);
}
if (this.params.filterProfanity) {
searchParams.set("filter_profanity", this.params.filterProfanity.toString());
}
if (this.params.speechModel === "u3-pro") {
console.warn("[Deprecation Warning] The speech model `u3-pro` is deprecated and will be removed in a future release. Please use `u3-rt-pro` instead.");
}
if (this.params.speechModel !== undefined) {
searchParams.set("speech_model", this.params.speechModel.toString());
}
if (this.params.languageCode !== undefined) {
console.warn("[Deprecation Warning] `languageCode` is deprecated and will be removed in a future release. Please use `languageCodes` instead.");
searchParams.set("language_code", this.params.languageCode);
}
if (this.params.languageCodes !== undefined) {
searchParams.set("language_codes", JSON.stringify(this.params.languageCodes));
}
if (this.params.languageDetection !== undefined) {
searchParams.set("language_detection", this.params.languageDetection.toString());
}
if (this.params.domain) {
searchParams.set("domain", this.params.domain);
}
if (this.params.inactivityTimeout !== undefined) {
searchParams.set("inactivity_timeout", this.params.inactivityTimeout.toString());
}
if (this.params.speakerLabels !== undefined) {
searchParams.set("speaker_labels", this.params.speakerLabels.toString());
}
if (this.params.maxSpeakers !== undefined) {
searchParams.set("max_speakers", this.params.maxSpeakers.toString());
}
if (this.params.voiceFocus) {
searchParams.set("voice_focus", this.params.voiceFocus);
}
if (this.params.voiceFocusThreshold !== undefined) {
searchParams.set("voice_focus_threshold", this.params.voiceFocusThreshold.toString());
}
if (this.params.continuousPartials !== undefined) {
searchParams.set("continuous_partials", this.params.continuousPartials.toString());
}
if (this.params.interruptionDelay !== undefined) {
searchParams.set("interruption_delay", this.params.interruptionDelay.toString());
}
if (this.params.turnLeftPadMs !== undefined) {
searchParams.set("turn_left_pad_ms", this.params.turnLeftPadMs.toString());
}
if (this.params.customerSupportAudioCapture) {
console.warn("`customerSupportAudioCapture=true` will record session audio. Only enable this when explicitly coordinating with AssemblyAI support.");
// The server's canonical wire name is `_customer_support_audio_capture`
// (leading underscore = "not officially supported / unstable"). The
// server also accepts `customer_support_audio_capture` via
// `populate_by_name=True`, but we send the underscore form to honor
// the server's stability marker.
searchParams.set("_customer_support_audio_capture", this.params.customerSupportAudioCapture.toString());
}
if (this.params.webhookUrl) {
searchParams.set("webhook_url", this.params.webhookUrl);
}
if (this.params.webhookAuthHeaderName) {
searchParams.set("webhook_auth_header_name", this.params.webhookAuthHeaderName);
}
if (this.params.webhookAuthHeaderValue) {
searchParams.set("webhook_auth_header_value", this.params.webhookAuthHeaderValue);
}
if (this.params.includePartialTurns !== undefined) {
searchParams.set("include_partial_turns", this.params.includePartialTurns.toString());
}
if (this.params.redactPii !== undefined) {
searchParams.set("redact_pii", this.params.redactPii.toString());
}
if (this.params.redactPiiPolicies !== undefined) {
searchParams.set("redact_pii_policies", JSON.stringify(this.params.redactPiiPolicies));
}
if (this.params.redactPiiSub !== undefined) {
searchParams.set("redact_pii_sub", this.params.redactPiiSub);
}
if (this.params.mode !== undefined) {
searchParams.set("mode", this.params.mode);
}
if (this.params.llmGateway !== undefined) {
searchParams.set("llm_gateway", JSON.stringify(this.params.llmGateway));
}
url.search = searchParams.toString();
return url;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
on(event, listener) {
this.listeners[event] = listener;
}
/**
* Open the streaming session.
*
* Resolves with the server's `Begin` event once the handshake completes. A
* single attempt is bounded by `connectTimeout` (default 1000ms); transient
* failures (timeout, network drop, unexpected close) are retried up to
* `maxConnectionRetries` times (default 2), waiting `connectionRetryDelay`
* (default 500ms) between attempts. Permanent failures (auth, insufficient
* funds, malformed config) are not retried.
*
* Unlike previously, a failed connection now rejects this promise rather
* than only invoking the `error` listener — necessary for the caller (and
* the retry loop) to observe the failure.
*/
async connect() {
if (this.socket) {
throw new Error("Already connected");
}
const maxRetries = this.params.maxConnectionRetries ?? DEFAULT_MAX_CONNECTION_RETRIES;
const retryDelay = this.params.connectionRetryDelay ?? DEFAULT_CONNECTION_RETRY_DELAY_MS;
let lastError;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
return await this.connectOnce();
}
catch (err) {
lastError = err;
const retryable = err.retryable === true;
if (!retryable || attempt === maxRetries) {
throw err;
}
console.warn(`Streaming connect attempt ${attempt + 1}/${maxRetries + 1} failed (${err.message}); retrying`);
if (retryDelay > 0) {
await new Promise((resolve) => setTimeout(resolve, retryDelay));
}
}
}
// The loop above always returns or throws; this only satisfies the type
// checker that a value is produced on every path.
throw lastError ?? new Error("Failed to connect to streaming server");
}
connectOnce() {
return new Promise((resolve, reject) => {
const url = this.connectionUrl();
const timeoutMs = this.params.connectTimeout ?? DEFAULT_CONNECT_TIMEOUT_MS;
// `settled` flips once this attempt has resolved (`Begin`) or rejected
// (timeout / pre-`Begin` close / error). Before it flips the socket
// handlers drive this promise; after it flips they revert to normal
// runtime dispatch (close / error / message listeners).
let settled = false;
let timer;
const failAttempt = (error) => {
if (settled)
return;
settled = true;
if (timer)
clearTimeout(timer);
this.discardPendingSocket();
reject(error);
};
const succeed = (begin) => {
if (settled)
return;
settled = true;
if (timer)
clearTimeout(timer);
resolve(begin);
};
if (timeoutMs > 0) {
timer = setTimeout(() => {
const err = new StreamingError(`Streaming connection timed out after ${timeoutMs}ms`);
err.retryable = true;
failAttempt(err);
}, timeoutMs);
}
if (this.token) {
this.socket = factory(url.toString());
}
else {
{
console.warn(`API key authentication is not supported for the StreamingTranscriber in browser environment. Use temporary token authentication instead.
Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/compat.md#browser-compatibility.`);
}
this.socket = factory(url.toString(), {
headers: { Authorization: this.apiKey },
});
}
this.socket.binaryType = "arraybuffer";
this.socket.onopen = () => { };
this.socket.onclose = ({ code, reason }) => {
if (!reason) {
if (code in StreamingErrorMessages) {
reason = StreamingErrorMessages[code];
}
}
// A close before `Begin` is a failed connection attempt — reject so
// connect() can retry (or surface a permanent failure).
if (!settled) {
const err = new StreamingError(reason || `Streaming connection closed (code=${code})`);
err.code = code;
err.retryable = isRetryableCloseCode(code);
failAttempt(err);
return;
}
// Stop the flush timer when the socket is gone (server-initiated close,
// network drop, etc.) — otherwise subsequent ticks call send() on a
// closed socket and spam the error listener.
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.flushTimer = undefined;
}
this.listeners.close?.(code, reason);
};
this.socket.onerror = (event) => {
const error = event.error ?? new Error(event.message);
// A socket error before `Begin` is a failed attempt → reject/retry.
if (!settled) {
error.retryable = true;
failAttempt(error);
return;
}
this.listeners.error?.(error);
};
this.socket.onmessage = ({ data }) => {
const message = JSON.parse(data.toString());
if ("error" in message) {
const err = new StreamingError(message.error);
if ("error_code" in message) {
err.code = message.error_code;
}
// A server error frame before `Begin` fails the attempt; the code
// decides whether a retry is worthwhile.
if (!settled) {
const attemptErr = err;
attemptErr.retryable =
err.code === undefined ? true : isRetryableCloseCode(err.code);
failAttempt(attemptErr);
return;
}
this.listeners.error?.(err);
return;
}
switch (message.type) {
case "Begin": {
succeed(message);
this.listeners.open?.(message);
break;
}
case "Turn": {
if (this.isDualChannel && this.timeline && this.attributionParams) {
attributeTurn(message, this.timeline, {
dominanceRatio: this.attributionParams.dominanceRatio,
});
switch (this.attributionParams.resolveUnknownChannelsMethod) {
case "window":
this.resolveUnknownChannelsByWindow(message);
break;
case "speaker-history":
this.resolveUnknownChannelsBySpeakerHistory(message);
break;
}
}
this.listeners.turn?.(message);
break;
}
case "SpeechStarted": {
this.listeners.speechStarted?.(message);
break;
}
case "LLMGatewayResponse": {
this.listeners.llmGatewayResponse?.(message);
break;
}
case "SpeakerRevision": {
this.listeners.speakerRevision?.(message);
break;
}
case "Warning": {
const warning = message;
console.warn(`Streaming warning (code=${warning.warning_code}): ${warning.warning}`);
this.listeners.warning?.(warning);
break;
}
case "Heartbeat": {
this.listeners.heartbeat?.(message);
break;
}
case "Termination": {
this.sessionTerminatedResolve?.();
break;
}
}
};
});
}
/** Tear down a half-open socket from a failed connection attempt. */
discardPendingSocket() {
if (!this.socket)
return;
try {
if (this.socket.removeAllListeners)
this.socket.removeAllListeners();
this.socket.close();
}
catch {
// Best-effort cleanup; a half-open socket may throw on close.
}
this.socket = undefined;
}
/**
* Returns a WritableStream that pumps PCM chunks into `sendAudio`. Single-channel
* only — in dual-channel mode use `sendAudio(pcm, { channel })` directly, since
* `WritableStream` has no place to carry a channel tag.
*/
stream() {
return new WritableStream({
write: (chunk) => {
this.sendAudio(chunk);
},
});
}
/**
* Send PCM audio.
*
* In single-channel mode, `audio` is forwarded directly to the WebSocket and
* `options` is ignored.
*
* In dual-channel mode (when `channels` is configured), `options.channel` is
* REQUIRED and must match one of the declared channel names. Per-channel PCM is
* fed into that channel's VAD, accumulated into a per-channel ring buffer, and
* a scheduled flush (`channelAttribution.flushIntervalMs`, default 50ms) mixes
* the buffers into mono before sending to the WebSocket.
*/
sendAudio(audio, options) {
if (!this.isDualChannel) {
this.send(audio);
return;
}
if (!options?.channel) {
throw new Error("StreamingTranscriber is in dual-channel mode; sendAudio requires { channel }.");
}
if (!this.channelNames.includes(options.channel)) {
throw new Error(`Unknown channel "${options.channel}"; declared channels: ${this.channelNames.join(", ")}.`);
}
this.ingestChannelAudio(options.channel, audio);
}
ingestChannelAudio(name, audio) {
const samples = toInt16View(audio);
const buf = this.channelBuffers.get(name);
const vadBuf = this.channelVadFloatBuffers.get(name);
let vadIdx = this.channelVadBufferIdx.get(name);
let received = this.channelSamplesReceived.get(name);
const vad = this.channelVads.get(name);
const sampleRate = this.params.sampleRate;
const frameSize = this.vadFrameSamples;
for (let i = 0; i < samples.length; i++) {
const s = samples[i];
buf.push(s);
vadBuf[vadIdx++] = s / 0x8000;
received++;
if (vadIdx === frameSize) {
const result = vad.process(vadBuf);
const frame = {
ts: (received / sampleRate) * 1000,
channel: name,
active: result.active,
rms: result.energy,
};
this.timeline.pushFrame(frame);
this.listeners.vad?.(frame);
vadIdx = 0;
}
}
this.channelVadBufferIdx.set(name, vadIdx);
this.channelSamplesReceived.set(name, received);
if (!this.flushTimer)
this.startFlushTimer();
}
startFlushTimer() {
this.flushTimer = setInterval(() => this.flushMix(), this.attributionParams.flushIntervalMs);
}
flushMix(force = false) {
if (!this.channelNames || !this.channelBuffers)
return;
const bufs = this.channelNames.map((n) => this.channelBuffers.get(n));
const divisor = bufs.length;
// Loop so a backlog (e.g. accumulated while a browser tab was throttled in
// the background) drains as multiple sends, each capped at MAX_CHUNK_MS.
// Without the cap a single message could exceed the server's 1000 ms input
// duration limit and be rejected with code 3007.
for (;;) {
let mixLen = Infinity;
for (const b of bufs)
if (b.length < mixLen)
mixLen = b.length;
if (!Number.isFinite(mixLen) || mixLen === 0)
return;
// The streaming server rejects audio messages shorter than 50 ms with
// `Input Duration Error`. Wait until both per-channel buffers have at
// least minChunkSamples worth queued before emitting. The `force` path
// (final flush on close) bypasses this so the trailing partial buffer
// still gets through.
if (!force && mixLen < this.minChunkSamples)
return;
if (mixLen > this.maxChunkSamples)
mixLen = this.maxChunkSamples;
const out = new Int16Array(mixLen);
for (let i = 0; i < mixLen; i++) {
let sum = 0;
for (let c = 0; c < divisor; c++)
sum += bufs[c][i];
const avg = Math.round(sum / divisor);
out[i] = avg < -32768 ? -32768 : avg > 32767 ? 32767 : avg;
}
for (const b of bufs)
b.splice(0, mixLen);
try {
this.send(out.buffer);
}
catch (err) {
this.listeners.error?.(err);
return;
}
}
}
/**
* Fill in words whose per-word VAD attribution was `"unknown"` by looking
* at the dominant non-`"unknown"` channel among ±N neighbors in the same
* turn. Words with no non-`"unknown"` neighbors stay `"unknown"`. Confident
* per-word VAD decisions are never modified.
*
* Local temporal heuristic — ignores `speaker_label`, so it works even when
* AAI's diarization re-uses the same label for two physically distinct
* voices. Each resolved word gets `channelResolved: true` so downstream
* renderers can distinguish inferred channels from directly-measured ones.
*/
resolveUnknownChannelsByWindow(turn) {
if (!this.attributionParams)
return;
const window = this.attributionParams.resolutionWindowWords;
const words = turn.words;
let mutated = false;
for (let i = 0; i < words.length; i++) {
if (words[i].channel !== "unknown")
continue;
const tally = new Map();
const lo = Math.max(0, i - window);
const hi = Math.min(words.length - 1, i + window);
for (let j = lo; j <= hi; j++) {
if (j === i)
continue;
const ch = words[j].channel;
if (!ch || ch === "unknown")
continue;
tally.set(ch, (tally.get(ch) ?? 0) + 1);
}
if (tally.size === 0)
continue;
// Pick the dominant neighbor channel. Ties → leave `"unknown"` (rare;
// would require an equal count of mic and system neighbors).
let top;
let topCount = 0;
let tied = false;
for (const [name, count] of tally) {
if (count > topCount) {
top = name;
topCount = count;
tied = false;
}
else if (count === topCount) {
tied = true;
}
}
if (top && !tied) {
words[i].channel = top;
words[i].channelResolved = true;
mutated = true;
}
}
// Recompute the rollup only