assemblyai
Version:
The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.
1,095 lines (1,079 loc) • 77.2 kB
JavaScript
import ws from 'ws';
/**
* Thrown when `DualChannelCapture` is constructed in a non-browser environment
* (no `globalThis.AudioContext`). The helper is intentionally surfaced from the
* main entrypoint so the import path is uniform across runtimes; the runtime
* guard moves to construction time.
*/
class BrowserOnlyError extends Error {
constructor(message = "DualChannelCapture requires a browser environment (AudioContext is undefined).") {
super(message);
this.name = "BrowserOnlyError";
}
}
/******************************************************************************
Copyright (c) Microsoft Corporation.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
***************************************************************************** */
/* global Reflect, Promise, SuppressedError, Symbol, Iterator */
function __rest(s, e) {
var t = {};
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
t[p] = s[p];
if (s != null && typeof Object.getOwnPropertySymbols === "function")
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
t[p[i]] = s[p[i]];
}
return t;
}
function __awaiter(thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
}
typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
var e = new Error(message);
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
};
const { WritableStream } = typeof window !== "undefined"
? window
: typeof global !== "undefined"
? global
: globalThis;
const factory = (url, params) => new ws(url, params);
const RealtimeErrorType = {
BadSampleRate: 4000,
AuthFailed: 4001,
/**
* @deprecated Use InsufficientFunds or FreeTierUser instead
*/
InsufficientFundsOrFreeAccount: 4002,
InsufficientFunds: 4002,
FreeTierUser: 4003,
NonexistentSessionId: 4004,
SessionExpired: 4008,
ClosedSession: 4010,
RateLimited: 4029,
UniqueSessionViolation: 4030,
SessionTimeout: 4031,
AudioTooShort: 4032,
AudioTooLong: 4033,
AudioTooSmallToTranscode: 4034,
/**
* @deprecated Don't use
*/
BadJson: 4100,
BadSchema: 4101,
TooManyStreams: 4102,
Reconnected: 4103,
/**
* @deprecated Don't use
*/
ReconnectAttemptsExhausted: 1013,
WordBoostParameterParsingFailed: 4104,
};
const RealtimeErrorMessages = {
[RealtimeErrorType.BadSampleRate]: "Sample rate must be a positive integer",
[RealtimeErrorType.AuthFailed]: "Not Authorized",
[RealtimeErrorType.InsufficientFunds]: "Insufficient funds",
[RealtimeErrorType.FreeTierUser]: "This feature is paid-only and requires you to add a credit card. Please visit https://app.assemblyai.com/ to add a credit card to your account.",
[RealtimeErrorType.NonexistentSessionId]: "Session ID does not exist",
[RealtimeErrorType.SessionExpired]: "Session has expired",
[RealtimeErrorType.ClosedSession]: "Session is closed",
[RealtimeErrorType.RateLimited]: "Rate limited",
[RealtimeErrorType.UniqueSessionViolation]: "Unique session violation",
[RealtimeErrorType.SessionTimeout]: "Session Timeout",
[RealtimeErrorType.AudioTooShort]: "Audio too short",
[RealtimeErrorType.AudioTooLong]: "Audio too long",
[RealtimeErrorType.AudioTooSmallToTranscode]: "Audio too small to transcode",
[RealtimeErrorType.BadJson]: "Bad JSON",
[RealtimeErrorType.BadSchema]: "Bad schema",
[RealtimeErrorType.TooManyStreams]: "Too many streams",
[RealtimeErrorType.Reconnected]: "This session has been reconnected. This WebSocket is no longer valid.",
[RealtimeErrorType.ReconnectAttemptsExhausted]: "Reconnect attempts exhausted",
[RealtimeErrorType.WordBoostParameterParsingFailed]: "Could not parse word boost parameter",
};
class RealtimeError extends Error {
}
const StreamingErrorType = {
BadSampleRate: 4000,
AuthFailed: 4001,
InsufficientFunds: 4002,
FreeTierUser: 4003,
NonexistentSessionId: 4004,
SessionExpired: 4008,
ClosedSession: 4010,
RateLimited: 4029,
UniqueSessionViolation: 4030,
SessionTimeout: 4031,
AudioTooShort: 4032,
AudioTooLong: 4033,
AudioTooSmallToTranscode: 4034,
BadSchema: 4101,
TooManyStreams: 4102,
Reconnected: 4103,
ServerError: 3005,
InputValidationError: 3006,
AudioChunkDurationViolation: 3007,
MaxSessionDurationExceeded: 3008,
ConcurrencyLimitExceeded: 3009,
};
const StreamingErrorMessages = {
[StreamingErrorType.ServerError]: "Server error",
[StreamingErrorType.InputValidationError]: "Input validation error",
[StreamingErrorType.AudioChunkDurationViolation]: "Audio chunk duration violation",
[StreamingErrorType.MaxSessionDurationExceeded]: "Session expired: maximum session duration exceeded",
[StreamingErrorType.ConcurrencyLimitExceeded]: "Too many concurrent sessions",
[StreamingErrorType.BadSampleRate]: "Sample rate must be a positive integer",
[StreamingErrorType.AuthFailed]: "Not Authorized",
[StreamingErrorType.InsufficientFunds]: "Insufficient funds",
[StreamingErrorType.FreeTierUser]: "This feature is paid-only and requires you to add a credit card. Please visit https://app.assemblyai.com/ to add a credit card to your account.",
[StreamingErrorType.NonexistentSessionId]: "Session ID does not exist",
[StreamingErrorType.SessionExpired]: "Session has expired",
[StreamingErrorType.ClosedSession]: "Session is closed",
[StreamingErrorType.RateLimited]: "Rate limited",
[StreamingErrorType.UniqueSessionViolation]: "Unique session violation",
[StreamingErrorType.SessionTimeout]: "Session Timeout",
[StreamingErrorType.AudioTooShort]: "Audio too short",
[StreamingErrorType.AudioTooLong]: "Audio too long",
[StreamingErrorType.AudioTooSmallToTranscode]: "Audio too small to transcode",
[StreamingErrorType.BadSchema]: "Bad schema",
[StreamingErrorType.TooManyStreams]: "Too many streams",
[StreamingErrorType.Reconnected]: "This session has been reconnected. This WebSocket is no longer valid.",
};
class StreamingError extends Error {
}
const defaultRealtimeUrl = "wss://api.assemblyai.com/v2/realtime/ws";
const forceEndOfUtteranceMessage = `{"force_end_utterance":true}`;
const terminateSessionMessage$1 = `{"terminate_session":true}`;
/**
* RealtimeTranscriber connects to the Streaming Speech-to-Text API and lets you transcribe audio in real-time.
*/
class RealtimeTranscriber {
/**
* Create a new RealtimeTranscriber.
* @param params - Parameters to configure the RealtimeTranscriber
*/
constructor(params) {
var _a, _b;
this.listeners = {};
this.realtimeUrl = (_a = params.realtimeUrl) !== null && _a !== void 0 ? _a : defaultRealtimeUrl;
this.sampleRate = (_b = params.sampleRate) !== null && _b !== void 0 ? _b : 16000;
this.wordBoost = params.wordBoost;
this.encoding = params.encoding;
this.endUtteranceSilenceThreshold = params.endUtteranceSilenceThreshold;
this.disablePartialTranscripts = params.disablePartialTranscripts;
if ("token" in params && params.token)
this.token = params.token;
if ("apiKey" in params && params.apiKey)
this.apiKey = params.apiKey;
if (!(this.token || this.apiKey)) {
throw new Error("API key or temporary token is required.");
}
}
connectionUrl() {
const url = new URL(this.realtimeUrl);
if (url.protocol !== "wss:") {
throw new Error("Invalid protocol, must be wss");
}
const searchParams = new URLSearchParams();
if (this.token) {
searchParams.set("token", this.token);
}
searchParams.set("sample_rate", this.sampleRate.toString());
if (this.wordBoost && this.wordBoost.length > 0) {
searchParams.set("word_boost", JSON.stringify(this.wordBoost));
}
if (this.encoding) {
searchParams.set("encoding", this.encoding);
}
searchParams.set("enable_extra_session_information", "true");
if (this.disablePartialTranscripts) {
searchParams.set("disable_partial_transcripts", this.disablePartialTranscripts.toString());
}
url.search = searchParams.toString();
return url;
}
/**
* Add a listener for an event.
* @param event - The event to listen for.
* @param listener - The function to call when the event is emitted.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
on(event, listener) {
this.listeners[event] = listener;
}
/**
* Connect to the server and begin a new session.
* @returns A promise that resolves when the connection is established and the session begins.
*/
connect() {
return new Promise((resolve) => {
if (this.socket) {
throw new Error("Already connected");
}
const url = this.connectionUrl();
if (this.token) {
this.socket = factory(url.toString());
}
else {
this.socket = factory(url.toString(), {
headers: { Authorization: this.apiKey },
});
}
this.socket.binaryType = "arraybuffer";
this.socket.onopen = () => {
if (this.endUtteranceSilenceThreshold === undefined ||
this.endUtteranceSilenceThreshold === null) {
return;
}
this.configureEndUtteranceSilenceThreshold(this.endUtteranceSilenceThreshold);
};
this.socket.onclose = ({ code, reason }) => {
var _a, _b;
if (!reason) {
if (code in RealtimeErrorMessages) {
reason = RealtimeErrorMessages[code];
}
}
(_b = (_a = this.listeners).close) === null || _b === void 0 ? void 0 : _b.call(_a, code, reason);
};
this.socket.onerror = (event) => {
var _a, _b, _c, _d;
if (event.error)
(_b = (_a = this.listeners).error) === null || _b === void 0 ? void 0 : _b.call(_a, event.error);
else
(_d = (_c = this.listeners).error) === null || _d === void 0 ? void 0 : _d.call(_c, new Error(event.message));
};
this.socket.onmessage = ({ data }) => {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
const message = JSON.parse(data.toString());
if ("error" in message) {
(_b = (_a = this.listeners).error) === null || _b === void 0 ? void 0 : _b.call(_a, new RealtimeError(message.error));
return;
}
switch (message.message_type) {
case "SessionBegins": {
const openObject = {
sessionId: message.session_id,
expiresAt: new Date(message.expires_at),
};
resolve(openObject);
(_d = (_c = this.listeners).open) === null || _d === void 0 ? void 0 : _d.call(_c, openObject);
break;
}
case "PartialTranscript": {
// message.created is actually a string when coming from the socket
message.created = new Date(message.created);
(_f = (_e = this.listeners).transcript) === null || _f === void 0 ? void 0 : _f.call(_e, message);
(_h = (_g = this.listeners)["transcript.partial"]) === null || _h === void 0 ? void 0 : _h.call(_g, message);
break;
}
case "FinalTranscript": {
// message.created is actually a string when coming from the socket
message.created = new Date(message.created);
(_k = (_j = this.listeners).transcript) === null || _k === void 0 ? void 0 : _k.call(_j, message);
(_m = (_l = this.listeners)["transcript.final"]) === null || _m === void 0 ? void 0 : _m.call(_l, message);
break;
}
case "SessionInformation": {
(_p = (_o = this.listeners).session_information) === null || _p === void 0 ? void 0 : _p.call(_o, message);
break;
}
case "SessionTerminated": {
(_q = this.sessionTerminatedResolve) === null || _q === void 0 ? void 0 : _q.call(this);
break;
}
}
};
});
}
/**
* Send audio data to the server.
* @param audio - The audio data to send to the server.
*/
sendAudio(audio) {
this.send(audio);
}
/**
* Create a writable stream that can be used to send audio data to the server.
* @returns A writable stream that can be used to send audio data to the server.
*/
stream() {
return new WritableStream({
write: (chunk) => {
this.sendAudio(chunk);
},
});
}
/**
* Manually end an utterance
*/
forceEndUtterance() {
this.send(forceEndOfUtteranceMessage);
}
/**
* Configure the threshold for how long to wait before ending an utterance. Default is 700ms.
* @param threshold - The duration of the end utterance silence threshold in milliseconds.
* This value must be an integer between 0 and 20_000.
*/
configureEndUtteranceSilenceThreshold(threshold) {
this.send(`{"end_utterance_silence_threshold":${threshold}}`);
}
send(data) {
if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
throw new Error("Socket is not open for communication");
}
this.socket.send(data);
}
/**
* Close the connection to the server.
* @param waitForSessionTermination - If true, the method will wait for the session to be terminated before closing the connection.
* While waiting for the session to be terminated, you will receive the final transcript and session information.
*/
close() {
return __awaiter(this, arguments, void 0, function* (waitForSessionTermination = true) {
var _a;
if (this.socket) {
if (this.socket.readyState === this.socket.OPEN) {
if (waitForSessionTermination) {
const sessionTerminatedPromise = new Promise((resolve) => {
this.sessionTerminatedResolve = resolve;
});
this.socket.send(terminateSessionMessage$1);
yield sessionTerminatedPromise;
}
else {
this.socket.send(terminateSessionMessage$1);
}
}
if ((_a = this.socket) === null || _a === void 0 ? void 0 : _a.removeAllListeners)
this.socket.removeAllListeners();
this.socket.close();
}
this.listeners = {};
this.socket = undefined;
});
}
}
/**
* @deprecated Use RealtimeTranscriber instead
*/
class RealtimeService extends RealtimeTranscriber {
}
/**
* Energy-based VAD with adaptive noise-floor tracking and hangover. Pure JS,
* no dependencies. Suitable for the "which physical channel is speaking" task
* because the channels are already physically separated at capture — the harder
* problem (speech vs. non-speech in the wild) is one a customer can swap in a
* DNN VAD for via the `createVad` parameter.
*
* Tuning notes:
* - thresholdRatio below 2 will treat anything above noise as speech (too sensitive).
* - thresholdRatio above 6 will miss quiet utterance onsets/offsets.
* - noiseFloorAlpha above 0.1 makes the floor track quickly (good for non-stationary
* background) but risks slowly adapting *up* to a sustained low voice.
*/
class EnergyVad {
constructor(params = {}) {
var _a, _b, _c, _d;
this.hangoverRemaining = 0;
this.thresholdRatio = (_a = params.thresholdRatio) !== null && _a !== void 0 ? _a : 3.0;
this.noiseFloorAlpha = (_b = params.noiseFloorAlpha) !== null && _b !== void 0 ? _b : 0.05;
this.hangoverFrames = (_c = params.hangoverFrames) !== null && _c !== void 0 ? _c : 10;
this.initialNoiseFloor = (_d = params.initialNoiseFloor) !== null && _d !== void 0 ? _d : 1e-4;
this.noiseFloor = this.initialNoiseFloor;
}
process(frame) {
let sumSq = 0;
for (let i = 0; i < frame.length; i++) {
sumSq += frame[i] * frame[i];
}
const rms = frame.length > 0 ? Math.sqrt(sumSq / frame.length) : 0;
const threshold = this.noiseFloor * this.thresholdRatio;
let active = rms > threshold;
if (active) {
this.hangoverRemaining = this.hangoverFrames;
}
else if (this.hangoverRemaining > 0) {
this.hangoverRemaining--;
active = true;
// While in hangover, do not update noise floor — RMS may still reflect tail energy.
}
else {
this.noiseFloor =
this.noiseFloor * (1 - this.noiseFloorAlpha) +
rms * this.noiseFloorAlpha;
}
return { active, energy: rms };
}
reset() {
this.noiseFloor = this.initialNoiseFloor;
this.hangoverRemaining = 0;
}
}
/**
* Append-only ring buffer of VAD frames in stream-relative ms order.
* `pushFrame` is O(1) amortized; `framesInWindow` is O(n) over kept frames,
* which is fine for the per-word lookups we do (a 30 s window at 50 frames/s
* per channel × 2 channels = 3000 entries, scanned once per word).
*
* Runtime-agnostic — no DOM or Web Audio dependencies.
*/
class VadTimeline {
constructor(windowMs) {
this.windowMs = windowMs;
this.frames = [];
this.head = 0;
}
pushFrame(frame) {
this.frames.push(frame);
const cutoff = frame.ts - this.windowMs;
while (this.head < this.frames.length &&
this.frames[this.head].ts < cutoff) {
this.head++;
}
if (this.head > 1024 && this.head * 2 > this.frames.length) {
this.frames = this.frames.slice(this.head);
this.head = 0;
}
}
framesInWindow(startMs, endMs) {
const out = [];
for (let i = this.head; i < this.frames.length; i++) {
const f = this.frames[i];
if (f.ts < startMs)
continue;
if (f.ts > endMs)
break;
out.push(f);
}
return out;
}
clear() {
this.frames = [];
this.head = 0;
}
}
/**
* Sum per-channel active RMS over a window. Returns a Map from channel name
* to total score. Channels with zero score are omitted.
*/
function scoreChannels(frames) {
var _a;
const scores = new Map();
for (const f of frames) {
if (!f.active)
continue;
scores.set(f.channel, ((_a = scores.get(f.channel)) !== null && _a !== void 0 ? _a : 0) + f.rms);
}
return scores;
}
/**
* Decide which channel was dominant during a word's `[start, end]` window.
*
* - If no channel has any active VAD energy → `"unknown"`.
* - If the top channel beats the runner-up by at least `dominanceRatio` → top channel.
* - Else: top channel wins on absolute score; exact ties → `"unknown"`.
*/
function attributeWord(word, timeline, params) {
const scores = scoreChannels(timeline.framesInWindow(word.start, word.end));
if (scores.size === 0)
return "unknown";
const sorted = [...scores.entries()].sort((a, b) => b[1] - a[1]);
if (sorted.length === 1)
return sorted[0][0];
const [topName, topScore] = sorted[0];
const [runnerName, runnerScore] = sorted[1];
if (topScore >= params.dominanceRatio * runnerScore)
return topName;
if (topScore > runnerScore)
return topName;
if (runnerScore > topScore)
return runnerName;
return "unknown";
}
/**
* Duration-weighted majority of word channels. `"unknown"` if there are no
* words, every word resolved to `"unknown"`, or two channels tie exactly.
*/
function rollUpTurnChannel(words) {
var _a;
const totals = new Map();
for (const w of words) {
if (!w.channel || w.channel === "unknown")
continue;
const dur = Math.max(0, w.end - w.start);
totals.set(w.channel, ((_a = totals.get(w.channel)) !== null && _a !== void 0 ? _a : 0) + dur);
}
if (totals.size === 0)
return "unknown";
const sorted = [...totals.entries()].sort((a, b) => b[1] - a[1]);
if (sorted.length === 1)
return sorted[0][0];
const [topName, topMs] = sorted[0];
const [, runnerMs] = sorted[1];
if (topMs === runnerMs)
return "unknown";
return topName;
}
/**
* Mutate `turn` in place: write `turn.words[i].channel` for every word and set
* `turn.channel` to the duration-weighted rollup.
*
* Returns `void` because the transcriber owns the `TurnEvent` ref and forwards
* the same object to the customer listener — no need to allocate a copy.
*/
function attributeTurn(turn, timeline, params) {
for (const w of turn.words) {
w.channel = attributeWord(w, timeline, params);
}
turn.channel = rollUpTurnChannel(turn.words);
}
/**
* View any `AudioData` (ArrayBuffer / ArrayBufferView / typed array) as a
* little-endian Int16 sample sequence without copying. Callers must guarantee
* the underlying byte length is even.
*/
function toInt16View(audio) {
// AudioData is ArrayBufferLike per the public type, but in practice callers
// pass ArrayBuffer or a typed-array view. Handle both without copying.
if (audio instanceof Int16Array)
return audio;
if (ArrayBuffer.isView(audio)) {
const view = audio;
return new Int16Array(view.buffer, view.byteOffset, Math.floor(view.byteLength / 2));
}
return new Int16Array(audio);
}
const defaultStreamingUrl = "wss://streaming.assemblyai.com/v3/ws";
const terminateSessionMessage = `{"type":"Terminate"}`;
const DEFAULT_CONNECT_TIMEOUT_MS = 1000;
const DEFAULT_MAX_CONNECTION_RETRIES = 2;
const DEFAULT_CONNECTION_RETRY_DELAY_MS = 500;
/**
* Close/error codes that signal a permanent client-side problem (auth,
* billing, malformed config). A retry would hit the same failure, so the
* connection is never retried on these.
*/
const NON_RETRYABLE_CLOSE_CODES = new Set([
StreamingErrorType.BadSampleRate,
StreamingErrorType.AuthFailed,
StreamingErrorType.InsufficientFunds,
StreamingErrorType.FreeTierUser,
StreamingErrorType.BadSchema,
]);
function isRetryableCloseCode(code) {
return code !== 1000 && !NON_RETRYABLE_CLOSE_CODES.has(code);
}
/**
* Per-send chunk cap in milliseconds for the dual-channel mixer. The streaming
* server rejects audio messages longer than 1000 ms (`Input Duration Error`).
* If a backlog accumulates (e.g. when a browser tab is backgrounded and
* `setInterval` is throttled to ~1 Hz), `flushMix` loops and emits multiple
* sends each ≤ this cap until the buffers drain.
*/
const MAX_CHUNK_MS = 200;
/**
* Per-send minimum chunk size in milliseconds. The streaming server also
* rejects audio messages shorter than 50 ms with the same
* `Input Duration Error`, so the mixer waits until both per-channel buffers
* have at least this much accumulated before emitting. Final-flush (close
* path) bypasses this floor so the trailing partial buffer still gets sent.
*/
const MIN_CHUNK_MS = 50;
class StreamingTranscriber {
constructor(params) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
this.listeners = {};
// Dual-channel mode state (allocated only when params.channels is set).
this.isDualChannel = false;
this.vadFrameSamples = 0;
this.minChunkSamples = 0;
this.maxChunkSamples = 0;
this.params = Object.assign(Object.assign({}, params), { websocketBaseUrl: params.websocketBaseUrl || defaultStreamingUrl });
if ("token" in params && params.token)
this.token = params.token;
if ("apiKey" in params && params.apiKey)
this.apiKey = params.apiKey;
if (!(this.token || this.apiKey)) {
throw new Error("API key or temporary token is required.");
}
const isSelfDescribing = params.encoding === "opus" ||
params.encoding === "ogg_opus" ||
params.encoding === "aac";
if (params.sampleRate === undefined &&
(!isSelfDescribing || params.channels)) {
throw new Error('`sampleRate` is required; it may only be omitted when `encoding` is "opus", "ogg_opus", or "aac" (these streams are self-describing) and dual-channel mode is not used.');
}
if (params.channels) {
if (params.channels.length !== 2) {
throw new Error("StreamingTranscriber.channels must have exactly 2 entries.");
}
const names = params.channels.map((c) => c.name);
if (new Set(names).size !== names.length) {
throw new Error("StreamingTranscriber.channels names must be unique.");
}
this.isDualChannel = true;
this.channelNames = names;
const att = (_a = params.channelAttribution) !== null && _a !== void 0 ? _a : {};
this.attributionParams = {
dominanceRatio: (_b = att.dominanceRatio) !== null && _b !== void 0 ? _b : 4,
timelineWindowMs: (_c = att.timelineWindowMs) !== null && _c !== void 0 ? _c : 30000,
createVad: (_d = att.createVad) !== null && _d !== void 0 ? _d : (() => new EnergyVad()),
flushIntervalMs: (_e = att.flushIntervalMs) !== null && _e !== void 0 ? _e : 50,
resolveUnknownChannelsMethod: (_f = att.resolveUnknownChannelsMethod) !== null && _f !== void 0 ? _f : "window",
resolutionWindowWords: (_g = att.resolutionWindowWords) !== null && _g !== void 0 ? _g : 2,
speakerHistoryMinRmsEvidence: (_h = att.speakerHistoryMinRmsEvidence) !== null && _h !== void 0 ? _h : 0.5,
speakerHistoryDominanceRatio: (_j = att.speakerHistoryDominanceRatio) !== null && _j !== void 0 ? _j : 3,
};
if (this.attributionParams.resolveUnknownChannelsMethod ===
"speaker-history") {
this.speakerHistory = new Map();
}
// 20 ms VAD frames at the transcriber's target sample rate. The
// constructor check above guarantees sampleRate in dual-channel mode.
const sampleRate = params.sampleRate;
this.vadFrameSamples = Math.max(1, Math.round(sampleRate * 0.02));
this.minChunkSamples = Math.max(1, Math.round(sampleRate * (MIN_CHUNK_MS / 1000)));
this.maxChunkSamples = Math.max(this.minChunkSamples, Math.round(sampleRate * (MAX_CHUNK_MS / 1000)));
this.channelBuffers = new Map(names.map((n) => [n, []]));
this.channelSamplesReceived = new Map(names.map((n) => [n, 0]));
this.channelVadFloatBuffers = new Map(names.map((n) => [n, new Float32Array(this.vadFrameSamples)]));
this.channelVadBufferIdx = new Map(names.map((n) => [n, 0]));
this.channelVads = new Map(names.map((n) => [n, this.attributionParams.createVad(n)]));
this.timeline = new VadTimeline(this.attributionParams.timelineWindowMs);
}
}
connectionUrl() {
var _a, _b;
const url = new URL((_a = this.params.websocketBaseUrl) !== null && _a !== void 0 ? _a : "");
if (url.protocol !== "wss:") {
throw new Error("Invalid protocol, must be wss");
}
const searchParams = new URLSearchParams();
if (this.token) {
searchParams.set("token", this.token);
}
if (this.params.sampleRate !== undefined) {
searchParams.set("sample_rate", this.params.sampleRate.toString());
}
if (this.params.endOfTurnConfidenceThreshold) {
searchParams.set("end_of_turn_confidence_threshold", this.params.endOfTurnConfidenceThreshold.toString());
}
if (this.params.minEndOfTurnSilenceWhenConfident !== undefined) {
if (this.params.minTurnSilence !== undefined) {
console.warn("[Deprecation Warning] Both `minEndOfTurnSilenceWhenConfident` and `minTurnSilence` are set. Using `minTurnSilence`; `minEndOfTurnSilenceWhenConfident` is deprecated.");
}
else {
console.warn("[Deprecation Warning] `minEndOfTurnSilenceWhenConfident` is deprecated and will be removed in a future release. Please use `minTurnSilence` instead.");
}
}
const effectiveMinTurnSilence = (_b = this.params.minTurnSilence) !== null && _b !== void 0 ? _b : this.params.minEndOfTurnSilenceWhenConfident;
if (effectiveMinTurnSilence !== undefined) {
searchParams.set("min_turn_silence", effectiveMinTurnSilence.toString());
}
if (this.params.maxTurnSilence) {
searchParams.set("max_turn_silence", this.params.maxTurnSilence.toString());
}
if (this.params.vadThreshold !== undefined) {
searchParams.set("vad_threshold", this.params.vadThreshold.toString());
}
if (this.params.formatTurns) {
searchParams.set("format_turns", this.params.formatTurns.toString());
}
if (this.params.sessionHeartbeat !== undefined) {
searchParams.set("session_heartbeat", this.params.sessionHeartbeat.toString());
}
if (this.params.encoding) {
searchParams.set("encoding", this.params.encoding.toString());
}
if (this.params.keytermsPrompt) {
searchParams.set("keyterms_prompt", JSON.stringify(this.params.keytermsPrompt));
}
else if (this.params.keyterms) {
console.warn("[Deprecation Warning] `keyterms` is deprecated and will be removed in a future release. Please use `keytermsPrompt` instead.");
searchParams.set("keyterms_prompt", JSON.stringify(this.params.keyterms));
}
if (this.params.prompt) {
searchParams.set("prompt", this.params.prompt);
}
if (this.params.agentContext) {
searchParams.set("agent_context", this.params.agentContext);
}
if (this.params.filterProfanity) {
searchParams.set("filter_profanity", this.params.filterProfanity.toString());
}
if (this.params.speechModel === "u3-pro") {
console.warn("[Deprecation Warning] The speech model `u3-pro` is deprecated and will be removed in a future release. Please use `u3-rt-pro` instead.");
}
if (this.params.speechModel !== undefined) {
searchParams.set("speech_model", this.params.speechModel.toString());
}
if (this.params.languageCode !== undefined) {
console.warn("[Deprecation Warning] `languageCode` is deprecated and will be removed in a future release. Please use `languageCodes` instead.");
searchParams.set("language_code", this.params.languageCode);
}
if (this.params.languageCodes !== undefined) {
searchParams.set("language_codes", JSON.stringify(this.params.languageCodes));
}
if (this.params.languageDetection !== undefined) {
searchParams.set("language_detection", this.params.languageDetection.toString());
}
if (this.params.domain) {
searchParams.set("domain", this.params.domain);
}
if (this.params.inactivityTimeout !== undefined) {
searchParams.set("inactivity_timeout", this.params.inactivityTimeout.toString());
}
if (this.params.speakerLabels !== undefined) {
searchParams.set("speaker_labels", this.params.speakerLabels.toString());
}
if (this.params.maxSpeakers !== undefined) {
searchParams.set("max_speakers", this.params.maxSpeakers.toString());
}
if (this.params.voiceFocus) {
searchParams.set("voice_focus", this.params.voiceFocus);
}
if (this.params.voiceFocusThreshold !== undefined) {
searchParams.set("voice_focus_threshold", this.params.voiceFocusThreshold.toString());
}
if (this.params.continuousPartials !== undefined) {
searchParams.set("continuous_partials", this.params.continuousPartials.toString());
}
if (this.params.interruptionDelay !== undefined) {
searchParams.set("interruption_delay", this.params.interruptionDelay.toString());
}
if (this.params.turnLeftPadMs !== undefined) {
searchParams.set("turn_left_pad_ms", this.params.turnLeftPadMs.toString());
}
if (this.params.customerSupportAudioCapture) {
console.warn("`customerSupportAudioCapture=true` will record session audio. Only enable this when explicitly coordinating with AssemblyAI support.");
// The server's canonical wire name is `_customer_support_audio_capture`
// (leading underscore = "not officially supported / unstable"). The
// server also accepts `customer_support_audio_capture` via
// `populate_by_name=True`, but we send the underscore form to honor
// the server's stability marker.
searchParams.set("_customer_support_audio_capture", this.params.customerSupportAudioCapture.toString());
}
if (this.params.webhookUrl) {
searchParams.set("webhook_url", this.params.webhookUrl);
}
if (this.params.webhookAuthHeaderName) {
searchParams.set("webhook_auth_header_name", this.params.webhookAuthHeaderName);
}
if (this.params.webhookAuthHeaderValue) {
searchParams.set("webhook_auth_header_value", this.params.webhookAuthHeaderValue);
}
if (this.params.includePartialTurns !== undefined) {
searchParams.set("include_partial_turns", this.params.includePartialTurns.toString());
}
if (this.params.redactPii !== undefined) {
searchParams.set("redact_pii", this.params.redactPii.toString());
}
if (this.params.redactPiiPolicies !== undefined) {
searchParams.set("redact_pii_policies", JSON.stringify(this.params.redactPiiPolicies));
}
if (this.params.redactPiiSub !== undefined) {
searchParams.set("redact_pii_sub", this.params.redactPiiSub);
}
if (this.params.mode !== undefined) {
searchParams.set("mode", this.params.mode);
}
if (this.params.llmGateway !== undefined) {
searchParams.set("llm_gateway", JSON.stringify(this.params.llmGateway));
}
url.search = searchParams.toString();
return url;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
on(event, listener) {
this.listeners[event] = listener;
}
/**
* Open the streaming session.
*
* Resolves with the server's `Begin` event once the handshake completes. A
* single attempt is bounded by `connectTimeout` (default 1000ms); transient
* failures (timeout, network drop, unexpected close) are retried up to
* `maxConnectionRetries` times (default 2), waiting `connectionRetryDelay`
* (default 500ms) between attempts. Permanent failures (auth, insufficient
* funds, malformed config) are not retried.
*
* Unlike previously, a failed connection now rejects this promise rather
* than only invoking the `error` listener — necessary for the caller (and
* the retry loop) to observe the failure.
*/
connect() {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b;
if (this.socket) {
throw new Error("Already connected");
}
const maxRetries = (_a = this.params.maxConnectionRetries) !== null && _a !== void 0 ? _a : DEFAULT_MAX_CONNECTION_RETRIES;
const retryDelay = (_b = this.params.connectionRetryDelay) !== null && _b !== void 0 ? _b : DEFAULT_CONNECTION_RETRY_DELAY_MS;
let lastError;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
return yield this.connectOnce();
}
catch (err) {
lastError = err;
const retryable = err.retryable === true;
if (!retryable || attempt === maxRetries) {
throw err;
}
console.warn(`Streaming connect attempt ${attempt + 1}/${maxRetries + 1} failed (${err.message}); retrying`);
if (retryDelay > 0) {
yield new Promise((resolve) => setTimeout(resolve, retryDelay));
}
}
}
// The loop above always returns or throws; this only satisfies the type
// checker that a value is produced on every path.
throw lastError !== null && lastError !== void 0 ? lastError : new Error("Failed to connect to streaming server");
});
}
connectOnce() {
return new Promise((resolve, reject) => {
var _a;
const url = this.connectionUrl();
const timeoutMs = (_a = this.params.connectTimeout) !== null && _a !== void 0 ? _a : DEFAULT_CONNECT_TIMEOUT_MS;
// `settled` flips once this attempt has resolved (`Begin`) or rejected
// (timeout / pre-`Begin` close / error). Before it flips the socket
// handlers drive this promise; after it flips they revert to normal
// runtime dispatch (close / error / message listeners).
let settled = false;
let timer;
const failAttempt = (error) => {
if (settled)
return;
settled = true;
if (timer)
clearTimeout(timer);
this.discardPendingSocket();
reject(error);
};
const succeed = (begin) => {
if (settled)
return;
settled = true;
if (timer)
clearTimeout(timer);
resolve(begin);
};
if (timeoutMs > 0) {
timer = setTimeout(() => {
const err = new StreamingError(`Streaming connection timed out after ${timeoutMs}ms`);
err.retryable = true;
failAttempt(err);
}, timeoutMs);
}
if (this.token) {
this.socket = factory(url.toString());
}
else {
this.socket = factory(url.toString(), {
headers: { Authorization: this.apiKey },
});
}
this.socket.binaryType = "arraybuffer";
this.socket.onopen = () => { };
this.socket.onclose = ({ code, reason }) => {
var _a, _b;
if (!reason) {
if (code in StreamingErrorMessages) {
reason = StreamingErrorMessages[code];
}
}
// A close before `Begin` is a failed connection attempt — reject so
// connect() can retry (or surface a permanent failure).
if (!settled) {
const err = new StreamingError(reason || `Streaming connection closed (code=${code})`);
err.code = code;
err.retryable = isRetryableCloseCode(code);
failAttempt(err);
return;
}
// Stop the flush timer when the socket is gone (server-initiated close,
// network drop, etc.) — otherwise subsequent ticks call send() on a
// closed socket and spam the error listener.
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.flushTimer = undefined;
}
(_b = (_a = this.listeners).close) === null || _b === void 0 ? void 0 : _b.call(_a, code, reason);
};
this.socket.onerror = (event) => {
var _a, _b, _c;
const error = (_a = event.error) !== null && _a !== void 0 ? _a : new Error(event.message);
// A socket error before `Begin` is a failed attempt → reject/retry.
if (!settled) {
error.retryable = true;
failAttempt(error);
return;
}
(_c = (_b = this.listeners).error) === null || _c === void 0 ? void 0 : _c.call(_b, error);
};
this.socket.onmessage = ({ data }) => {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
const message = JSON.parse(data.toString());
if ("error" in message) {
const err = new StreamingError(message.error);
if ("error_code" in message) {
err.code = message.error_code;
}
// A server error frame before `Begin` fails the attempt; the code
// decides whether a retry is worthwhile.
if (!settled) {
const attemptErr = err;
attemptErr.retryable =
err.code === undefined ? true : isRetryableCloseCode(err.code);
failAttempt(attemptErr);
return;
}
(_b = (_a = this.listeners).error) === null || _b === void 0 ? void 0 : _b.call(_a, err);
return;
}
switch (message.type) {
case "Begin": {
succeed(message);
(_d = (_c = this.listeners).open) === null || _d === void 0 ? void 0 : _d.call(_c, message);
break;
}
case "Turn": {
if (this.isDualChannel && this.timeline && this.attributionParams) {
attributeTurn(message, this.timeline, {
dominanceRatio: this.attributionParams.dominanceRatio,
});
switch (this.attributionParams.resolveUnknownChannelsMethod) {
case "window":
this.resolveUnknownChannelsByWindow(message);
break;
case "speaker-history":
this.resolveUnknownChannelsBySpeakerHistory(message);
break;
}
}
(_f = (_e = this.listeners).turn) === null || _f === void 0 ? void 0 : _f.call(_e, message);
break;
}
case "SpeechStarted": {
(_h = (_g = this.listeners).speechStarted) === null || _h === void 0 ? void 0 : _h.call(_g, message);
break;
}
case "LLMGatewayResponse": {
(_k = (_j = this.listeners).llmGatewayResponse) === null || _k === void 0 ? void 0 : _k.call(_j, message);
break;
}
case "SpeakerRevision": {
(_m = (_l = this.listeners).speakerRevision) === null || _m === void 0 ? void 0 : _m.call(_l, message);
break;
}
case "Warning": {
const warning = message;
console.warn(`Streaming warning (code=${warning.warning_code}): ${warning.warning}`);
(_p = (_o = this.listeners).warning) === null || _p === void 0 ? void 0 : _p.call(_o, warning);
break;
}
case "Heartbeat": {
(_r = (_q = this.listeners).heartbeat) === null || _r === void 0 ? void 0 : _r.call(_q, message);
break;
}
case "Termination": {
(_s = this.sessionTerminatedResolve) === null || _s === void 0 ? void 0 : _s.call(this);
break;
}
}
};
});
}
/** Tear down a half-open socket from a failed connection attempt. */
discardPendingSocket() {
if (!this.socket)
return;
try {
if (this.socket.removeAllListeners)
this.socket.removeAllListeners();
this.socket.close();
}
catch (_a) {
// Best-effort cleanup; a half-open socket may throw on close.
}
this.socket = undefined;
}
/**
* Returns a WritableStream that pumps PCM chunks into `sendAudio`. Single-channel
* only — in dual-channel mode use `sendAudio(pcm, { channel })` directly, since
* `WritableStream` has no place to carry a channel tag.
*/
stream() {
return new WritableStream({
write: (chunk) => {
this.sendAudio(chunk);
},
});
}
/**
* Send PCM audio.
*
* In single-channel mode, `audio` is forwarded directly to the WebSocket and
* `options` is ignored.
*
* In dual-channel mode (when `channels` is configured), `options.channel` is
* REQUIRED and must match one of the declared channel names. Per-channel PCM is
* fed into that channel's VAD, accumulated into a per-channel ring buffer, and
* a scheduled flush (`channelAttribution.flushIntervalMs`, default 50ms) mixes
* the buffers into mono before sending to the WebSocket.
*/
sendAudio(audio, options) {
if (!this.isDualChannel) {
this.send(audio);
return;
}
if (!(options === null || options === void 0 ? void 0 : options.channel)) {
throw new Error("StreamingTranscriber is in dual-channel mode; sendAudio requires { channel }.");
}
if (!this.channelNames.includes(options.channel)) {
throw new Error(`Unknown channel "${options.channel}"; declared channels: ${this.channelNames.join(", ")}.`);
}
this.ingestChannelAudio(options.channel, audio);
}
ingestChannelAudio(name, audio) {
var _a, _b;
const samples = toInt16View(audio);
const buf = this.channelBuffers.get(name);
const vadBuf = this.channelVadFloatBuffers.get(name);
let vadIdx = this.channelVadBufferIdx.get(name);
let received = this.channelSamplesReceived.get(name);
const vad = this.channelVads.get(name);
const sampleRate = this.params.sampleRate;
const frameSize = this.vadFrameSamples;
for (let i = 0; i < samples.length; i++) {
const s = samples[i];
buf.push(s);
vadBuf[vadIdx++] = s / 0x8000;
received++;
if (vadIdx === frameSize) {
const result = vad.process(vadBuf);
const frame = {
ts: (received / sampleRate) * 1000,
channel: name,
active: result.active,
rms: result.energy,
};
this.timeline.pushFrame(frame);
(_b = (_a = this.listeners).vad) =