assemblyai
Version:
The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.
1,333 lines (1,312 loc) • 96.3 kB
JavaScript
import ws from 'ws';
/**
* The default speech model for synchronous transcription.
*/
const defaultSyncSpeechModel = "universal-3-5-pro";
/**
* Thrown when `DualChannelCapture` is constructed in a non-browser environment
* (no `globalThis.AudioContext`). The helper is intentionally surfaced from the
* main entrypoint so the import path is uniform across runtimes; the runtime
* guard moves to construction time.
*/
class BrowserOnlyError extends Error {
constructor(message = "DualChannelCapture requires a browser environment (AudioContext is undefined).") {
super(message);
this.name = "BrowserOnlyError";
}
}
const readFile = async (path) => Bun.file(path).stream();
const DEFAULT_FETCH_INIT = {
cache: "no-store",
};
const buildUserAgent = (userAgent) => defaultUserAgentString +
(userAgent === false
? ""
: " AssemblyAI/1.0 (" +
Object.entries({ ...defaultUserAgent, ...userAgent })
.map(([key, item]) => item ? `${key}=${item.name}/${item.version}` : "")
.join(" ") +
")");
let defaultUserAgentString = "";
if (typeof navigator !== "undefined" && navigator.userAgent) {
defaultUserAgentString += navigator.userAgent;
}
const defaultUserAgent = {
sdk: { name: "JavaScript", version: "4.36.4" },
};
if (typeof process !== "undefined") {
if (process.versions.node && defaultUserAgentString.indexOf("Node") === -1) {
defaultUserAgent.runtime_env = {
name: "Node",
version: process.versions.node,
};
}
if (process.versions.bun && defaultUserAgentString.indexOf("Bun") === -1) {
defaultUserAgent.runtime_env = {
name: "Bun",
version: process.versions.bun,
};
}
}
if (typeof Deno !== "undefined") {
if (process.versions.bun && defaultUserAgentString.indexOf("Deno") === -1) {
defaultUserAgent.runtime_env = { name: "Deno", version: Deno.version.deno };
}
}
/**
* Base class for services that communicate with the API.
*/
class BaseService {
/**
* Create a new service.
* @param params - The parameters to use for the service.
*/
constructor(params) {
this.params = params;
if (params.userAgent === false) {
this.userAgent = undefined;
}
else {
this.userAgent = buildUserAgent(params.userAgent || {});
}
}
async fetchResponse(input, init) {
init = { ...DEFAULT_FETCH_INIT, ...init };
let headers = {
Authorization: this.params.apiKey,
};
// FormData bodies must let fetch set the multipart boundary itself.
if (!(init.body instanceof FormData)) {
headers["Content-Type"] = "application/json";
}
if (DEFAULT_FETCH_INIT?.headers)
headers = { ...headers, ...DEFAULT_FETCH_INIT.headers };
if (init?.headers)
headers = { ...headers, ...init.headers };
if (this.userAgent) {
headers["User-Agent"] = this.userAgent;
}
init.headers = headers;
if (!input.startsWith("http"))
input = this.params.baseUrl + input;
return await fetch(input, init);
}
async fetch(input, init) {
const response = await this.fetchResponse(input, init);
if (response.status >= 400) {
let json;
const text = await response.text();
if (text) {
try {
json = JSON.parse(text);
}
catch {
/* empty */
}
if (json?.error)
throw new Error(json.error);
throw new Error(text);
}
throw new Error(`HTTP Error: ${response.status} ${response.statusText}`);
}
return response;
}
async fetchJson(input, init) {
const response = await this.fetch(input, init);
return response.json();
}
}
/**
* Error thrown when a synchronous transcription request fails.
*/
class SyncTranscriptError extends Error {
/**
* Create a new SyncTranscriptError.
* @param message - The human-readable error message.
* @param status - The HTTP status code of the failed request.
* @param errorCode - Machine-readable code — the snake_cased
* problem-details `title` from the server (e.g. `bad_audio`,
* `audio_too_large`, `capacity_exceeded`, `inference_timeout`).
* @param retryAfter - Seconds to wait before retrying, from the
* `Retry-After` header on 429/503 responses.
*/
constructor(message, status, errorCode, retryAfter) {
super(message);
this.status = status;
this.errorCode = errorCode;
this.retryAfter = retryAfter;
this.name = "SyncTranscriptError";
}
}
function getPath(path) {
if (path.startsWith("http"))
return null;
if (path.startsWith("https"))
return null;
if (path.startsWith("data:"))
return null;
if (path.startsWith("file://"))
return path.substring(7);
if (path.startsWith("file:"))
return path.substring(5);
return path;
}
// Canonical paths since the sync API gained a /v1 prefix (#18103); the
// unprefixed routes remain served for SDK versions that predate it.
const transcribeEndpoint = "/v1/transcribe";
const warmEndpoint = "/v1/warm";
const modelHeader = "X-AAI-Model";
// Kept above the server's 30 s deadline so the client doesn't race it.
const defaultTimeoutMs = 60_000;
const warmTimeoutMs = 10_000;
const maxPromptLength = 4096;
const maxKeytermsPromptLength = 2048;
const maxContextTurns = 100;
const maxContextLength = 4096;
// Extensions that signal raw S16LE PCM rather than a WAV container.
const pcmSuffixes = [".pcm", ".raw"];
/**
* The synchronous transcription service: audio in, transcript out,
* one request.
*
* Unlike `client.transcripts` (which submits a job to the async API and
* polls for completion), `SyncTranscriber` posts the audio to the sync
* API and returns the finished transcript in the HTTP response. There is no
* job id or status to poll. Accepts a local file path, raw audio bytes, a
* Blob, or a readable stream — but not a URL.
*/
class SyncTranscriber extends BaseService {
/**
* Create a new synchronous transcription service.
* @param params - The parameters to use for the service.
*/
constructor(params) {
super(params);
}
/**
* Transcribe audio and return the finished transcript in one request.
* @param audio - A local file path, raw audio bytes, a Blob, or a readable
* stream. Raw PCM also requires `sample_rate` and `channels` on the config.
* @param config - Options for this transcription request.
* @param options - Client-side options, such as the request timeout.
* @returns A promise that resolves to the finished transcript.
* @throws SyncTranscriptError when the request fails.
*/
async transcribe(audio, config = {}, options = {}) {
const { bytes, filename, contentType } = await resolveAudio(audio, config);
const body = new FormData();
body.append("audio", new Blob([bytes], { type: contentType }), filename);
const configJson = buildConfigJson(config);
if (configJson) {
body.append("config", new Blob([JSON.stringify(configJson)], { type: "application/json" }));
}
const response = await this.fetchResponse(transcribeEndpoint, {
method: "POST",
body,
headers: { [modelHeader]: config.model ?? defaultSyncSpeechModel },
signal: AbortSignal.timeout(options.timeout ?? defaultTimeoutMs),
});
if (response.status !== 200)
throw await errorFromResponse(response);
return (await response.json());
}
/**
* Open the connection to the sync API ahead of time.
*
* The sync API is a single request/response, so a `transcribe()` that
* opens its connection on demand pays the full DNS + TCP + TLS handshake
* on the critical path. Call `warm()` as soon as you know audio is coming —
* typically while the clip is still being recorded — so the next
* `transcribe()` reuses the already-open connection. `warm()` is idempotent
* and cheap; call it shortly before `transcribe()` so the pooled connection
* hasn't idled out.
* @param params - Optionally the model to route the probe to, so the warmed
* connection lands on the same backend as the eventual transcription.
* @returns A promise that resolves to `true` once the connection is open
* (any HTTP response — even a non-200 — means the socket is
* established), or `false` if the connection could not be opened.
*/
async warm(params) {
try {
await this.fetchResponse(warmEndpoint, {
method: "GET",
headers: { [modelHeader]: params?.model ?? defaultSyncSpeechModel },
signal: AbortSignal.timeout(warmTimeoutMs),
});
return true;
}
catch {
return false;
}
}
}
/**
* Read the audio input into bytes and decide its multipart content type.
*
* PCM is selected when the source has a `.pcm`/`.raw` extension or when
* `sample_rate`/`channels` are set on the config (the fields the sync API
* requires only for raw PCM) — and both must then be present. Everything
* else is treated as a WAV container. URLs are rejected — the sync API has
* no URL ingestion.
*/
async function resolveAudio(input, config) {
let bytes;
let filename;
let suffix = "";
if (typeof input === "string") {
if (/^https?:\/\//i.test(input)) {
throw new Error("SyncTranscriber does not accept URLs. Pass a local file path or " +
"audio bytes, or use client.transcripts for URL/async transcription.");
}
if (input.startsWith("data:")) {
bytes = dataUrlToBytes(input);
}
else {
const path = getPath(input) ?? input;
bytes = await readStream(await readFile(path));
filename = basename(path);
suffix = extname(filename);
}
}
else if (input instanceof Uint8Array) {
bytes = input;
}
else if (input instanceof ArrayBuffer) {
bytes = new Uint8Array(input);
}
else if (input instanceof Blob) {
bytes = new Uint8Array(await input.arrayBuffer());
// File instances carry a name; the File global itself needs Node >= 20.
const name = input.name;
if (name) {
filename = basename(name);
suffix = extname(filename);
}
}
else if (isWebReadableStream(input)) {
bytes = await readStream(input);
}
else if (isAsyncIterable(input)) {
bytes = await readAsyncIterable(input);
// fs.ReadStream carries the path it was opened from.
const path = input.path;
if (typeof path === "string") {
filename = basename(path);
suffix = extname(filename);
}
}
else {
throw new TypeError("unsupported audio input type");
}
const wantsPcm = config.sample_rate !== undefined || config.channels !== undefined;
const isPcm = pcmSuffixes.includes(suffix) || wantsPcm;
if (isPcm &&
(config.sample_rate === undefined || config.channels === undefined)) {
throw new Error("raw PCM audio requires both sample_rate and channels in the config");
}
const contentType = isPcm ? "audio/pcm" : "audio/wav";
if (!filename)
filename = isPcm ? "audio.pcm" : "audio.wav";
return { bytes, filename, contentType };
}
/**
* Serialize the config to the JSON `config` part, validating and normalizing
* field values to match the server's caps. The routing `model` is never
* included — it travels in the `X-AAI-Model` header. Returns `undefined`
* when there is nothing to send, so the part can be omitted entirely.
*/
function buildConfigJson(config) {
if (config.prompt !== undefined && config.prompt.length > maxPromptLength) {
throw new Error(`prompt exceeds ${maxPromptLength} characters (got ${config.prompt.length})`);
}
const json = {};
if (config.prompt !== undefined)
json["prompt"] = config.prompt;
const keytermsPrompt = normalizeKeytermsPrompt(config.keyterms_prompt);
if (keytermsPrompt)
json["keyterms_prompt"] = keytermsPrompt;
const context = normalizeConversationContext(config.conversation_context);
if (context)
json["conversation_context"] = context;
if (config.language_codes !== undefined)
json["language_codes"] = config.language_codes;
if (config.sample_rate !== undefined)
json["sample_rate"] = config.sample_rate;
if (config.channels !== undefined)
json["channels"] = config.channels;
if (config.timestamps !== undefined)
json["timestamps"] = config.timestamps;
return Object.keys(json).length > 0 ? json : undefined;
}
function normalizeKeytermsPrompt(keytermsPrompt) {
if (!keytermsPrompt)
return undefined;
const terms = keytermsPrompt
.map((term) => term.trim())
.filter((term) => term.length > 0);
const total = terms.reduce((sum, term) => sum + term.length, 0);
if (total > maxKeytermsPromptLength) {
throw new Error(`keyterms_prompt exceeds ${maxKeytermsPromptLength} characters (got ${total})`);
}
return terms.length > 0 ? terms : undefined;
}
function normalizeConversationContext(context) {
if (context === undefined)
return undefined;
let turns = (typeof context === "string" ? [context] : context)
.map((turn) => turn.trim())
.filter((turn) => turn.length > 0);
let total = turns.reduce((sum, turn) => sum + turn.length, 0);
// Over-cap context is trimmed oldest-first, never rejected.
while (turns.length > 0 &&
(turns.length > maxContextTurns || total > maxContextLength)) {
total -= turns[0].length;
turns = turns.slice(1);
}
return turns.length > 0 ? turns : undefined;
}
/**
* Build a SyncTranscriptError from a non-200 response. The primary format
* is an RFC 9457 problem-details body (`status`/`title`/`detail`); legacy
* `{error_code, message}` and `{detail}`-only bodies are also accepted.
*/
async function errorFromResponse(response) {
let errorCode;
let message;
const text = await response.text();
try {
const body = JSON.parse(text);
if (body && typeof body === "object" && !Array.isArray(body)) {
if (typeof body.error_code === "string")
errorCode = body.error_code;
if (errorCode === undefined && typeof body.title === "string") {
errorCode = body.title.toLowerCase().replace(/ /g, "_");
}
if (typeof body.detail === "string")
message = body.detail;
else if (typeof body.message === "string")
message = body.message;
}
}
catch {
if (text)
message = text;
}
if (!message) {
message = `sync transcription failed with status ${response.status}`;
}
const retryHeader = response.headers.get("retry-after");
const retryAfter = retryHeader && /^\d+$/.test(retryHeader)
? parseInt(retryHeader, 10)
: undefined;
return new SyncTranscriptError(message, response.status, errorCode, retryAfter);
}
function basename(path) {
return path.split(/[\\/]/).pop() ?? path;
}
function extname(filename) {
const dotIndex = filename.lastIndexOf(".");
return dotIndex > 0 ? filename.slice(dotIndex).toLowerCase() : "";
}
function dataUrlToBytes(dataUrl) {
const base64 = dataUrl.split(",")[1];
const binary = atob(base64);
const bytes = new Uint8Array(binary.length);
for (let i = 0; i < binary.length; i++)
bytes[i] = binary.charCodeAt(i);
return bytes;
}
function isWebReadableStream(input) {
return typeof input?.getReader === "function";
}
function isAsyncIterable(input) {
return (typeof input?.[Symbol.asyncIterator] ===
"function");
}
async function readStream(stream) {
const chunks = [];
const reader = stream.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done)
break;
chunks.push(value);
}
return concatChunks(chunks);
}
async function readAsyncIterable(iterable) {
const chunks = [];
for await (const chunk of iterable)
chunks.push(chunk);
return concatChunks(chunks);
}
function concatChunks(chunks) {
const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const bytes = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) {
bytes.set(chunk, offset);
offset += chunk.length;
}
return bytes;
}
const RealtimeErrorType = {
BadSampleRate: 4000,
AuthFailed: 4001,
/**
* @deprecated Use InsufficientFunds or FreeTierUser instead
*/
InsufficientFundsOrFreeAccount: 4002,
InsufficientFunds: 4002,
FreeTierUser: 4003,
NonexistentSessionId: 4004,
SessionExpired: 4008,
ClosedSession: 4010,
RateLimited: 4029,
UniqueSessionViolation: 4030,
SessionTimeout: 4031,
AudioTooShort: 4032,
AudioTooLong: 4033,
AudioTooSmallToTranscode: 4034,
/**
* @deprecated Don't use
*/
BadJson: 4100,
BadSchema: 4101,
TooManyStreams: 4102,
Reconnected: 4103,
/**
* @deprecated Don't use
*/
ReconnectAttemptsExhausted: 1013,
WordBoostParameterParsingFailed: 4104,
};
const RealtimeErrorMessages = {
[RealtimeErrorType.BadSampleRate]: "Sample rate must be a positive integer",
[RealtimeErrorType.AuthFailed]: "Not Authorized",
[RealtimeErrorType.InsufficientFunds]: "Insufficient funds",
[RealtimeErrorType.FreeTierUser]: "This feature is paid-only and requires you to add a credit card. Please visit https://app.assemblyai.com/ to add a credit card to your account.",
[RealtimeErrorType.NonexistentSessionId]: "Session ID does not exist",
[RealtimeErrorType.SessionExpired]: "Session has expired",
[RealtimeErrorType.ClosedSession]: "Session is closed",
[RealtimeErrorType.RateLimited]: "Rate limited",
[RealtimeErrorType.UniqueSessionViolation]: "Unique session violation",
[RealtimeErrorType.SessionTimeout]: "Session Timeout",
[RealtimeErrorType.AudioTooShort]: "Audio too short",
[RealtimeErrorType.AudioTooLong]: "Audio too long",
[RealtimeErrorType.AudioTooSmallToTranscode]: "Audio too small to transcode",
[RealtimeErrorType.BadJson]: "Bad JSON",
[RealtimeErrorType.BadSchema]: "Bad schema",
[RealtimeErrorType.TooManyStreams]: "Too many streams",
[RealtimeErrorType.Reconnected]: "This session has been reconnected. This WebSocket is no longer valid.",
[RealtimeErrorType.ReconnectAttemptsExhausted]: "Reconnect attempts exhausted",
[RealtimeErrorType.WordBoostParameterParsingFailed]: "Could not parse word boost parameter",
};
class RealtimeError extends Error {
}
const StreamingErrorType = {
BadSampleRate: 4000,
AuthFailed: 4001,
InsufficientFunds: 4002,
FreeTierUser: 4003,
NonexistentSessionId: 4004,
SessionExpired: 4008,
ClosedSession: 4010,
RateLimited: 4029,
UniqueSessionViolation: 4030,
SessionTimeout: 4031,
AudioTooShort: 4032,
AudioTooLong: 4033,
AudioTooSmallToTranscode: 4034,
BadSchema: 4101,
TooManyStreams: 4102,
Reconnected: 4103,
ServerError: 3005,
InputValidationError: 3006,
AudioChunkDurationViolation: 3007,
MaxSessionDurationExceeded: 3008,
ConcurrencyLimitExceeded: 3009,
};
const StreamingErrorMessages = {
[StreamingErrorType.ServerError]: "Server error",
[StreamingErrorType.InputValidationError]: "Input validation error",
[StreamingErrorType.AudioChunkDurationViolation]: "Audio chunk duration violation",
[StreamingErrorType.MaxSessionDurationExceeded]: "Session expired: maximum session duration exceeded",
[StreamingErrorType.ConcurrencyLimitExceeded]: "Too many concurrent sessions",
[StreamingErrorType.BadSampleRate]: "Sample rate must be a positive integer",
[StreamingErrorType.AuthFailed]: "Not Authorized",
[StreamingErrorType.InsufficientFunds]: "Insufficient funds",
[StreamingErrorType.FreeTierUser]: "This feature is paid-only and requires you to add a credit card. Please visit https://app.assemblyai.com/ to add a credit card to your account.",
[StreamingErrorType.NonexistentSessionId]: "Session ID does not exist",
[StreamingErrorType.SessionExpired]: "Session has expired",
[StreamingErrorType.ClosedSession]: "Session is closed",
[StreamingErrorType.RateLimited]: "Rate limited",
[StreamingErrorType.UniqueSessionViolation]: "Unique session violation",
[StreamingErrorType.SessionTimeout]: "Session Timeout",
[StreamingErrorType.AudioTooShort]: "Audio too short",
[StreamingErrorType.AudioTooLong]: "Audio too long",
[StreamingErrorType.AudioTooSmallToTranscode]: "Audio too small to transcode",
[StreamingErrorType.BadSchema]: "Bad schema",
[StreamingErrorType.TooManyStreams]: "Too many streams",
[StreamingErrorType.Reconnected]: "This session has been reconnected. This WebSocket is no longer valid.",
};
class StreamingError extends Error {
}
class LemurService extends BaseService {
summary(params, signal) {
return this.fetchJson("/lemur/v3/generate/summary", {
method: "POST",
body: JSON.stringify(params),
signal,
});
}
questionAnswer(params, signal) {
return this.fetchJson("/lemur/v3/generate/question-answer", {
method: "POST",
body: JSON.stringify(params),
signal,
});
}
actionItems(params, signal) {
return this.fetchJson("/lemur/v3/generate/action-items", {
method: "POST",
body: JSON.stringify(params),
signal,
});
}
task(params, signal) {
return this.fetchJson("/lemur/v3/generate/task", {
method: "POST",
body: JSON.stringify(params),
signal,
});
}
getResponse(id, signal) {
return this.fetchJson(`/lemur/v3/${id}`, { signal });
}
/**
* Delete the data for a previously submitted LeMUR request.
* @param id - ID of the LeMUR request
* @param signal - Optional AbortSignal to cancel the request
*/
purgeRequestData(id, signal) {
return this.fetchJson(`/lemur/v3/${id}`, {
method: "DELETE",
signal,
});
}
}
const { WritableStream } = typeof window !== "undefined"
? window
: typeof global !== "undefined"
? global
: globalThis;
const factory = (url, params) => new ws(url, params);
const defaultRealtimeUrl = "wss://api.assemblyai.com/v2/realtime/ws";
const forceEndOfUtteranceMessage = `{"force_end_utterance":true}`;
const terminateSessionMessage$1 = `{"terminate_session":true}`;
/**
* RealtimeTranscriber connects to the Streaming Speech-to-Text API and lets you transcribe audio in real-time.
*/
class RealtimeTranscriber {
/**
* Create a new RealtimeTranscriber.
* @param params - Parameters to configure the RealtimeTranscriber
*/
constructor(params) {
this.listeners = {};
this.realtimeUrl = params.realtimeUrl ?? defaultRealtimeUrl;
this.sampleRate = params.sampleRate ?? 16_000;
this.wordBoost = params.wordBoost;
this.encoding = params.encoding;
this.endUtteranceSilenceThreshold = params.endUtteranceSilenceThreshold;
this.disablePartialTranscripts = params.disablePartialTranscripts;
if ("token" in params && params.token)
this.token = params.token;
if ("apiKey" in params && params.apiKey)
this.apiKey = params.apiKey;
if (!(this.token || this.apiKey)) {
throw new Error("API key or temporary token is required.");
}
}
connectionUrl() {
const url = new URL(this.realtimeUrl);
if (url.protocol !== "wss:") {
throw new Error("Invalid protocol, must be wss");
}
const searchParams = new URLSearchParams();
if (this.token) {
searchParams.set("token", this.token);
}
searchParams.set("sample_rate", this.sampleRate.toString());
if (this.wordBoost && this.wordBoost.length > 0) {
searchParams.set("word_boost", JSON.stringify(this.wordBoost));
}
if (this.encoding) {
searchParams.set("encoding", this.encoding);
}
searchParams.set("enable_extra_session_information", "true");
if (this.disablePartialTranscripts) {
searchParams.set("disable_partial_transcripts", this.disablePartialTranscripts.toString());
}
url.search = searchParams.toString();
return url;
}
/**
* Add a listener for an event.
* @param event - The event to listen for.
* @param listener - The function to call when the event is emitted.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
on(event, listener) {
this.listeners[event] = listener;
}
/**
* Connect to the server and begin a new session.
* @returns A promise that resolves when the connection is established and the session begins.
*/
connect() {
return new Promise((resolve) => {
if (this.socket) {
throw new Error("Already connected");
}
const url = this.connectionUrl();
if (this.token) {
this.socket = factory(url.toString());
}
else {
this.socket = factory(url.toString(), {
headers: { Authorization: this.apiKey },
});
}
this.socket.binaryType = "arraybuffer";
this.socket.onopen = () => {
if (this.endUtteranceSilenceThreshold === undefined ||
this.endUtteranceSilenceThreshold === null) {
return;
}
this.configureEndUtteranceSilenceThreshold(this.endUtteranceSilenceThreshold);
};
this.socket.onclose = ({ code, reason }) => {
if (!reason) {
if (code in RealtimeErrorMessages) {
reason = RealtimeErrorMessages[code];
}
}
this.listeners.close?.(code, reason);
};
this.socket.onerror = (event) => {
if (event.error)
this.listeners.error?.(event.error);
else
this.listeners.error?.(new Error(event.message));
};
this.socket.onmessage = ({ data }) => {
const message = JSON.parse(data.toString());
if ("error" in message) {
this.listeners.error?.(new RealtimeError(message.error));
return;
}
switch (message.message_type) {
case "SessionBegins": {
const openObject = {
sessionId: message.session_id,
expiresAt: new Date(message.expires_at),
};
resolve(openObject);
this.listeners.open?.(openObject);
break;
}
case "PartialTranscript": {
// message.created is actually a string when coming from the socket
message.created = new Date(message.created);
this.listeners.transcript?.(message);
this.listeners["transcript.partial"]?.(message);
break;
}
case "FinalTranscript": {
// message.created is actually a string when coming from the socket
message.created = new Date(message.created);
this.listeners.transcript?.(message);
this.listeners["transcript.final"]?.(message);
break;
}
case "SessionInformation": {
this.listeners.session_information?.(message);
break;
}
case "SessionTerminated": {
this.sessionTerminatedResolve?.();
break;
}
}
};
});
}
/**
* Send audio data to the server.
* @param audio - The audio data to send to the server.
*/
sendAudio(audio) {
this.send(audio);
}
/**
* Create a writable stream that can be used to send audio data to the server.
* @returns A writable stream that can be used to send audio data to the server.
*/
stream() {
return new WritableStream({
write: (chunk) => {
this.sendAudio(chunk);
},
});
}
/**
* Manually end an utterance
*/
forceEndUtterance() {
this.send(forceEndOfUtteranceMessage);
}
/**
* Configure the threshold for how long to wait before ending an utterance. Default is 700ms.
* @param threshold - The duration of the end utterance silence threshold in milliseconds.
* This value must be an integer between 0 and 20_000.
*/
configureEndUtteranceSilenceThreshold(threshold) {
this.send(`{"end_utterance_silence_threshold":${threshold}}`);
}
send(data) {
if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
throw new Error("Socket is not open for communication");
}
this.socket.send(data);
}
/**
* Close the connection to the server.
* @param waitForSessionTermination - If true, the method will wait for the session to be terminated before closing the connection.
* While waiting for the session to be terminated, you will receive the final transcript and session information.
*/
async close(waitForSessionTermination = true) {
if (this.socket) {
if (this.socket.readyState === this.socket.OPEN) {
if (waitForSessionTermination) {
const sessionTerminatedPromise = new Promise((resolve) => {
this.sessionTerminatedResolve = resolve;
});
this.socket.send(terminateSessionMessage$1);
await sessionTerminatedPromise;
}
else {
this.socket.send(terminateSessionMessage$1);
}
}
if (this.socket?.removeAllListeners)
this.socket.removeAllListeners();
this.socket.close();
}
this.listeners = {};
this.socket = undefined;
}
}
/**
* @deprecated Use RealtimeTranscriber instead
*/
class RealtimeService extends RealtimeTranscriber {
}
class RealtimeTranscriberFactory extends BaseService {
constructor(params) {
super(params);
this.rtFactoryParams = params;
}
/**
* @deprecated Use transcriber(...) instead
*/
createService(params) {
return this.transcriber(params);
}
transcriber(params) {
const serviceParams = { ...params };
if (!serviceParams.token && !serviceParams.apiKey) {
serviceParams.apiKey = this.rtFactoryParams.apiKey;
}
return new RealtimeTranscriber(serviceParams);
}
async createTemporaryToken(params) {
const data = await this.fetchJson("/v2/realtime/token", {
method: "POST",
body: JSON.stringify(params),
});
return data.token;
}
}
/**
* @deprecated Use RealtimeTranscriberFactory instead
*/
class RealtimeServiceFactory extends RealtimeTranscriberFactory {
}
class TranscriptService extends BaseService {
constructor(params, files) {
super(params);
this.files = files;
}
/**
* Transcribe an audio file. This will create a transcript and wait until the transcript status is "completed" or "error".
* @param params - The parameters to transcribe an audio file.
* @param options - The options to transcribe an audio file.
* @returns A promise that resolves to the transcript. The transcript status is "completed" or "error".
*/
async transcribe(params, options) {
const transcript = await this.submit(params);
return await this.waitUntilReady(transcript.id, options);
}
/**
* Submits a transcription job for an audio file. This will not wait until the transcript status is "completed" or "error".
* @param params - The parameters to start the transcription of an audio file.
* @returns A promise that resolves to the queued transcript.
*/
async submit(params) {
let audioUrl;
let transcriptParams = undefined;
if ("audio" in params) {
const { audio, ...audioTranscriptParams } = params;
if (typeof audio === "string") {
const path = getPath(audio);
if (path !== null) {
// audio is local path, upload local file
audioUrl = await this.files.upload(path);
}
else {
if (audio.startsWith("data:")) {
audioUrl = await this.files.upload(audio);
}
else {
// audio is not a local path, and not a data-URI, assume it's a normal URL
audioUrl = audio;
}
}
}
else {
// audio is of uploadable type
audioUrl = await this.files.upload(audio);
}
transcriptParams = { ...audioTranscriptParams, audio_url: audioUrl };
}
else {
transcriptParams = params;
}
const data = await this.fetchJson("/v2/transcript", {
method: "POST",
body: JSON.stringify(transcriptParams),
});
return data;
}
/**
* Create a transcript.
* @param params - The parameters to create a transcript.
* @param options - The options used for creating the new transcript.
* @returns A promise that resolves to the transcript.
* @deprecated Use `transcribe` instead to transcribe a audio file that includes polling, or `submit` to transcribe a audio file without polling.
*/
async create(params, options) {
const path = getPath(params.audio_url);
if (path !== null) {
const uploadUrl = await this.files.upload(path);
params.audio_url = uploadUrl;
}
const data = await this.fetchJson("/v2/transcript", {
method: "POST",
body: JSON.stringify(params),
});
if (options?.poll ?? true) {
return await this.waitUntilReady(data.id, options);
}
return data;
}
/**
* Wait until the transcript ready, either the status is "completed" or "error".
* @param transcriptId - The ID of the transcript.
* @param options - The options to wait until the transcript is ready.
* @returns A promise that resolves to the transcript. The transcript status is "completed" or "error".
*/
async waitUntilReady(transcriptId, options) {
const pollingInterval = options?.pollingInterval ?? 3_000;
const pollingTimeout = options?.pollingTimeout ?? -1;
const startTime = Date.now();
// eslint-disable-next-line no-constant-condition
while (true) {
const transcript = await this.get(transcriptId);
if (transcript.status === "completed" || transcript.status === "error") {
return transcript;
}
else if (pollingTimeout > 0 &&
Date.now() - startTime > pollingTimeout) {
throw new Error("Polling timeout");
}
else {
await new Promise((resolve) => setTimeout(resolve, pollingInterval));
}
}
}
/**
* Retrieve a transcript.
* @param id - The identifier of the transcript.
* @returns A promise that resolves to the transcript.
*/
get(id) {
return this.fetchJson(`/v2/transcript/${id}`);
}
/**
* Retrieves a page of transcript listings.
* @param params - The parameters to filter the transcript list by, or the URL to retrieve the transcript list from.
*/
async list(params) {
let url = "/v2/transcript";
if (typeof params === "string") {
url = params;
}
else if (params) {
url = `${url}?${new URLSearchParams(Object.keys(params).map((key) => [
key,
params[key]?.toString() || "",
]))}`;
}
const data = await this.fetchJson(url);
for (const transcriptListItem of data.transcripts) {
transcriptListItem.created = new Date(transcriptListItem.created);
if (transcriptListItem.completed) {
transcriptListItem.completed = new Date(transcriptListItem.completed);
}
}
return data;
}
/**
* Delete a transcript
* @param id - The identifier of the transcript.
* @returns A promise that resolves to the transcript.
*/
delete(id) {
return this.fetchJson(`/v2/transcript/${id}`, { method: "DELETE" });
}
/**
* Search through the transcript for a specific set of keywords.
* You can search for individual words, numbers, or phrases containing up to five words or numbers.
* @param id - The identifier of the transcript.
* @param words - Keywords to search for.
* @returns A promise that resolves to the sentences.
*/
wordSearch(id, words) {
const params = new URLSearchParams({ words: words.join(",") });
return this.fetchJson(`/v2/transcript/${id}/word-search?${params.toString()}`);
}
/**
* Retrieve all sentences of a transcript.
* @param id - The identifier of the transcript.
* @returns A promise that resolves to the sentences.
*/
sentences(id) {
return this.fetchJson(`/v2/transcript/${id}/sentences`);
}
/**
* Retrieve all paragraphs of a transcript.
* @param id - The identifier of the transcript.
* @returns A promise that resolves to the paragraphs.
*/
paragraphs(id) {
return this.fetchJson(`/v2/transcript/${id}/paragraphs`);
}
/**
* Retrieve subtitles of a transcript.
* @param id - The identifier of the transcript.
* @param format - The format of the subtitles.
* @param chars_per_caption - The maximum number of characters per caption.
* @returns A promise that resolves to the subtitles text.
*/
async subtitles(id, format = "srt", chars_per_caption) {
let url = `/v2/transcript/${id}/${format}`;
if (chars_per_caption) {
const params = new URLSearchParams();
params.set("chars_per_caption", chars_per_caption.toString());
url += `?${params.toString()}`;
}
const response = await this.fetch(url);
return await response.text();
}
/**
* Retrieve the redacted audio URL of a transcript.
* @param id - The identifier of the transcript.
* @returns A promise that resolves to the details of the redacted audio.
* @deprecated Use `redactedAudio` instead.
*/
redactions(id) {
return this.redactedAudio(id);
}
/**
* Retrieve the redacted audio URL of a transcript.
* @param id - The identifier of the transcript.
* @returns A promise that resolves to the details of the redacted audio.
*/
redactedAudio(id) {
return this.fetchJson(`/v2/transcript/${id}/redacted-audio`);
}
/**
* Retrieve the redacted audio file of a transcript.
* @param id - The identifier of the transcript.
* @returns A promise that resolves to the fetch HTTP response of the redacted audio file.
*/
async redactedAudioFile(id) {
const { redacted_audio_url, status } = await this.redactedAudio(id);
if (status !== "redacted_audio_ready") {
throw new Error(`Redacted audio status is ${status}`);
}
const response = await fetch(redacted_audio_url);
if (!response.ok) {
throw new Error(`Failed to fetch redacted audio: ${response.statusText}`);
}
return {
arrayBuffer: response.arrayBuffer.bind(response),
blob: response.blob.bind(response),
body: response.body,
bodyUsed: response.bodyUsed,
};
}
}
class FileService extends BaseService {
/**
* Upload a local file to AssemblyAI.
* @param input - The local file path to upload, or a stream or buffer of the file to upload.
* @returns A promise that resolves to the uploaded file URL.
*/
async upload(input) {
let fileData;
if (typeof input === "string") {
if (input.startsWith("data:")) {
fileData = dataUrlToBlob(input);
}
else {
fileData = await readFile(input);
}
}
else
fileData = input;
const data = await this.fetchJson("/v2/upload", {
method: "POST",
body: fileData,
headers: {
"Content-Type": "application/octet-stream",
},
duplex: "half",
});
return data.upload_url;
}
}
function dataUrlToBlob(dataUrl) {
const arr = dataUrl.split(",");
const mime = arr[0].match(/:(.*?);/)[1];
const bstr = atob(arr[1]);
let n = bstr.length;
const u8arr = new Uint8Array(n);
while (n--) {
u8arr[n] = bstr.charCodeAt(n);
}
return new Blob([u8arr], { type: mime });
}
/**
* Energy-based VAD with adaptive noise-floor tracking and hangover. Pure JS,
* no dependencies. Suitable for the "which physical channel is speaking" task
* because the channels are already physically separated at capture — the harder
* problem (speech vs. non-speech in the wild) is one a customer can swap in a
* DNN VAD for via the `createVad` parameter.
*
* Tuning notes:
* - thresholdRatio below 2 will treat anything above noise as speech (too sensitive).
* - thresholdRatio above 6 will miss quiet utterance onsets/offsets.
* - noiseFloorAlpha above 0.1 makes the floor track quickly (good for non-stationary
* background) but risks slowly adapting *up* to a sustained low voice.
*/
class EnergyVad {
constructor(params = {}) {
this.hangoverRemaining = 0;
this.thresholdRatio = params.thresholdRatio ?? 3.0;
this.noiseFloorAlpha = params.noiseFloorAlpha ?? 0.05;
this.hangoverFrames = params.hangoverFrames ?? 10;
this.initialNoiseFloor = params.initialNoiseFloor ?? 1e-4;
this.noiseFloor = this.initialNoiseFloor;
}
process(frame) {
let sumSq = 0;
for (let i = 0; i < frame.length; i++) {
sumSq += frame[i] * frame[i];
}
const rms = frame.length > 0 ? Math.sqrt(sumSq / frame.length) : 0;
const threshold = this.noiseFloor * this.thresholdRatio;
let active = rms > threshold;
if (active) {
this.hangoverRemaining = this.hangoverFrames;
}
else if (this.hangoverRemaining > 0) {
this.hangoverRemaining--;
active = true;
// While in hangover, do not update noise floor — RMS may still reflect tail energy.
}
else {
this.noiseFloor =
this.noiseFloor * (1 - this.noiseFloorAlpha) +
rms * this.noiseFloorAlpha;
}
return { active, energy: rms };
}
reset() {
this.noiseFloor = this.initialNoiseFloor;
this.hangoverRemaining = 0;
}
}
/**
* Append-only ring buffer of VAD frames in stream-relative ms order.
* `pushFrame` is O(1) amortized; `framesInWindow` is O(n) over kept frames,
* which is fine for the per-word lookups we do (a 30 s window at 50 frames/s
* per channel × 2 channels = 3000 entries, scanned once per word).
*
* Runtime-agnostic — no DOM or Web Audio dependencies.
*/
class VadTimeline {
constructor(windowMs) {
this.windowMs = windowMs;
this.frames = [];
this.head = 0;
}
pushFrame(frame) {
this.frames.push(frame);
const cutoff = frame.ts - this.windowMs;
while (this.head < this.frames.length &&
this.frames[this.head].ts < cutoff) {
this.head++;
}
if (this.head > 1024 && this.head * 2 > this.frames.length) {
this.frames = this.frames.slice(this.head);
this.head = 0;
}
}
framesInWindow(startMs, endMs) {
const out = [];
for (let i = this.head; i < this.frames.length; i++) {
const f = this.frames[i];
if (f.ts < startMs)
continue;
if (f.ts > endMs)
break;
out.push(f);
}
return out;
}
clear() {
this.frames = [];
this.head = 0;
}
}
/**
* Sum per-channel active RMS over a window. Returns a Map from channel name
* to total score. Channels with zero score are omitted.
*/
function scoreChannels(frames) {
const scores = new Map();
for (const f of frames) {
if (!f.active)
continue;
scores.set(f.channel, (scores.get(f.channel) ?? 0) + f.rms);
}
return scores;
}
/**
* Decide which channel was dominant during a word's `[start, end]` window.
*
* - If no channel has any active VAD energy → `"unknown"`.
* - If the top channel beats the runner-up by at least `dominanceRatio` → top channel.
* - Else: top channel wins on absolute score; exact ties → `"unknown"`.
*/
function attributeWord(word, timeline, params) {
const scores = scoreChannels(timeline.framesInWindow(word.start, word.end));
if (scores.size === 0)
return "unknown";
const sorted = [...scores.entries()].sort((a, b) => b[1] - a[1]);
if (sorted.length === 1)
return sorted[0][0];
const [topName, topScore] = sorted[0];
const [runnerName, runnerScore] = sorted[1];
if (topScore >= params.dominanceRatio * runnerScore)
return topName;
if (topScore > runnerScore)
return topName;
if (runnerScore > topScore)
return runnerName;
return "unknown";
}
/**
* Duration-weighted majority of word channels. `"unknown"` if there are no
* words, every word resolved to `"unknown"`, or two channels tie exactly.
*/
function rollUpTurnChannel(words) {
const totals = new Map();
for (const w of words) {
if (!w.channel || w.channel === "unknown")
continue;
const dur = Math.max(0, w.end - w.start);
totals.set(w.channel, (totals.get(w.channel) ?? 0) + dur);
}
if (totals.size === 0)
return "unknown";
const sorted = [...totals.entries()].sort((a, b) => b[1] - a[1]);
if (sorted.length === 1)
return sorted[0][0];
const [topName, topMs] = sorted[0];
const [, runnerMs] = sorted[1];
if (topMs === runnerMs)
return "unknown";
return topName;
}
/**
* Mutate `turn` in place: write `turn.words[i].channel` for every word and set
* `turn.channel` to the duration-weighted rollup.
*
* Returns `void` because the transcriber owns the `TurnEvent` ref and forwards
* the same object to the customer listener — no need to allocate a copy.
*/
function attributeTurn(turn, timeline, params) {
for (const w of turn.words) {
w.channel = attributeWord(w, timeline, params);
}
turn.channel = rollUpTurnChannel(turn.words);
}
/**
* View any `AudioData` (ArrayBuffer / ArrayBufferView / typed array) as a
* little-endian Int16 sample sequence without copying. Callers must guarantee
* the underlying byte length is even.
*/
function toInt16View(audio) {
// AudioData is ArrayBufferLike per the public type, but in practice callers
// pass ArrayBuffer or a typed-array view. Handle both without copying.
if (audio instanceof Int16Array)
return audio;
if (ArrayBuffer.isView(audio)) {
const view = audio;
return new Int16Array(view.buffer, view.byteOffset, Math.floor(view.byteLength / 2));
}
return new Int16Array(audio);
}
const defaultStreamingUrl$1 = "wss://streaming.assemblyai.com/v3/ws";
const terminateSessionMessage = `{"type":"Terminate"}`;
const DEFAULT_CONNECT_TIMEOUT_MS = 1000;
const DEFAULT_MAX_CONNECTION_RETRIES = 2;
const DEFAULT_CONNECTION_RETRY_DELAY_MS = 500;
/**
* Close/error codes that signal a permanent client-side problem (auth,
* billing, malformed config). A retry would hit the same failure, so the
* connection is never retried on these.
*/
const NON_RETRYABLE_CLOSE_CODES = new Set([
StreamingErrorType.BadSampleRate,
StreamingErrorType.AuthFailed,
StreamingErrorType.InsufficientFunds,
StreamingErrorType.FreeTierUser,
StreamingErrorType.BadSchema,
]);
function isRetryableCloseCode(co