openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
1,270 lines (1,269 loc) • 48.2 kB
JavaScript
import { C as resolveExpiresAtMsFromDurationMs, m as isFutureDateTimestampMs } from "./number-coercion-CJQ8TR--.js";
import { i as formatErrorMessage } from "./errors-BXgSefBE.js";
import { t as createSubsystemLogger } from "./subsystem-BzXSmsuh.js";
import "./error-runtime-C8vbtAJt.js";
import "./number-runtime-DBLVDypr.js";
import "./runtime-env-D_2q8-VK.js";
import { A as buildRealtimeVoiceAgentConsultWorkingResponse, B as readRealtimeVoiceConsultQuestion, H as createTalkSessionController, L as createRealtimeVoiceForcedConsultCoordinator, T as REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, V as readSpeakableRealtimeVoiceToolResult, W as recordTalkObservabilityEvent, s as createRealtimeVoiceBridgeSession } from "./session-log-runtime-GkK49CJb.js";
import "./realtime-voice-BRCFRGg7.js";
import { randomUUID } from "node:crypto";
import WebSocket, { WebSocketServer } from "ws";
import "node:http";
//#region extensions/voice-call/src/webhook/realtime-audio-pacer.ts
const TELEPHONY_SAMPLE_RATE = 8e3;
const TELEPHONY_CHUNK_BYTES = 160;
const TELEPHONY_CHUNK_MS = 20;
const DEFAULT_SPEECH_RMS_THRESHOLD = .035;
const DEFAULT_REQUIRED_LOUD_CHUNKS = 4;
const DEFAULT_REQUIRED_QUIET_CHUNKS = 12;
const DEFAULT_MAX_QUEUED_AUDIO_BYTES = TELEPHONY_SAMPLE_RATE * 120;
const PCM16_MAX_AMPLITUDE = 32768;
const MULAW_LINEAR_SAMPLES = new Int16Array(256);
for (let i = 0; i < MULAW_LINEAR_SAMPLES.length; i += 1) MULAW_LINEAR_SAMPLES[i] = decodeMulawSample(i);
/** Paces outgoing mulaw audio frames at telephony cadence. */
var RealtimeAudioPacer = class {
constructor(params) {
this.params = params;
this.queue = [];
this.timer = null;
this.queuedAudioBytes = 0;
this.closed = false;
}
/** Queue mulaw audio and split it into 20ms-ish telephony chunks. */
sendAudio(muLaw) {
if (this.closed || muLaw.length === 0) return;
const maxQueuedAudioBytes = this.params.maxQueuedAudioBytes ?? DEFAULT_MAX_QUEUED_AUDIO_BYTES;
for (let offset = 0; offset < muLaw.length; offset += TELEPHONY_CHUNK_BYTES) {
const chunk = Buffer.from(muLaw.subarray(offset, offset + TELEPHONY_CHUNK_BYTES));
if (this.queuedAudioBytes + chunk.length > maxQueuedAudioBytes) {
this.failBackpressure();
return;
}
this.queue.push({
type: "audio",
chunk,
durationMs: Math.max(1, Math.round(chunk.length / TELEPHONY_SAMPLE_RATE * 1e3))
});
this.queuedAudioBytes += chunk.length;
}
this.ensurePump();
}
/** Queue a provider mark frame after prior audio frames. */
sendMark(name) {
if (this.closed || !name) return;
this.queue.push({
type: "mark",
name
});
this.ensurePump();
}
/** Clear queued audio and notify the provider stream. */
clearAudio() {
if (this.closed) return 0;
const clearedAudioBytes = this.queuedAudioBytes;
this.clearTimer();
this.queue = [];
this.queuedAudioBytes = 0;
this.params.send(this.params.serializer.clear());
return clearedAudioBytes;
}
/** Stop sending and discard queued frames. */
close() {
this.closed = true;
this.clearTimer();
this.queue = [];
this.queuedAudioBytes = 0;
}
/** Clear the scheduled pump timer. */
clearTimer() {
if (!this.timer) return;
clearTimeout(this.timer);
this.timer = null;
}
/** Start the pump when queued work exists and no timer is active. */
ensurePump() {
if (!this.timer) this.pump();
}
/** Close the pacer and notify the caller about queued-audio backpressure. */
failBackpressure() {
this.close();
this.params.onBackpressure?.();
}
/** Send one queued item and schedule the next send based on audio duration. */
pump() {
this.timer = null;
if (this.closed) return;
const item = this.queue.shift();
if (!item) return;
let delayMs = 0;
let sent;
if (item.type === "audio") {
this.queuedAudioBytes = Math.max(0, this.queuedAudioBytes - item.chunk.length);
sent = this.params.send(this.params.serializer.media(item.chunk.toString("base64")));
delayMs = item.durationMs || TELEPHONY_CHUNK_MS;
} else sent = this.params.send(this.params.serializer.mark(item.name));
if (!sent) {
this.queue = [];
this.queuedAudioBytes = 0;
return;
}
if (this.queue.length > 0) this.timer = setTimeout(() => this.pump(), delayMs);
}
};
/** Calculate normalized RMS from mulaw bytes. */
function calculateMulawRms(muLaw) {
if (muLaw.length === 0) return 0;
let sum = 0;
for (const sample of muLaw) {
const normalized = (MULAW_LINEAR_SAMPLES[sample] ?? 0) / PCM16_MAX_AMPLITUDE;
sum += normalized * normalized;
}
return Math.sqrt(sum / muLaw.length);
}
/** Detect likely speech start from consecutive loud mulaw chunks. */
var RealtimeMulawSpeechStartDetector = class {
constructor(params = {}) {
this.params = params;
this.loudChunks = 0;
this.quietChunks = DEFAULT_REQUIRED_QUIET_CHUNKS;
this.speaking = false;
}
/** Accept one mulaw chunk and return true only on transition into speaking. */
accept(muLaw) {
if (calculateMulawRms(muLaw) >= (this.params.rmsThreshold ?? DEFAULT_SPEECH_RMS_THRESHOLD)) {
this.quietChunks = 0;
this.loudChunks += 1;
const requiredLoudChunks = this.params.requiredLoudChunks ?? DEFAULT_REQUIRED_LOUD_CHUNKS;
if (!this.speaking && this.loudChunks >= requiredLoudChunks) {
this.speaking = true;
return true;
}
return false;
}
this.loudChunks = 0;
this.quietChunks += 1;
const requiredQuietChunks = this.params.requiredQuietChunks ?? DEFAULT_REQUIRED_QUIET_CHUNKS;
if (this.quietChunks >= requiredQuietChunks) this.speaking = false;
return false;
}
};
/** Decode one G.711 mulaw byte to a linear PCM sample. */
function decodeMulawSample(value) {
const muLaw = ~value & 255;
const sign = muLaw & 128;
const exponent = muLaw >> 4 & 7;
let sample = ((muLaw & 15) << 3) + 132 << exponent;
sample -= 132;
return sign ? -sample : sample;
}
//#endregion
//#region extensions/voice-call/src/webhook/stream-frame-adapter.ts
/** Parse numeric timestamps sent as numbers or integer strings. */
function parseTimestampMs(value) {
if (typeof value === "number" && Number.isFinite(value)) return value;
if (typeof value === "string" && /^[+-]?\d+$/.test(value.trim())) {
const parsed = Number(value.trim());
return Number.isSafeInteger(parsed) ? parsed : void 0;
}
}
/** Parse a JSON object frame, returning null for invalid or non-object payloads. */
function tryParseJson(rawMessage) {
try {
const parsed = JSON.parse(rawMessage);
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed;
} catch {}
return null;
}
/** Read an object-valued field from a parsed frame. */
function readRecordField(record, field) {
const value = record[field];
return typeof value === "object" && value !== null ? value : void 0;
}
/** Normalize base64/base64url padding differences for validation. */
function normalizeBase64ForCompare(value) {
return value.replace(/=+$/u, "").replace(/-/gu, "+").replace(/_/gu, "/");
}
/** Return true when a payload round-trips as base64. */
function isValidBase64Payload(value) {
return normalizeBase64ForCompare(Buffer.from(value, "base64").toString("base64")) === normalizeBase64ForCompare(value);
}
/** Parse a common provider media frame. */
function parseMediaFrame(msg) {
const mediaData = readRecordField(msg, "media");
const payload = typeof mediaData?.payload === "string" ? mediaData.payload : void 0;
if (!payload || !isValidBase64Payload(payload)) return { kind: "ignored" };
return {
kind: "media",
payloadBase64: payload,
timestampMs: parseTimestampMs(mediaData?.timestamp),
track: typeof mediaData?.track === "string" ? mediaData.track : void 0
};
}
/** Parse a common provider mark frame. */
function parseMarkFrame(msg) {
const markData = readRecordField(msg, "mark");
return {
kind: "mark",
name: typeof markData?.name === "string" ? markData.name : void 0
};
}
/** Parse common media, mark, and stop frames shared by supported providers. */
function parseCommonInboundFrame(event, msg) {
if (event === "media") return parseMediaFrame(msg);
if (event === "mark") return parseMarkFrame(msg);
if (event === "stop") return { kind: "stop" };
}
/** Parse one provider frame with provider-specific start/error hooks. */
function parseProviderInboundFrame(rawMessage, parseStartFrame, parseExtraFrame) {
const msg = tryParseJson(rawMessage);
if (!msg) return { kind: "ignored" };
const event = msg.event;
if (event === "start") return parseStartFrame(msg) ?? { kind: "ignored" };
return parseCommonInboundFrame(event, msg) ?? parseExtraFrame?.(event, msg) ?? { kind: "ignored" };
}
/** Include streamSid only when Twilio has already supplied one. */
function withOptionalStreamSid(streamSid) {
return streamSid === void 0 ? {} : { streamSid };
}
/** Serialize a provider media frame. */
function serializeMediaFrame(payloadBase64, streamSid) {
return JSON.stringify({
event: "media",
...withOptionalStreamSid(streamSid),
media: { payload: payloadBase64 }
});
}
/** Serialize a provider clear frame. */
function serializeClearFrame(streamSid) {
return JSON.stringify({
event: "clear",
...withOptionalStreamSid(streamSid)
});
}
/** Serialize a provider mark frame. */
function serializeMarkFrame(name, streamSid) {
return JSON.stringify({
event: "mark",
...withOptionalStreamSid(streamSid),
mark: { name }
});
}
/** Twilio media stream adapter, retaining streamSid for outbound frames. */
var TwilioStreamFrameAdapter = class {
constructor() {
this.providerName = "twilio";
this.streamSid = "";
}
/** Parse one Twilio websocket message into a normalized frame. */
parseInbound(rawMessage) {
return parseProviderInboundFrame(rawMessage, (msg) => {
const startData = readRecordField(msg, "start");
const streamSid = typeof startData?.streamSid === "string" ? startData.streamSid : "";
const callSid = typeof startData?.callSid === "string" ? startData.callSid : "";
if (!streamSid || !callSid) return;
this.streamSid = streamSid;
return {
kind: "start",
streamId: streamSid,
providerCallId: callSid
};
});
}
/** Serialize Twilio media with the active streamSid. */
serializeMedia(payloadBase64) {
return serializeMediaFrame(payloadBase64, this.streamSid);
}
/** Serialize Twilio clear with the active streamSid. */
serializeClear() {
return serializeClearFrame(this.streamSid);
}
/** Serialize Twilio mark with the active streamSid. */
serializeMark(name) {
return serializeMarkFrame(name, this.streamSid);
}
};
/** Telnyx media stream adapter. */
var TelnyxStreamFrameAdapter = class {
constructor() {
this.providerName = "telnyx";
}
/** Parse one Telnyx websocket message into a normalized frame. */
parseInbound(rawMessage) {
return parseProviderInboundFrame(rawMessage, (msg) => {
const topLevelStreamId = typeof msg.stream_id === "string" && msg.stream_id ? msg.stream_id : void 0;
const startData = readRecordField(msg, "start");
const providerCallId = typeof startData?.call_control_id === "string" && startData.call_control_id ? startData.call_control_id : void 0;
if (!topLevelStreamId || !providerCallId) return;
return {
kind: "start",
streamId: topLevelStreamId,
providerCallId
};
}, (event, msg) => {
if (event !== "error") return;
const errorData = readRecordField(msg, "payload");
return {
kind: "error",
code: typeof errorData?.code === "string" || typeof errorData?.code === "number" ? String(errorData.code) : void 0,
title: typeof errorData?.title === "string" ? errorData.title : void 0,
detail: typeof errorData?.detail === "string" ? errorData.detail : void 0
};
});
}
/** Serialize Telnyx media. */
serializeMedia(payloadBase64) {
return serializeMediaFrame(payloadBase64);
}
/** Serialize Telnyx clear. */
serializeClear() {
return serializeClearFrame();
}
/** Serialize Telnyx mark. */
serializeMark(name) {
return serializeMarkFrame(name);
}
};
//#endregion
//#region extensions/voice-call/src/webhook/realtime-handler.ts
const STREAM_TOKEN_TTL_MS = 3e4;
const DEFAULT_HOST = "localhost:8443";
const MAX_REALTIME_MESSAGE_BYTES = 256 * 1024;
const MAX_REALTIME_WS_BUFFERED_BYTES = 1024 * 1024;
const FORCED_CONSULT_FALLBACK_DELAY_MS = 200;
const FORCED_CONSULT_NATIVE_DEDUPE_MS = 2e3;
const FORCED_CONSULT_RESULT_MAX_CHARS = 1800;
const FORCED_CONSULT_REASON = "provider_final_transcript_without_openclaw_agent_consult";
const CONSULT_TRANSCRIPT_SETTLE_MS = 350;
const CONSULT_TRANSCRIPT_SETTLE_MAX_MS = 1e3;
const MAX_PARTIAL_USER_TRANSCRIPT_CHARS = 1200;
const RECENT_FINAL_USER_TRANSCRIPT_TTL_MS = 2e3;
const BARGE_IN_REQUIRED_LOUD_CHUNKS = 2;
const logger = createSubsystemLogger("voice-call/realtime");
function normalizePath(pathname) {
const trimmed = pathname.trim();
if (!trimmed) return "/";
const prefixed = trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
if (prefixed === "/") return prefixed;
return prefixed.endsWith("/") ? prefixed.slice(0, -1) : prefixed;
}
function buildGreetingInstructions(baseInstructions, greeting) {
const trimmedGreeting = greeting?.trim();
if (!trimmedGreeting) return;
const intro = "Start the call by greeting the caller naturally. Include this greeting in your first spoken reply:";
return baseInstructions ? `${baseInstructions}\n\n${intro} "${trimmedGreeting}"` : `${intro} "${trimmedGreeting}"`;
}
function readConsultArgText(args, key) {
if (!args || typeof args !== "object" || Array.isArray(args)) return;
const value = args[key];
return typeof value === "string" && value.trim() ? value.trim() : void 0;
}
function readConsultQuestionText(args) {
return readRealtimeVoiceConsultQuestion(args);
}
function normalizeTranscriptText(text) {
return text.replace(/\s+/g, " ").trim();
}
function findTextOverlap(base, next) {
const max = Math.min(base.length, next.length);
for (let size = max; size > 0; size -= 1) if (base.slice(-size) === next.slice(0, size)) return size;
return 0;
}
function shouldInsertTranscriptSpace(base, next) {
if (!base || !next) return false;
const last = base.at(-1);
if (/\s$/.test(base) || last === "(" || last === "[" || last === "{" || last === "\"" || last === "'" || /^[\s,.;:!?)]/.test(next)) return false;
return true;
}
function appendTranscriptText(base, fragment) {
const next = normalizeTranscriptText(fragment);
if (!next) return base ?? "";
const current = normalizeTranscriptText(base ?? "");
if (!current) return next;
const currentLower = current.toLowerCase();
const nextLower = next.toLowerCase();
if (currentLower === nextLower || currentLower.endsWith(nextLower)) return current;
if (nextLower.startsWith(currentLower)) return next;
const overlap = findTextOverlap(currentLower, nextLower);
if (overlap >= 6 || overlap >= 3 && next.length <= 12) return `${current}${next.slice(overlap)}`.trim();
return `${current}${shouldInsertTranscriptSpace(current, next) ? " " : ""}${next}`.trim();
}
function limitPartialUserTranscript(text) {
if (text.length <= MAX_PARTIAL_USER_TRANSCRIPT_CHARS) return text;
const tail = text.slice(-1200);
return tail.replace(/^\S+\s+/, "").trimStart() || tail.trimStart();
}
function withFallbackConsultQuestion(args, fallback) {
const providerQuestion = readConsultQuestionText(args);
const question = fallback?.trim();
if (providerQuestion) {
if (question && providerQuestion.length <= 40 && question.length >= providerQuestion.length + 8) {
const context = readConsultArgText(args, "context");
const fallbackContext = `Realtime provider supplied a shorter consult question: ${providerQuestion}`;
return args && typeof args === "object" && !Array.isArray(args) ? {
...args,
question,
context: context ? `${context}\n\n${fallbackContext}` : fallbackContext
} : {
question,
context: fallbackContext
};
}
return args;
}
if (!question) return args;
return args && typeof args === "object" && !Array.isArray(args) ? {
...args,
question
} : { question };
}
function buildForcedConsultSpeechPrompt(result) {
const trimmed = result.trim();
return [
"Internal OpenClaw consult result is ready.",
"Do not call tools for this internal result.",
"Speak the following answer to the caller now, briefly and naturally:",
trimmed.length <= FORCED_CONSULT_RESULT_MAX_CHARS ? trimmed : `${trimmed.slice(0, FORCED_CONSULT_RESULT_MAX_CHARS - 16).trimEnd()} [truncated]`
].join("\n");
}
function appendRecentTalkEventMetadata(call, event) {
if (!call) return;
const metadata = call.metadata ?? {};
const previous = Array.isArray(metadata.recentTalkEvents) ? metadata.recentTalkEvents : [];
metadata.lastTalkEventAt = event.timestamp;
metadata.lastTalkEventType = event.type;
metadata.recentTalkEvents = [...previous, {
id: event.id,
brain: event.brain,
mode: event.mode,
provider: event.provider,
seq: event.seq,
sessionId: event.sessionId,
timestamp: event.timestamp,
transport: event.transport,
type: event.type,
...event.turnId ? { turnId: event.turnId } : {},
...event.final !== void 0 ? { final: event.final } : {}
}].slice(-12);
call.metadata = metadata;
}
var RealtimeCallHandler = class {
constructor(config, manager, provider, realtimeProvider, providerConfig, servePath, coreConfig) {
this.config = config;
this.manager = manager;
this.provider = provider;
this.realtimeProvider = realtimeProvider;
this.providerConfig = providerConfig;
this.servePath = servePath;
this.coreConfig = coreConfig;
this.toolHandlers = /* @__PURE__ */ new Map();
this.pendingStreamTokens = /* @__PURE__ */ new Map();
this.activeBridgesByCallId = /* @__PURE__ */ new Map();
this.activeTelephonyClosersByCallId = /* @__PURE__ */ new Map();
this.partialUserTranscriptsByCallId = /* @__PURE__ */ new Map();
this.partialUserTranscriptUpdatedAtByCallId = /* @__PURE__ */ new Map();
this.recentFinalUserTranscriptsByCallId = /* @__PURE__ */ new Map();
this.recentFinalUserTranscriptTimersByCallId = /* @__PURE__ */ new Map();
this.forcedConsultCoordinatorsByCallId = /* @__PURE__ */ new Map();
this.forcedConsultsByCallId = /* @__PURE__ */ new Map();
this.nativeConsultsInFlightByCallId = /* @__PURE__ */ new Map();
this.publicOrigin = null;
this.publicPathPrefix = "";
}
setPublicUrl(url) {
try {
const parsed = new URL(url);
this.publicOrigin = parsed.host;
const normalizedServePath = normalizePath(this.servePath);
const normalizedPublicPath = normalizePath(parsed.pathname);
const idx = normalizedPublicPath.indexOf(normalizedServePath);
this.publicPathPrefix = idx > 0 ? normalizedPublicPath.slice(0, idx) : "";
} catch {
this.publicOrigin = null;
this.publicPathPrefix = "";
}
}
getStreamPathPattern() {
return `${this.publicPathPrefix}${normalizePath(this.config.streamPath ?? "/voice/stream/realtime")}`;
}
buildTwiMLPayload(req, params) {
const rawDirection = params?.get("Direction");
const previousOrigin = this.publicOrigin;
if (!previousOrigin) this.publicOrigin = req.headers.host ?? DEFAULT_HOST;
try {
const { streamUrl } = this.issueStreamSession({
providerName: "twilio",
from: params?.get("From") ?? void 0,
to: params?.get("To") ?? void 0,
direction: rawDirection?.startsWith("outbound") ? "outbound" : "inbound"
});
return {
statusCode: 200,
headers: { "Content-Type": "text/xml" },
body: `<?xml version="1.0" encoding="UTF-8"?>
<Response>
<Connect>
<Stream url="${streamUrl}" />
</Connect>
</Response>`
};
} finally {
this.publicOrigin = previousOrigin;
}
}
handleWebSocketUpgrade(request, socket, head) {
const token = new URL(request.url ?? "/", "wss://localhost").pathname.split("/").pop() ?? null;
const callerMeta = token ? this.consumeStreamToken(token) : null;
if (!callerMeta) {
socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
socket.destroy();
return;
}
const adapter = (callerMeta.providerName ?? "twilio") === "telnyx" ? new TelnyxStreamFrameAdapter() : new TwilioStreamFrameAdapter();
new WebSocketServer({
noServer: true,
maxPayload: MAX_REALTIME_MESSAGE_BYTES
}).handleUpgrade(request, socket, head, (ws) => {
let bridge = null;
let initialized = false;
let activeCallSid = "unknown";
let stopReceived = false;
let lastMediaTimestamp;
let lastMediaGapWarnAt = 0;
ws.on("message", (data) => {
try {
const frame = adapter.parseInbound(data.toString());
if (frame.kind === "ignored") return;
if (frame.kind === "start") {
if (initialized) return;
initialized = true;
activeCallSid = frame.providerCallId;
const nextBridge = this.handleCall(frame.streamId, frame.providerCallId, ws, callerMeta, adapter);
if (!nextBridge) return;
bridge = nextBridge;
return;
}
if (!bridge) return;
if (frame.kind === "media") {
const audio = Buffer.from(frame.payloadBase64, "base64");
bridge.sendAudio(audio);
if (frame.timestampMs !== void 0) {
if (lastMediaTimestamp !== void 0) {
const gapMs = frame.timestampMs - lastMediaTimestamp;
const now = Date.now();
if ((gapMs > 120 || gapMs < 0) && now - lastMediaGapWarnAt > 5e3) {
lastMediaGapWarnAt = now;
console.warn(`[voice-call] realtime media timestamp gap providerCallId=${activeCallSid} gapMs=${gapMs} timestamp=${frame.timestampMs}`);
}
}
lastMediaTimestamp = frame.timestampMs;
bridge.setMediaTimestamp(frame.timestampMs);
}
return;
}
if (frame.kind === "mark") {
bridge.acknowledgeMark();
return;
}
if (frame.kind === "error") {
console.error(`[voice-call] realtime WS error frame providerCallId=${activeCallSid} code=${frame.code ?? "?"} title=${frame.title ?? ""} detail=${frame.detail ?? ""}`);
return;
}
if (frame.kind === "stop") {
stopReceived = true;
this.closeTelephonyBridge(activeCallSid, bridge, "completed");
}
} catch (error) {
console.error("[voice-call] realtime WS parse failed:", error);
}
});
ws.on("close", (code) => {
const reason = stopReceived || code === 1e3 || code === 1005 ? "completed" : "error";
this.closeTelephonyBridge(activeCallSid, bridge, reason);
});
ws.on("error", (error) => {
console.error("[voice-call] realtime WS error:", error);
});
});
}
registerToolHandler(name, fn) {
this.toolHandlers.set(name, fn);
}
speak(callId, instructions) {
const bridge = this.activeBridgesByCallId.get(callId);
if (!bridge) return {
success: false,
error: "No active realtime bridge for call"
};
try {
bridge.triggerGreeting(instructions);
return { success: true };
} catch (error) {
return {
success: false,
error: formatErrorMessage(error)
};
}
}
issueStreamSession(request = {}) {
const token = this.issueStreamToken({
providerName: request.providerName ?? "twilio",
callId: request.callId,
from: request.from,
to: request.to,
direction: request.direction
});
return {
token,
streamUrl: `wss://${this.publicOrigin || DEFAULT_HOST}${this.getStreamPathPattern()}/${token}`
};
}
issueStreamToken(meta = {}) {
const token = randomUUID();
const now = Date.now();
const expiry = resolveExpiresAtMsFromDurationMs(STREAM_TOKEN_TTL_MS, { nowMs: now });
if (expiry !== void 0) this.pendingStreamTokens.set(token, {
expiry,
...meta
});
for (const [candidate, entry] of this.pendingStreamTokens) if (!isFutureDateTimestampMs(entry.expiry, { nowMs: now })) this.pendingStreamTokens.delete(candidate);
return token;
}
consumeStreamToken(token) {
const entry = this.pendingStreamTokens.get(token);
if (!entry) return null;
this.pendingStreamTokens.delete(token);
if (!isFutureDateTimestampMs(entry.expiry)) return null;
return {
from: entry.from,
to: entry.to,
direction: entry.direction,
providerName: entry.providerName,
callId: entry.callId
};
}
handleCall(streamSid, callSid, ws, callerMeta, adapter) {
const registration = this.registerCallInManager(callSid, callerMeta);
if (!registration) {
ws.close(1008, "Caller rejected by policy");
return null;
}
const { callId, initialGreetingInstructions } = registration;
const callRecord = this.manager.getCallByProviderCallId(callSid);
const talk = createTalkSessionController({
sessionId: `voice-call:${callId}:realtime`,
mode: "realtime",
transport: "gateway-relay",
brain: "agent-consult",
provider: this.realtimeProvider.id
}, { onEvent: recordTalkObservabilityEvent });
const rememberTalkEvent = (event) => {
if (event) appendRecentTalkEventMetadata(callRecord, event);
return event;
};
const emitTalkEvent = (input) => {
return rememberTalkEvent(talk.emit(input));
};
const ensureTalkTurn = () => {
const turn = talk.ensureTurn({ payload: {
callId,
providerCallId: callSid
} });
rememberTalkEvent(turn.event);
return turn.turnId;
};
const endTalkTurn = (reason = "completed") => {
const ended = talk.endTurn({ payload: {
callId,
providerCallId: callSid,
reason
} });
if (ended.ok) rememberTalkEvent(ended.event);
};
const finishOutputAudio = (reason) => {
rememberTalkEvent(talk.finishOutputAudio({ payload: {
callId,
providerCallId: callSid,
reason
} }));
};
emitTalkEvent({
type: "session.started",
payload: {
callId,
providerCallId: callSid,
streamSid
}
});
console.log(`[voice-call] Realtime bridge starting for call ${callId} (providerCallId=${callSid}, initialGreeting=${initialGreetingInstructions ? "queued" : "absent"})`);
let callEndEmitted = false;
const emitCallEnd = (reason) => {
if (callEndEmitted) return;
callEndEmitted = true;
this.endCallInManager(callSid, callId, reason);
};
const sendString = (message) => {
if (ws.readyState !== WebSocket.OPEN) return false;
if (ws.bufferedAmount > MAX_REALTIME_WS_BUFFERED_BYTES) {
console.warn(`[voice-call] realtime outbound websocket backpressure before send callId=${callId} providerCallId=${callSid} bufferedBytes=${ws.bufferedAmount}`);
ws.close(1013, "Backpressure: send buffer exceeded");
return false;
}
ws.send(message);
if (ws.bufferedAmount > MAX_REALTIME_WS_BUFFERED_BYTES) {
console.warn(`[voice-call] realtime outbound websocket backpressure after send callId=${callId} providerCallId=${callSid} bufferedBytes=${ws.bufferedAmount}`);
ws.close(1013, "Backpressure: send buffer exceeded");
return false;
}
return true;
};
const audioPacer = new RealtimeAudioPacer({
send: sendString,
serializer: {
media: (payload) => adapter.serializeMedia(payload),
clear: () => adapter.serializeClear(),
mark: (name) => adapter.serializeMark(name)
},
onBackpressure: () => {
console.warn(`[voice-call] realtime paced audio backpressure callId=${callId} providerCallId=${callSid}`);
if (ws.readyState === WebSocket.OPEN) ws.close(1013, "Backpressure: paced audio queue exceeded");
}
});
const speechDetector = new RealtimeMulawSpeechStartDetector({ requiredLoudChunks: BARGE_IN_REQUIRED_LOUD_CHUNKS });
const session = createRealtimeVoiceBridgeSession({
provider: this.realtimeProvider,
cfg: this.coreConfig,
providerConfig: this.providerConfig,
instructions: this.config.instructions,
tools: this.config.tools,
initialGreetingInstructions,
triggerGreetingOnReady: Boolean(initialGreetingInstructions),
audioSink: {
isOpen: () => ws.readyState === WebSocket.OPEN,
sendAudio: (muLaw) => {
const turnId = ensureTalkTurn();
rememberTalkEvent(talk.startOutputAudio({
turnId,
payload: {
callId,
providerCallId: callSid
}
}).event);
emitTalkEvent({
type: "output.audio.delta",
turnId,
payload: { byteLength: muLaw.length }
});
audioPacer.sendAudio(muLaw);
},
clearAudio: () => {
const clearedBytes = audioPacer.clearAudio();
console.log(`[voice-call] realtime outbound audio clear requested callId=${callId} providerCallId=${callSid} queuedBytes=${clearedBytes}`);
finishOutputAudio("clear");
},
sendMark: (markName) => {
audioPacer.sendMark(markName);
}
},
onTranscript: (role, text, isFinal) => {
const turnId = ensureTalkTurn();
emitTalkEvent({
type: role === "assistant" ? isFinal ? "output.text.done" : "output.text.delta" : isFinal ? "transcript.done" : "transcript.delta",
turnId,
payload: role === "assistant" ? { text } : {
role,
text
},
final: isFinal
});
if (role === "user" && isFinal) emitTalkEvent({
type: "input.audio.committed",
turnId,
payload: {
callId,
providerCallId: callSid
},
final: true
});
if (!isFinal) {
if (role === "user" && text.trim()) {
const transcript = this.recordPartialUserTranscript(callId, text);
console.log(`[voice-call] realtime input transcript callId=${callId} providerCallId=${callSid} final=false chars=${text.trim().length} aggregateChars=${transcript.length}`);
}
return;
}
if (role === "user") {
const transcript = this.recordPartialUserTranscript(callId, text);
this.clearPartialUserTranscript(callId);
this.setRecentFinalUserTranscript(callId, transcript);
console.log(`[voice-call] realtime input transcript callId=${callId} providerCallId=${callSid} final=true chars=${text.trim().length} aggregateChars=${transcript.length}`);
const event = {
id: `realtime-speech-${callSid}-${Date.now()}`,
type: "call.speech",
callId,
providerCallId: callSid,
timestamp: Date.now(),
transcript,
isFinal: true
};
this.manager.processEvent(event);
this.scheduleForcedAgentConsult({
session,
callId,
callSid,
transcript,
clearAudio: () => {
const clearedBytes = audioPacer.clearAudio();
console.log(`[voice-call] realtime forced consult cleared outbound audio callId=${callId} providerCallId=${callSid} queuedBytes=${clearedBytes}`);
}
});
return;
}
this.manager.processEvent({
id: `realtime-bot-${callSid}-${Date.now()}`,
type: "call.speaking",
callId,
providerCallId: callSid,
timestamp: Date.now(),
text
});
},
onToolCall: (toolEvent, sessionLocal) => {
const turnId = ensureTalkTurn();
emitTalkEvent({
type: "tool.call",
turnId,
itemId: toolEvent.itemId,
callId: toolEvent.callId,
payload: {
name: toolEvent.name,
args: toolEvent.args
}
});
console.log(`[voice-call] realtime tool call received callId=${callId} providerCallId=${callSid} tool=${toolEvent.name}`);
this.executeToolCall(sessionLocal, callId, toolEvent.callId || toolEvent.itemId, toolEvent.name, toolEvent.args, turnId, emitTalkEvent);
},
onEvent: (event) => {
if (event.type === "input_audio_buffer.speech_started") {
ensureTalkTurn();
return;
}
if (event.type === "input_audio_buffer.speech_stopped") {
const turnId = talk.activeTurnId;
if (!turnId) return;
emitTalkEvent({
type: "input.audio.committed",
turnId,
payload: {
callId,
providerCallId: callSid,
source: event.type
},
final: true
});
return;
}
if (event.type === "response.done") {
finishOutputAudio("response.done");
endTalkTurn("response.done");
return;
}
if (event.type === "error") emitTalkEvent({
type: "session.error",
payload: { message: event.detail ?? "Realtime provider error" },
final: true
});
},
onReady: () => {
emitTalkEvent({
type: "session.ready",
payload: {
callId,
providerCallId: callSid
}
});
},
onError: (error) => {
console.error("[voice-call] realtime voice error:", error.message);
emitTalkEvent({
type: "session.error",
payload: { message: error.message },
final: true
});
},
onClose: (reason) => {
this.activeBridgesByCallId.delete(callId);
this.activeBridgesByCallId.delete(callSid);
this.activeTelephonyClosersByCallId.delete(callId);
this.activeTelephonyClosersByCallId.delete(callSid);
this.clearUserTranscriptState(callId);
finishOutputAudio(reason);
emitTalkEvent({
type: "session.closed",
payload: { reason },
final: true
});
if (reason !== "error") return;
emitCallEnd("error");
if (ws.readyState === WebSocket.OPEN) ws.close(1011, "Bridge disconnected");
this.provider.hangupCall({
callId,
providerCallId: callSid,
reason: "error"
}).catch((error) => {
console.warn(`[voice-call] Failed to hang up realtime call ${callSid}: ${formatErrorMessage(error)}`);
});
}
});
const closeTelephony = (reason) => {
emitCallEnd(reason);
session.close();
};
this.activeBridgesByCallId.set(callId, session);
this.activeBridgesByCallId.set(callSid, session);
this.activeTelephonyClosersByCallId.set(callId, closeTelephony);
this.activeTelephonyClosersByCallId.set(callSid, closeTelephony);
const sendAudioToSession = session.sendAudio.bind(session);
session.sendAudio = (audio) => {
if (speechDetector.accept(audio)) {
const interruptedTurnId = ensureTalkTurn();
const clearedBytes = audioPacer.clearAudio();
console.log(`[voice-call] realtime outbound audio cleared by barge-in callId=${callId} providerCallId=${callSid} queuedBytes=${clearedBytes}`);
finishOutputAudio("barge-in");
const cancelled = talk.cancelTurn({
turnId: interruptedTurnId,
payload: {
callId,
providerCallId: callSid,
reason: "barge-in"
}
});
if (cancelled.ok) rememberTalkEvent(cancelled.event);
}
emitTalkEvent({
type: "input.audio.delta",
turnId: ensureTalkTurn(),
payload: { byteLength: audio.length }
});
sendAudioToSession(audio);
};
const closeSession = session.close.bind(session);
session.close = () => {
this.activeBridgesByCallId.delete(callId);
this.activeBridgesByCallId.delete(callSid);
this.activeTelephonyClosersByCallId.delete(callId);
this.activeTelephonyClosersByCallId.delete(callSid);
this.clearUserTranscriptState(callId);
this.clearForcedConsultState(callId);
audioPacer.close();
closeSession();
};
session.connect().catch((error) => {
console.error("[voice-call] Failed to connect realtime bridge:", error);
session.close();
emitCallEnd("error");
ws.close(1011, "Failed to connect");
});
return session;
}
recordPartialUserTranscript(callId, text) {
const next = limitPartialUserTranscript(appendTranscriptText(this.partialUserTranscriptsByCallId.get(callId), text));
this.partialUserTranscriptsByCallId.set(callId, next);
this.partialUserTranscriptUpdatedAtByCallId.set(callId, Date.now());
return next;
}
clearPartialUserTranscript(callId) {
this.partialUserTranscriptsByCallId.delete(callId);
this.partialUserTranscriptUpdatedAtByCallId.delete(callId);
}
setRecentFinalUserTranscript(callId, text) {
this.clearRecentFinalUserTranscript(callId);
this.recentFinalUserTranscriptsByCallId.set(callId, text);
const timer = setTimeout(() => {
if (this.recentFinalUserTranscriptsByCallId.get(callId) === text) this.recentFinalUserTranscriptsByCallId.delete(callId);
this.recentFinalUserTranscriptTimersByCallId.delete(callId);
}, RECENT_FINAL_USER_TRANSCRIPT_TTL_MS);
timer.unref?.();
this.recentFinalUserTranscriptTimersByCallId.set(callId, timer);
}
clearRecentFinalUserTranscript(callId) {
const timer = this.recentFinalUserTranscriptTimersByCallId.get(callId);
if (timer) {
clearTimeout(timer);
this.recentFinalUserTranscriptTimersByCallId.delete(callId);
}
this.recentFinalUserTranscriptsByCallId.delete(callId);
}
clearUserTranscriptState(callId) {
this.clearPartialUserTranscript(callId);
this.clearRecentFinalUserTranscript(callId);
}
resolveUserTranscriptContext(callId) {
return this.partialUserTranscriptsByCallId.get(callId) ?? this.recentFinalUserTranscriptsByCallId.get(callId);
}
consumePartialUserTranscript(callId, consumed) {
const text = consumed?.trim();
if (!text) return;
const current = this.partialUserTranscriptsByCallId.get(callId);
if (!current) return;
if (current === text) {
this.clearPartialUserTranscript(callId);
return;
}
if (current.toLowerCase().startsWith(text.toLowerCase())) {
const remaining = current.slice(text.length).trimStart();
if (remaining) this.partialUserTranscriptsByCallId.set(callId, remaining);
else this.clearPartialUserTranscript(callId);
}
const recent = this.recentFinalUserTranscriptsByCallId.get(callId);
if (!recent) return;
if (recent === text || recent.toLowerCase().startsWith(text.toLowerCase())) this.clearRecentFinalUserTranscript(callId);
}
async waitForConsultTranscriptSettle(callId, startedAt) {
const deadline = startedAt + CONSULT_TRANSCRIPT_SETTLE_MAX_MS;
while (true) {
const updatedAt = this.partialUserTranscriptUpdatedAtByCallId.get(callId);
if (!updatedAt) return;
const now = Date.now();
const quietFor = now - updatedAt;
if (quietFor >= CONSULT_TRANSCRIPT_SETTLE_MS || now >= deadline) return;
await new Promise((resolve) => {
setTimeout(resolve, Math.min(CONSULT_TRANSCRIPT_SETTLE_MS - quietFor, deadline - now));
});
}
}
clearForcedConsultState(callId) {
this.forcedConsultCoordinatorsByCallId.get(callId)?.clear();
this.forcedConsultCoordinatorsByCallId.delete(callId);
this.forcedConsultsByCallId.delete(callId);
}
forcedConsultCoordinator(callId) {
const existing = this.forcedConsultCoordinatorsByCallId.get(callId);
if (existing) return existing;
const created = createRealtimeVoiceForcedConsultCoordinator();
this.forcedConsultCoordinatorsByCallId.set(callId, created);
return created;
}
closeTelephonyBridge(callIdOrSid, bridge, reason) {
const closer = this.activeTelephonyClosersByCallId.get(callIdOrSid);
if (closer) {
closer(reason);
return;
}
bridge?.close();
}
scheduleForcedAgentConsult(params) {
if (this.config.consultPolicy !== "always") return;
const question = params.transcript.trim();
if (!question) return;
const handler = this.toolHandlers.get(REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME);
if (!handler) return;
const existingForcedConsult = this.forcedConsultsByCallId.get(params.callId);
if (existingForcedConsult && !existingForcedConsult.completedAt) return;
const coordinator = this.forcedConsultCoordinator(params.callId);
if (coordinator.hasRecentNativeConsult(question, { allowUnknownQuestion: true })) return;
coordinator.clearPending();
const pending = coordinator.prepare(question);
if (!pending) return;
coordinator.schedule(pending, FORCED_CONSULT_FALLBACK_DELAY_MS, (handle) => {
const activeForcedConsult = this.forcedConsultsByCallId.get(params.callId);
if (activeForcedConsult && !activeForcedConsult.completedAt) return;
this.runForcedAgentConsult({
...params,
handle,
handler
});
});
}
async runForcedAgentConsult(params) {
const coordinator = this.forcedConsultCoordinator(params.callId);
coordinator.markStarted(params.handle);
const startedAt = Date.now();
logger.debug(`[voice-call] realtime forced agent consult reason=${FORCED_CONSULT_REASON} consultPolicy=always callId=${params.callId} providerCallId=${params.callSid} chars=${params.handle.question.length}`);
console.log(`[voice-call] realtime forced agent consult starting callId=${params.callId} providerCallId=${params.callSid} chars=${params.handle.question.length}`);
params.clearAudio();
const state = {
sendSpeechPrompt: true,
promise: Promise.resolve().then(() => params.handler({ question: params.handle.question }, params.callId, {}))
};
this.forcedConsultsByCallId.set(params.callId, state);
try {
const result = await state.promise;
state.completedAt = Date.now();
coordinator.markDelivered(params.handle);
const text = readSpeakableRealtimeVoiceToolResult(result, {
keys: ["text", "output"],
maxChars: FORCED_CONSULT_RESULT_MAX_CHARS
});
if (!text) {
console.warn(`[voice-call] realtime forced agent consult returned no speakable text callId=${params.callId} providerCallId=${params.callSid}`);
return;
}
if (state.sendSpeechPrompt) {
params.clearAudio();
params.session.sendUserMessage(buildForcedConsultSpeechPrompt(text));
}
console.log(`[voice-call] realtime forced agent consult completed callId=${params.callId} providerCallId=${params.callSid} elapsedMs=${Date.now() - startedAt}`);
this.consumePartialUserTranscript(params.callId, params.handle.question);
} catch (error) {
console.warn(`[voice-call] realtime forced agent consult failed callId=${params.callId} providerCallId=${params.callSid} error=${formatErrorMessage(error)}`);
} finally {
setTimeout(() => {
if (this.forcedConsultsByCallId.get(params.callId) === state) {
this.forcedConsultsByCallId.delete(params.callId);
coordinator.remove(params.handle);
}
}, FORCED_CONSULT_NATIVE_DEDUPE_MS).unref?.();
}
}
registerCallInManager(callSid, callerMeta = {}) {
const baseFields = {
providerCallId: callSid,
timestamp: Date.now(),
direction: callerMeta.direction ?? "inbound",
...callerMeta.from ? { from: callerMeta.from } : {},
...callerMeta.to ? { to: callerMeta.to } : {}
};
const callRecord = this.resolveRealtimeCall(callSid, callerMeta, baseFields);
if (!callRecord) return null;
const initialGreeting = this.extractInitialGreeting(callRecord);
console.log(`[voice-call] Realtime call ${callRecord.callId} initial greeting ${initialGreeting ? "queued" : "absent"}`);
if (callRecord.metadata) delete callRecord.metadata.initialMessage;
this.manager.processEvent({
id: `realtime-answered-${callSid}`,
callId: callRecord.callId,
type: "call.answered",
...baseFields
});
return {
callId: callRecord.callId,
initialGreetingInstructions: buildGreetingInstructions(this.config.instructions, initialGreeting)
};
}
resolveRealtimeCall(callSid, callerMeta, baseFields) {
if (callerMeta.callId) {
const call = this.manager.getCall(callerMeta.callId);
return call?.providerCallId === callSid ? call : null;
}
this.manager.processEvent({
id: `realtime-initiated-${callSid}`,
callId: callSid,
type: "call.initiated",
...baseFields
});
return this.manager.getCallByProviderCallId(callSid) ?? null;
}
extractInitialGreeting(call) {
return typeof call.metadata?.initialMessage === "string" ? call.metadata.initialMessage : void 0;
}
endCallInManager(callSid, callId, reason) {
this.manager.processEvent({
id: `realtime-ended-${callSid}-${Date.now()}`,
type: "call.ended",
callId,
providerCallId: callSid,
timestamp: Date.now(),
reason
});
}
async executeToolCall(bridge, callId, bridgeCallId, name, args, turnId, emitTalkEvent) {
const handler = this.toolHandlers.get(name);
const startedAt = Date.now();
const hasResultError = (result) => {
return Boolean(result && typeof result === "object" && !Array.isArray(result) && "error" in result);
};
const emitFinalToolEvent = (result) => {
emitTalkEvent?.({
type: hasResultError(result) ? "tool.error" : "tool.result",
turnId,
callId: bridgeCallId,
payload: {
name,
result
},
final: true
});
};
const submitFinalToolResult = (result) => {
bridge.submitToolResult(bridgeCallId, result);
emitFinalToolEvent(result);
};
const submitWorkingResponse = () => {
if (handler && name === "openclaw_agent_consult" && bridge.bridge.supportsToolResultContinuation && !this.config.fastContext.enabled) {
emitTalkEvent?.({
type: "tool.progress",
turnId,
callId: bridgeCallId,
payload: {
name,
status: "working"
}
});
bridge.submitToolResult(bridgeCallId, buildRealtimeVoiceAgentConsultWorkingResponse("caller"), { willContinue: true });
}
};
if (name === "openclaw_agent_consult") {
const coordinator = this.forcedConsultCoordinator(callId);
const forcedMatch = coordinator.recordNativeConsult(args, bridgeCallId);
if (forcedMatch.kind === "none") {
const pending = coordinator.consumePending();
if (pending) coordinator.remove(pending);
}
const forcedConsult = this.forcedConsultsByCallId.get(callId);
if (forcedConsult) {
if (forcedConsult.completedAt || forcedMatch.kind === "already_delivered") {
submitFinalToolResult({
status: "already_delivered",
message: "OpenClaw already delivered this consult result internally. Do not repeat it."
});
return;
}
forcedConsult.sendSpeechPrompt = false;
submitFinalToolResult(await forcedConsult.promise.catch((error) => ({ error: formatErrorMessage(error) })));
return;
}
const existingNativeConsult = this.nativeConsultsInFlightByCallId.get(callId);
if (existingNativeConsult) {
console.log(`[voice-call] realtime tool call sharing in-flight agent consult callId=${callId} ageMs=${Date.now() - existingNativeConsult.startedAt}`);
submitWorkingResponse();
submitFinalToolResult(await existingNativeConsult.promise);
return;
}
submitWorkingResponse();
const state = {
startedAt,
promise: Promise.resolve()
};
state.promise = (async () => {
await this.waitForConsultTranscriptSettle(callId, startedAt);
const context = { partialUserTranscript: this.resolveUserTranscriptContext(callId) };
state.partialUserTranscript = context.partialUserTranscript;
const handlerArgs = withFallbackConsultQuestion(args, context.partialUserTranscript);
console.log(`[voice-call] realtime tool call executing callId=${callId} tool=${name} hasHandler=${Boolean(handler)}`);
return !handler ? { error: `Tool "${name}" not available` } : await handler(handlerArgs, callId, context);
})().catch((error) => ({ error: formatErrorMessage(error) }));
this.nativeConsultsInFlightByCallId.set(callId, state);
try {
const result = await state.promise;
const status = result && typeof result === "object" && !Array.isArray(result) && "error" in result ? "error" : "ok";
const error = status === "error" && result && typeof result === "object" && !Array.isArray(result) ? formatErrorMessage(result.error ?? "unknown") : void 0;
console.log(`[voice-call] realtime tool call completed callId=${callId} tool=${name} status=${status} elapsedMs=${Date.now() - startedAt}${error ? ` error=${error}` : ""}`);
submitFinalToolResult(result);
if (status === "ok") this.consumePartialUserTranscript(callId, state.partialUserTranscript);
} finally {
if (this.nativeConsultsInFlightByCallId.get(callId) === state) this.nativeConsultsInFlightByCallId.delete(callId);
}
return;
}
console.log(`[voice-call] realtime tool call executing callId=${callId} tool=${name} hasHandler=${Boolean(handler)}`);
const context = { partialUserTranscript: this.resolveUserTranscriptContext(callId) };
const handlerArgs = name === "openclaw_agent_consult" ? withFallbackConsultQuestion(args, context.partialUserTranscript) : args;
const result = !handler ? { error: `Tool "${name}" not available` } : await handler(handlerArgs, callId, context).catch((error) => ({ error: formatErrorMessage(error) }));
const status = result && typeof result === "object" && !Array.isArray(result) && "error" in result ? "error" : "ok";
const error = status === "error" && result && typeof result === "object" && !Array.isArray(result) ? formatErrorMessage(result.error ?? "unknown") : void 0;
console.log(`[voice-call] realtime tool call completed callId=${callId} tool=${name} status=${status} elapsedMs=${Date.now() - startedAt}${error ? ` error=${error}` : ""}`);
submitFinalToolResult(result);
if (name === "openclaw_agent_consult" && status === "ok") this.consumePartialUserTranscript(callId, context.partialUserTranscript);
}
};
//#endregion
export { RealtimeCallHandler };