agents/dist/types-_Faxb570.d.ts

A home for your AI agents

//#region src/voice/types.d.ts
/**
 * Shared types for the voice pipeline.
 *
 * Used by both the server (index.ts) and client (client.ts)
 * to ensure protocol consistency.
 */
/**
 * Current voice protocol version.
 * Bump this when making backwards-incompatible wire protocol changes.
 * The server sends this in the initial `welcome` message so clients
 * can detect version mismatches.
 */
declare const VOICE_PROTOCOL_VERSION = 1;
type VoiceStatus = "idle" | "listening" | "thinking" | "speaking";
/** Server-side diagnostic configuration shared by both voice mixins. */
interface VoiceDiagnosticsOptions {
  /** Forward safe server diagnostics and enable browser console logging. */
  browserConsole?: boolean;
}
/**
 * A bounded diagnostic record forwarded by the voice protocol.
 * Event names and metadata are intentionally open and are not stable API.
 */
interface VoiceDiagnosticEvent {
  event: string;
  timestamp: number;
  data?: Record<string, unknown>;
}
type VoiceTurnSource = "speech" | "text";
type VoiceTurnOutcome =
  | "completed"
  | "no_output"
  | "output_limit"
  | "content_filtered"
  | "model_error"
  | "tts_error"
  | "aborted"
  | "skipped"
  | "error";
/**
 * Stable, content-free summary of one allocated voice turn.
 *
 * `turnId`, `source`, and `outcome` are dimensions used to correlate and
 * interpret the measurements; every other field is a duration. Optional
 * timings are omitted when their lifecycle landmark was not reached. Timings
 * use one server clock, overlap, and are not additive. Browser playback timing
 * is excluded because browser and Worker clocks are independent.
 */
interface VoiceTurnMetrics {
  /** SDK-assigned correlation ID for this turn. */
  turnId: string;
  /** Whether the turn originated from finalized speech or a text message. */
  source: VoiceTurnSource;
  /** Terminal result, emitted exactly once for the allocated turn. */
  outcome: VoiceTurnOutcome;
  /** Turn allocation to terminal summary, in milliseconds. */
  turnTotalMs: number;
  /** Provider speech start to the first interim transcript. */
  speechStartToFirstInterimMs?: number;
  /** Provider speech start to the finalized transcript. */
  speechStartToFinalMs?: number;
  /** Time spent in the server's `afterTranscribe` hook. */
  afterTranscribeMs?: number;
  /** Model invocation to the first non-whitespace text delta. */
  modelToFirstTextMs?: number;
  /** Cumulative duration of reasoning blocks exposed by the model stream. */
  exposedReasoningMs?: number;
  /** Model invocation through normalized stream consumption. */
  modelStreamConsumptionMs?: number;
  /** Finalized input through the first server audio send. */
  finalInputToFirstAudioMs?: number;
  /** First TTS provider invocation through the first server audio send. */
  ttsToFirstAudioMs?: number;
  /** First TTS provider invocation through completion of all sentence work. */
  ttsWallMs?: number;
  /** Cumulative overlapping TTS sentence hook and provider work. */
  ttsWorkMs?: number;
}
/** Audio format the server uses for binary audio payloads. */
type VoiceAudioFormat = "mp3" | "pcm16" | "wav" | "opus";
type VoiceRole = "user" | "assistant";
/** Stable machine-readable error codes emitted by the voice protocol. */
type VoiceErrorCode = "stt_startup_failed" | "stt_connection_lost";
/** Stable pipeline stage associated with a structured voice error. */
type VoiceErrorStage = "stt";
/** Client-safe error detail. `message` remains for string-event compatibility. */
interface VoiceError {
  message: string;
  code?: VoiceErrorCode;
  stage?: VoiceErrorStage;
  retryable?: boolean;
}
/** Stable machine-readable outcomes for non-ordinary LLM completions. */
type VoiceCompletionOutcomeCode =
  | "no_output"
  | "output_limit"
  | "content_filtered"
  | "model_error";
/** Normalized finish reasons emitted by the supported AI SDK stream shape. */
type VoiceModelFinishReason =
  | "stop"
  | "length"
  | "content-filter"
  | "tool-calls"
  | "error"
  | "other";
/** Bounded completion metadata exposed by the voice protocol. */
interface VoiceCompletionOutcome {
  code: VoiceCompletionOutcomeCode;
  stage: "llm";
  finishReason?: VoiceModelFinishReason;
  partialOutput: boolean;
}
type VoiceClientMessage =
  | {
      type: "hello";
      protocol_version?: number;
    }
  | {
      type: "start_call";
      preferred_format?: VoiceAudioFormat;
    }
  | {
      type: "end_call";
    }
  | {
      type: "start_of_speech";
    }
  | {
      type: "end_of_speech";
    }
  | {
      type: "interrupt";
    }
  | {
      type: "text_message";
      text: string;
    };
type VoiceServerMessage =
  | {
      type: "welcome";
      protocol_version: number;
      diagnostics?: {
        browser_console: true;
      };
    }
  | ({
      type: "diagnostic";
    } & VoiceDiagnosticEvent)
  | {
      type: "status";
      status: VoiceStatus;
    }
  | {
      type: "audio_config";
      format: VoiceAudioFormat;
      sampleRate?: number;
    }
  | {
      type: "transcript";
      role: VoiceRole;
      text: string;
    }
  | {
      type: "transcript_start";
      role: VoiceRole;
    }
  | {
      type: "transcript_delta";
      text: string;
    }
  | {
      type: "transcript_end";
      text: string;
    }
  | {
      type: "transcript_interim";
      text: string;
    }
  | {
      type: "playback_interrupt";
    }
  | {
      type: "metrics";
      llm_ms: number;
      tts_ms: number;
      first_audio_ms: number;
      total_ms: number;
    }
  | ({
      type: "turn_metrics";
    } & VoiceTurnMetrics)
  | ({
      type: "completion_outcome";
    } & VoiceCompletionOutcome)
  | ({
      type: "error";
    } & VoiceError);
/**
 * Compact compatibility summary for successful, non-empty speech turns.
 * These overlapping latency landmarks and work totals are not additive.
 * Use `VoiceTurnMetrics` for stable detailed timing and terminal summaries of
 * unsuccessful, aborted, skipped, or text turns.
 */
interface VoicePipelineMetrics {
  /**
   * Time from immediately before `onTurn()` until normalized model-stream
   * consumption completes. This can include tool work and consumer waits while
   * consuming the stream. It is not time to first text.
   */
  llm_ms: number;
  /**
   * Cumulative per-sentence work from immediately before `beforeSynthesize`
   * until that sentence's synthesis and hook work settles. Sentence work can
   * overlap both other sentences and model consumption, so this value can
   * exceed wall time and overlaps the other metrics.
   */
  tts_ms: number;
  /**
   * Time from turn-pipeline start, before `afterTranscribe`, to the first server
   * audio send. Includes post-STT hooks, model work, and TTS, but excludes STT
   * and browser playback. `0` means the server sent no audio.
   */
  first_audio_ms: number;
  /**
   * Time from the same turn-pipeline start until model consumption and TTS
   * draining complete. Measured before final context, persistence, and status
   * work. Excludes STT and browser playback.
   */
  total_ms: number;
}
interface TranscriptMessage {
  role: VoiceRole;
  text: string;
  timestamp: number;
}
interface TTSProvider {
  synthesize(text: string, signal?: AbortSignal): Promise<ArrayBuffer | null>;
}
interface StreamingTTSProvider {
  synthesizeStream(
    text: string,
    signal?: AbortSignal
  ): AsyncGenerator<ArrayBuffer>;
}
/**
 * Continuous speech-to-text provider.
 *
 * Creates a per-call session that receives audio continuously from
 * `start_call` to `end_call`. The model handles turn detection
 * internally — there is no client-side speech boundary signaling
 * required for STT.
 *
 * The session fires `onUtterance` when the model detects a complete
 * utterance (e.g. Flux `EndOfTurn`, Nova 3 `speech_final` +
 * endpointing). The voice pipeline maps this to `onTurn` (withVoice)
 * or `onTranscript` (withVoiceInput).
 */
interface Transcriber {
  /** Create a new transcription session for one call. */
  createSession(options?: TranscriberSessionOptions): TranscriberSession;
}
interface TranscriberSessionOptions {
  /** Language code (e.g. "en"). */
  language?: string;
  /**
   * Called when the provider produces an interim (unstable) transcript.
   * This text may change as more audio arrives.
   */
  onInterim?: (text: string) => void;
  /**
   * Called when the model detects the start of user speech.
   *
   * Providers can use this for low-latency barge-in before a final
   * utterance is available. The transcript may be omitted or unstable.
   */
  onSpeechStart?: (text?: string) => void;
  /**
   * Called when the model detects a complete utterance.
   * The transcript is the stable text for this turn.
   *
   * For Flux: fires on `EndOfTurn`.
   * For Nova 3: fires on `Results` with `speech_final: true`.
   */
  onUtterance?: (transcript: string) => void;
  /**
   * Called when the session can no longer transcribe because its provider
   * connection failed or closed unexpectedly. Providers must not call this
   * for teardown initiated by {@link TranscriberSession.close}.
   */
  onFatalError?: (error: Error) => void;
}
/**
 * A per-call transcription session. Lives for the entire call duration.
 *
 * Unlike per-utterance sessions, this session is never finished or
 * aborted mid-call. It receives all audio continuously and the model
 * handles speech boundary detection. On interrupt, the LLM+TTS
 * pipeline is aborted but the transcriber session stays alive.
 */
interface TranscriberSession {
  /**
   * Feed raw PCM audio (16kHz mono 16-bit LE).
   * Fire-and-forget — the session buffers internally as needed.
   */
  feed(chunk: ArrayBuffer): void;
  /**
   * Resolves when the session is ready to accept audio and emit transcripts.
   * Optional so existing custom transcribers can start synchronously.
   */
  waitUntilReady?(): Promise<void>;
  /**
   * Optional. Provide the agent's most recent spoken reply (the text sent to
   * TTS) as conversational context for the next user turn.
   *
   * The pipeline calls this after the agent finishes speaking each reply and
   * greeting. Providers that support context carryover (e.g. AssemblyAI's
   * `agent_context`) use it to better recognize short or contextual answers
   * ("yes", "7pm", an email spelled aloud). Providers that don't support it
   * simply omit this method — it is a no-op for them.
   */
  updateAgentContext?(text: string): void;
  /**
   * Close the session and release resources.
   * Called at end_call or disconnect — not on interrupt.
   */
  close(): void;
}
/**
 * Pluggable audio input source for VoiceClient.
 *
 * When provided via `VoiceClientOptions.audioInput`, VoiceClient delegates
 * mic capture to this object instead of using its built-in AudioWorklet.
 * The audio input is responsible for capturing audio and routing it to the
 * server (however it chooses — WebRTC, SFU, direct binary, etc.).
 *
 * It must call `onAudioLevel` with RMS values so VoiceClient can run
 * silence detection, interrupt detection, and update the audio level UI.
 *
 * @example
 * ```typescript
 * class SFUAudioInput implements VoiceAudioInput {
 *   onAudioLevel: ((rms: number) => void) | null = null;
 *   async start() {
 *     // Set up WebRTC peer connection, SFU session, etc.
 *     // In a monitoring loop, call this.onAudioLevel?.(rms)
 *   }
 *   stop() {
 *     // Tear down WebRTC
 *   }
 * }
 * ```
 */
interface VoiceAudioInput {
  /** Start capturing audio. Called by VoiceClient on startCall(). */
  start(): Promise<void>;
  /** Stop capturing audio. Called by VoiceClient on endCall() or disconnect(). */
  stop(): void;
  /**
   * Set by VoiceClient before start(). The audio input must call this
   * with RMS audio level values on each frame so VoiceClient can run
   * silence detection, interrupt detection, and update the UI.
   */
  onAudioLevel: ((rms: number) => void) | null;
  /**
   * Set by VoiceClient before start(). If the audio input provides
   * raw PCM audio (16kHz mono 16-bit LE), call this callback and
   * VoiceClient will forward the data to the server via its transport.
   *
   * This is needed when audio reaches the server through the same
   * WebSocket as protocol messages (e.g. SFU in local dev where the
   * SFU adapter can't connect back to localhost).
   *
   * If the audio input routes audio to the server through an external
   * path (e.g. SFU WebSocket adapter in production), this can be left
   * unused — the audio will arrive on a separate connection.
   */
  onAudioData?: ((pcm: ArrayBuffer) => void) | null;
}
/** Details a transport can provide when its connection closes. */
interface VoiceTransportCloseInfo {
  code?: number;
  reason?: string;
  wasClean?: boolean;
}
/**
 * Abstraction over the data channel between client and server.
 * The default implementation wraps PartySocket (WebSocket).
 * Implement this interface to use WebRTC, SFU, or other transports.
 */
interface VoiceTransport {
  /** Send a JSON-serializable message to the server. */
  sendJSON(data: Record<string, unknown>): void;
  /** Send raw binary audio to the server. */
  sendBinary(data: ArrayBuffer): void;
  /** Open the connection. */
  connect(): void;
  /** Close the connection and release resources. */
  disconnect(): void;
  /** Whether the transport is currently connected and ready to send. */
  readonly connected: boolean;
  onopen: (() => void) | null;
  onclose: ((info?: VoiceTransportCloseInfo) => void) | null;
  onerror: ((error?: unknown) => void) | null;
  /** Called when a JSON string message arrives from the server. */
  onmessage: ((data: string | ArrayBuffer | Blob) => void) | null;
}
//#endregion
export {
  VoiceTransport as C,
  VoiceTurnSource as D,
  VoiceTurnOutcome as E,
  VoiceStatus as S,
  VoiceTurnMetrics as T,
  VoiceErrorStage as _,
  TranscriberSessionOptions as a,
  VoiceRole as b,
  VoiceAudioFormat as c,
  VoiceCompletionOutcome as d,
  VoiceCompletionOutcomeCode as f,
  VoiceErrorCode as g,
  VoiceError as h,
  TranscriberSession as i,
  VoiceAudioInput as l,
  VoiceDiagnosticsOptions as m,
  TTSProvider as n,
  TranscriptMessage as o,
  VoiceDiagnosticEvent as p,
  Transcriber as r,
  VOICE_PROTOCOL_VERSION as s,
  StreamingTTSProvider as t,
  VoiceClientMessage as u,
  VoiceModelFinishReason as v,
  VoiceTransportCloseInfo as w,
  VoiceServerMessage as x,
  VoicePipelineMetrics as y
};
//# sourceMappingURL=types-_Faxb570.d.ts.map