@juspay/neurolink/dist/types/livekit.d.ts

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * Types for the LiveKit voice agent integration.
 *
 * The integration uses LiveKit (WebRTC transport, VAD, turn detection,
 * interruption, worker-per-call scaling) as the real-time loop, and NeuroLink
 * as the brain (LLM, tools, memory). These types describe the brain seam, the
 * worker configuration, and the join-token request — all transport-agnostic
 * except where a LiveKit-specific concept is named explicitly.
 *
 * See docs/features/livekit-voice-agent.md.
 */
import type { NeuroLinkEvents, TypedEventEmitter } from "./common.js";
import type { StreamOptions, StreamResult } from "./stream.js";
/**
 * Minimal structural shape of the NeuroLink instance the brain depends on.
 *
 * Declared structurally (rather than importing the `NeuroLink` class) so the
 * brain layer stays decoupled from SDK construction and can be unit-tested with
 * a lightweight stub. The real `NeuroLink` instance satisfies this shape.
 *
 * `getEventEmitter` is optional so lightweight stubs remain valid; the real
 * `NeuroLink` instance provides it, and the data-channel event bridge uses it
 * to forward tool/text/HITL events to the browser.
 */
export type LiveKitNeuroLinkStreamer = {
    stream: (options: StreamOptions) => Promise<StreamResult>;
    getEventEmitter?: () => TypedEventEmitter<NeuroLinkEvents>;
};
/**
 * Configuration for the transport-agnostic voice brain.
 *
 * The brain owns the conversation: it calls `neurolink.stream()` with a stable
 * `conversationId` so NeuroLink's memory layer is the source of truth, and it
 * leaves tool-calling to the NeuroLink instance.
 */
export type LiveKitBrainConfig = {
    /** Configured NeuroLink instance (memory + tools registered on it). */
    neurolink: LiveKitNeuroLinkStreamer;
    /** LLM provider name passed to `stream()` (e.g. "bedrock"). */
    provider?: string;
    /** LLM model name passed to `stream()` (e.g. "claude-sonnet-4-6"). */
    model?: string;
    /** System prompt applied to every turn. */
    systemPrompt?: string;
    /** Sampling temperature for spoken-style responses. */
    temperature?: number;
    /** Upper bound on tokens per turn. */
    maxTokens?: number;
    /** Optional user identifier recorded alongside memory. */
    userId?: string;
};
/** A single user turn handed to the brain. */
export type LiveKitBrainTurn = {
    /** Final transcript of the user's utterance. */
    transcript: string;
    /** Stable conversation id keying NeuroLink memory for this session. */
    conversationId: string;
    /** Cancellation signal; aborting stops the in-flight LLM and tool calls. */
    signal?: AbortSignal;
};
/**
 * The brain's public surface: stream the assistant reply as text deltas.
 * The transport layer converts these deltas into audio (TTS).
 */
export type LiveKitVoiceBrain = {
    streamReply: (turn: LiveKitBrainTurn) => AsyncGenerator<string, void, unknown>;
};
/** Speech-to-text plugin selection for the LiveKit worker. */
export type LiveKitSttConfig = {
    provider: string;
    model?: string;
    language?: string;
    /**
     * Soniox only: maximum delay (ms) between speech cessation and the STT
     * endpoint. Raise it so Soniox does not finalize on short pauses — that lets
     * VAD silence (not the STT endpoint) decide when the turn ends.
     */
    maxEndpointDelayMs?: number;
};
/** Text-to-speech plugin selection for the LiveKit worker. */
export type LiveKitTtsConfig = {
    provider: string;
    voice?: string;
    model?: string;
};
/**
 * Silero VAD tuning. Stricter values reject background noise (higher threshold,
 * longer minimum speech). Durations are in seconds.
 */
export type LiveKitVadConfig = {
    /** Probability cutoff for "this is speech" (default 0.6). Higher = stricter. */
    activationThreshold?: number;
    /** Minimum speech length before a turn starts, seconds (default 0.2). */
    minSpeechDuration?: number;
    /** Silence before a turn ends, seconds (default 0.6) — tolerates pauses. */
    minSilenceDuration?: number;
};
/**
 * Turn-detection (end-of-utterance) tuning.
 *
 * `mode` selects what decides the user's turn is over:
 * - `"vad"`  — Silero VAD silence (see `LiveKitVadConfig.minSilenceDuration`).
 *   Tolerates natural mid-sentence pauses; the turn only ends after a full
 *   silence window. This mirrors Clairvoyance's behavior.
 * - `"stt"`  — the STT provider's own endpoint detection (e.g. Soniox). Often
 *   much faster/aggressive — short pauses can prematurely split one utterance
 *   into several turns.
 * - `"realtime_llm"` / `"manual"` — advanced/manual strategies.
 *
 * `minEndpointingDelay` / `maxEndpointingDelay` are the framework's endpointing
 * window in milliseconds (in VAD mode the effective end delay is
 * `max(VAD silence, minEndpointingDelay)`).
 */
export type LiveKitTurnConfig = {
    mode?: "stt" | "vad" | "realtime_llm" | "manual";
    minEndpointingDelay?: number;
    maxEndpointingDelay?: number;
};
/**
 * Interruption (barge-in) tuning. Requiring real words / a minimum duration
 * stops background noise from cutting off the assistant.
 */
export type LiveKitInterruptionConfig = {
    /** Minimum recognized words to count as an interruption (default 2). */
    minWords?: number;
    /** Minimum audio duration to count as an interruption, ms (default 600). */
    minDuration?: number;
};
/**
 * Options for `defineVoiceAgent` — the agent definition placed as the default
 * export of the worker entry file.
 *
 * LiveKit runs each call as a Job in its own child process and re-imports the
 * entry file there, so the NeuroLink instance cannot be passed as a live object
 * from a parent. Instead, `createNeuroLink` is invoked **inside each job
 * process** to build the brain (and register its tools) for that call.
 */
export type LiveKitVoiceAgentConfig = {
    /**
     * Factory that builds the NeuroLink instance for a job process.
     * Called once per call, inside the job's own process.
     */
    createNeuroLink: () => LiveKitNeuroLinkStreamer | Promise<LiveKitNeuroLinkStreamer>;
    /** Realtime speech-to-text selection. */
    stt: LiveKitSttConfig;
    /** Realtime text-to-speech selection. */
    tts: LiveKitTtsConfig;
    /** LLM provider/model overrides (default to env-resolved values). */
    provider?: string;
    model?: string;
    systemPrompt?: string;
    temperature?: number;
    maxTokens?: number;
    /** Prefix used when deriving a per-room conversation id (default "voice"). */
    conversationIdPrefix?: string;
    /** Optional user id recorded alongside memory. */
    userId?: string;
    greeting?: string;
    /** Silero VAD tuning (stricter = ignores background noise). */
    vad?: LiveKitVadConfig;
    /** Turn-detection tuning (VAD vs STT endpointing, delays). */
    turn?: LiveKitTurnConfig;
    /** Interruption tuning (require words/duration so noise can't barge in). */
    interruption?: LiveKitInterruptionConfig;
    /**
     * Data-channel event bridge: forward NeuroLink events (text, tool calls,
     * tool results, HITL prompts, status) to the browser over the LiveKit data
     * channel, and accept control messages (HITL responses) back. Disabled
     * unless `enabled` is `true`.
     */
    events?: LiveKitEventBridgeConfig;
};
/** Options for `startVoiceAgentWorker` — launches the LiveKit Agents worker. */
export type LiveKitWorkerLaunchOptions = {
    /**
     * Absolute path to the entry file whose default export is the result of
     * `defineVoiceAgent`. LiveKit re-imports this file in each job process.
     */
    agentFile: string;
    /** Name the worker registers under for dispatch (default "neurolink-voice"). */
    agentName?: string;
};
/** Resolved LiveKit server connection settings. */
export type LiveKitServerConfig = {
    /** LiveKit server URL (ws/wss). */
    url: string;
    /** API key for token signing and worker registration. */
    apiKey: string;
    /** API secret for token signing and worker registration. */
    apiSecret: string;
};
/** LLM defaults resolved from the environment for the brain. */
export type LiveKitBrainDefaults = {
    provider: string;
    model: string;
};
/** Arguments for minting a browser join token. */
export type LiveKitTokenRequest = {
    /** Participant identity (e.g. the authenticated user id). */
    identity: string;
    /** Room name to join (auto-created on first join). */
    room: string;
    /** LiveKit API key. */
    apiKey: string;
    /** LiveKit API secret. */
    apiSecret: string;
    /** Token lifetime in seconds (default 600; clamped to a 3600 max). */
    ttlSeconds?: number;
};
/** Discriminant tags for outbound voice events. */
export type LiveKitVoiceEventType = "user-text" | "text" | "tool-start" | "tool-result" | "status" | "hitl-prompt" | "done";
/**
 * A user STT transcript for display. Interim partials stream with
 * `final: false`; the end-of-utterance result has `final: true`. The client
 * updates one live bubble and commits it on `final`.
 *
 * `replacesPrevious` is set on the committed (`final: true`) text of a turn that
 * absorbed a previous turn the user interrupted before it produced any reply
 * (strict barge-in club). The client removes the orphaned previous user bubble
 * so the merged utterance shows as one bubble.
 */
export type LiveKitVoiceUserTextEvent = {
    type: "user-text";
    data: {
        text: string;
        final: boolean;
        replacesPrevious?: boolean;
    };
};
/** A streamed chunk of the assistant's spoken/written reply. */
export type LiveKitVoiceTextEvent = {
    type: "text";
    data: {
        delta: string;
    };
};
/** A tool invocation has started (best-effort status; may not always fire). */
export type LiveKitVoiceToolStartEvent = {
    type: "tool-start";
    data: {
        id?: string;
        name: string;
        input?: unknown;
    };
};
/**
 * A tool invocation has finished. `result` carries the tool's structured
 * output (for example, a chart payload) for the client to render.
 */
export type LiveKitVoiceToolResultEvent = {
    type: "tool-result";
    data: {
        id?: string;
        name: string;
        result?: unknown;
        success?: boolean;
        error?: string;
    };
};
/** Coarse agent state, useful for UI indicators (e.g. "thinking…"). */
export type LiveKitVoiceStatusEvent = {
    type: "status";
    data: {
        state: "thinking" | "speaking" | "listening" | "error";
        detail?: string;
    };
};
/** A human-in-the-loop confirmation the user must approve or reject. */
export type LiveKitVoiceHitlPromptEvent = {
    type: "hitl-prompt";
    data: {
        confirmationId: string;
        toolName: string;
        actionType?: string;
        arguments?: unknown;
        timeoutMs?: number;
        allowModification?: boolean;
    };
};
/** The current turn has finished. */
export type LiveKitVoiceDoneEvent = {
    type: "done";
    data: {
        reason?: string;
    };
};
/** Discriminated union of all outbound voice events (before enveloping). */
export type LiveKitVoiceEvent = LiveKitVoiceUserTextEvent | LiveKitVoiceTextEvent | LiveKitVoiceToolStartEvent | LiveKitVoiceToolResultEvent | LiveKitVoiceStatusEvent | LiveKitVoiceHitlPromptEvent | LiveKitVoiceDoneEvent;
/**
 * Wire format published to the browser: a `LiveKitVoiceEvent` plus a monotonic
 * sequence number and a timestamp so the client can order and de-duplicate.
 */
export type LiveKitVoiceEventEnvelope = LiveKitVoiceEvent & {
    seq: number;
    ts: number;
};
/** Control messages sent from the browser back to the agent. */
export type LiveKitVoiceControlMessage = {
    action: "hitl:accept";
    confirmationId: string;
    modifiedArguments?: unknown;
} | {
    action: "hitl:reject";
    confirmationId: string;
    reason?: string;
};
/**
 * Configuration for the data-channel event bridge, set on
 * `LiveKitVoiceAgentConfig.events`.
 */
export type LiveKitEventBridgeConfig = {
    /** Master switch — the bridge is inert unless this is `true`. */
    enabled?: boolean;
    /** Data-channel topic for outbound events (default "ai-events"). */
    eventsTopic?: string;
    /** Data-channel topic for inbound control messages (default "ai-control"). */
    controlTopic?: string;
    /** If set, only these event types are forwarded (default: all). */
    include?: LiveKitVoiceEventType[];
    /**
     * Payloads encoded larger than this many bytes are sent via the chunked text
     * stream API instead of a single reliable data packet (default 12000).
     */
    maxInlineBytes?: number;
};
/**
 * Minimal structural view of the LiveKit room the bridge needs: a local
 * participant to publish on, and event (un)subscription. Declared structurally
 * so `src/lib/types` carries no dependency on `@livekit/rtc-node`; the real
 * `Room` from a job context satisfies this shape.
 */
export type LiveKitBridgeRoom = {
    localParticipant?: {
        publishData(data: Uint8Array, options: {
            reliable?: boolean;
            topic?: string;
        }): Promise<void>;
        sendText(text: string, options?: {
            topic?: string;
        }): Promise<unknown>;
    };
    on(event: string, listener: (...args: unknown[]) => void): unknown;
    off(event: string, listener: (...args: unknown[]) => void): unknown;
};
/**
 * Normalized tool fields extracted from a `tool:start` / `tool:end` emitter
 * payload, used internally by the event bridge.
 */
export type LiveKitToolEventFields = {
    name: string;
    id?: string;
    input?: unknown;
    result?: unknown;
    success?: boolean;
    error?: string;
};
/** Inputs to `attachEventBridge`. */
export type LiveKitEventBridgeParams = {
    /** The LiveKit room for this call (from the job context). */
    room: LiveKitBridgeRoom;
    /** NeuroLink's event emitter (`neurolink.getEventEmitter()`). */
    emitter: TypedEventEmitter<NeuroLinkEvents>;
    /** Bridge options (topics, filtering, chunking threshold). */
    options?: LiveKitEventBridgeConfig;
};
/** Handle returned by `attachEventBridge` for teardown. */
export type LiveKitEventBridgeHandle = {
    /** Remove all listeners and stop publishing. Idempotent. */
    dispose: () => void;
};
/**
 * Credentials for LiveKit server-side REST calls (room create, agent dispatch).
 * `url` accepts `ws(s)://` or `http(s)://`; helpers convert it to https.
 */
export type LiveKitServerCredentials = {
    url: string;
    apiKey: string;
    apiSecret: string;
};
/**
 * Realtime voice configuration resolved from the environment.
 *
 * In speech-to-speech mode one realtime model (Gemini Live on Vertex) does STT,
 * reasoning, TTS, and turn detection — so there is no separate STT/TTS/VAD/EOU
 * config. `resolveRealtimeVoiceConfig` fills every field from `process.env`
 * (with defaults); `RealtimeVoiceAgentConfig` lets a caller override any of them.
 */
export type RealtimeVoiceConfig = {
    /** Vertex project id (from VERTEX_PROJECT / GOOGLE_AUTH_* / GOOGLE_CLOUD_PROJECT_ID). */
    project: string | undefined;
    /** Vertex location; native-audio Live is served on `global`, not regionally. */
    location: string;
    /** Realtime model id (e.g. "gemini-live-2.5-flash"). */
    model: string;
    /** Optional Gemini voice name; omit for the plugin default. */
    voice: string | undefined;
    /** Response modality: "AUDIO" (native S2S) or "TEXT" (half-cascade). */
    responseModality: string;
    /** System prompt / instructions for the agent. */
    systemPrompt: string;
    /** Opening line the agent speaks on connect ("" disables). */
    greeting: string;
    /** Whether to bridge Lighthouse MCP tools as Gemini function tools. */
    toolsEnabled: boolean;
    /** Full URL of the MCP server the tools are bridged from. */
    mcpUrl: string;
    /** Grace period after the caller leaves before the job shuts down (ms). */
    emptyRoomGraceMs: number;
    /** Deadline for a participant to join before the job shuts down (ms). */
    joinDeadlineMs: number;
    /** How long a HITL confirmation waits before being treated as a decline (ms). */
    hitlTimeoutMs: number;
    /** Interval for the RSS/heap metrics log (ms). */
    metricsIntervalMs: number;
};
/** A single log record handed to a `RealtimeVoiceAgentConfig.onLog` sink. */
export type RealtimeVoiceLogEntry = {
    level: "debug" | "info" | "warn" | "error";
    message: string;
    timestamp: number;
    data?: unknown;
};
export type RealtimeVoiceLogContext = {
    room: string;
};
/**
 * Options for `defineRealtimeVoiceAgent`. Every field is optional: omitted
 * values fall back to `resolveRealtimeVoiceConfig()` (i.e. the environment), so
 * a caller can use `defineRealtimeVoiceAgent()` with no arguments and configure
 * everything via env.
 */
export type RealtimeVoiceAgentConfig = {
    project?: string;
    location?: string;
    model?: string;
    voice?: string;
    responseModality?: string;
    systemPrompt?: string;
    greeting?: string;
    /** MCP tool bridging overrides. */
    tools?: {
        enabled?: boolean;
        mcpUrl?: string;
    };
    /** Data-channel topic for outbound events (default "ai-events"). */
    eventsTopic?: string;
    /** Data-channel topic for inbound control messages (default "ai-control"). */
    controlTopic?: string;
    /**
     * Optional sink for the agent's own logs. When set, the realtime agent wires
     * NeuroLink's logger to this callback for the duration of the call, so a host
     * can forward worker logs into its logging pipeline. Each record is tagged
     * with per-call context (the room name). Subject to the logger's level gate:
     * without debug mode only `error` records are emitted (set `NEUROLINK_DEBUG`).
     */
    onLog?: (entry: RealtimeVoiceLogEntry, ctx: RealtimeVoiceLogContext) => void;
};
/** Auth token + base64 MCP execution context decoded from a room's metadata. */
export type LiveKitRoomCallContext = {
    /** Lighthouse access JWT used as `x-auth-token` to the MCP server. */
    authToken: string;
    /** base64(JSON) MCP execution context used as `x-context` (or "" if absent). */
    xContext: string;
};
/** Publishes a single voice event envelope onto the room data channel. */
export type RealtimeEventPublisher = (type: LiveKitVoiceEventType, data: Record<string, unknown>) => void;
/** Requests a HITL confirmation and resolves to the user's decision. */
export type RealtimeConfirmationRequester = (toolName: string, args: Record<string, unknown>) => Promise<boolean>;
/** Handle returned by `attachRealtimeEventBridge`. */
export type RealtimeEventBridgeHandle = {
    /** Publish an outbound event to the browser (data packet or text stream). */
    publishEvent: RealtimeEventPublisher;
    /** Open a HITL prompt and await the browser's decision (timeout = decline). */
    requestConfirmation: RealtimeConfirmationRequester;
    /** Remove the control-channel listener and clear pending confirmations. */
    dispose: () => void;
};
/** Inputs to `attachRealtimeEventBridge`. */
export type RealtimeEventBridgeParams = {
    /** The LiveKit room for this call (from the job context). */
    room: LiveKitBridgeRoom;
    /** HITL confirmation timeout in ms before a request is auto-declined. */
    hitlTimeoutMs?: number;
    /** Outbound events topic (default "ai-events"). */
    eventsTopic?: string;
    /** Inbound control topic (default "ai-control"). */
    controlTopic?: string;
    /** Payloads larger than this are sent via the chunked text stream (default 12000). */
    maxInlineBytes?: number;
};
/** Inputs to `buildRealtimeMcpTools`. */
export type BuildRealtimeMcpToolsParams = {
    /** Full URL of the MCP server (e.g. ".../ai/mcp/v2"). */
    mcpUrl: string;
    /** Lighthouse access JWT forwarded as `x-auth-token`. */
    authToken: string;
    /** base64(JSON) execution context forwarded as `x-context`. */
    xContext: string;
    /** Publishes tool start/result events to the browser. */
    publishEvent: RealtimeEventPublisher;
    /** Opens a HITL confirmation for destructive tools and awaits the decision. */
    requestConfirmation: RealtimeConfirmationRequester;
    /**
     * Hard cap per MCP tool call, in milliseconds (default 30000).
     *
     * Without one, a stalled MCP server holds the realtime turn open forever:
     * Gemini waits on the function result, so the user gets silence rather than
     * an error. Bounding the call turns that into a normal tool failure the
     * model can talk about.
     */
    toolTimeoutMs?: number;
};