@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
425 lines (424 loc) • 17 kB
JavaScript
/**
* LiveKit Agents agent definition.
*
* `defineVoiceAgent` returns the agent object placed as the default export of a
* worker entry file. The framework runs it as a Job (one per call, in its own
* process): it connects to the room, builds the NeuroLink brain via the
* supplied factory, wires Silero VAD + STT/TTS plugins, and overrides `llmNode`
* so every turn is generated by `neurolink.stream()`.
*
* `@livekit/agents` and the plugins are optional dependencies, imported
* dynamically so the core package does not require them unless the LiveKit
* voice agent is used. Type-only imports are erased at build time and add no
* runtime dependency.
*
* See docs/features/livekit-voice-agent.md.
*/
import { ReadableStream } from "node:stream/web";
import { logger } from "../../utils/logger.js";
import { createVoiceBrain } from "./brain.js";
import { resolveBrainDefaults, resolveEouTurnDetection } from "./config.js";
import { attachEventBridge } from "./eventBridge.js";
const DEFAULT_CONVERSATION_PREFIX = "voice";
// Turn-end timing defaults (approach A: silence tuning). Longer silence +
// endpointing floor so natural mid-sentence pauses ("...and so, [pause] um")
// don't end the turn early and split one utterance into two. Overridable via
// config (vad.minSilenceDuration / turn.minEndpointingDelay).
const DEFAULT_MIN_SILENCE_DURATION = 1.0; // seconds (Silero VAD)
const DEFAULT_MIN_ENDPOINTING_DELAY = 1000; // ms (framework endpointing floor)
/**
* Find the most recent user utterance in a chat context.
* Uses the `type === "message"` discriminant — no type assertions.
*/
function latestUserText(chatCtx) {
const items = chatCtx.items;
for (let i = items.length - 1; i >= 0; i -= 1) {
const item = items[i];
if (item.type === "message" && item.role === "user") {
return item.textContent;
}
}
return undefined;
}
/**
* Build a text stream for a single turn from the brain, abortable on cancel.
* When the framework cancels the stream (barge-in), the brain's turn is aborted.
*/
function brainTurnStream(brain, transcript, conversationId, onAbortedBeforeOutput) {
const controller = new AbortController();
const generator = brain.streamReply({
transcript,
conversationId,
signal: controller.signal,
});
const iterator = generator[Symbol.asyncIterator]();
let producedOutput = false;
return new ReadableStream({
async pull(streamController) {
const next = await iterator.next();
if (next.done === true) {
streamController.close();
return;
}
producedOutput = true;
streamController.enqueue(next.value);
},
cancel() {
controller.abort();
if (!producedOutput) {
onAbortedBeforeOutput?.();
}
},
});
}
/**
* Construct the English semantic EOU turn detector, or `undefined` if disabled.
*
* Layered on top of VAD: VAD detects acoustic silence, then this model decides
* whether the user's turn is semantically complete, so natural mid-sentence
* pauses don't split one utterance. Opt-in via `LIVEKIT_EOU_TURN_DETECTION`.
* The runner is registered in the worker process (see `voiceAgentWorker.ts`);
* here we only construct the model handle, which dispatches inference to the
* shared executor via the running job context.
*/
async function loadEouTurnDetector() {
const { enabled, unlikelyThreshold } = resolveEouTurnDetection();
if (!enabled) {
return undefined;
}
const { turnDetector } = await import("@livekit/agents-plugin-livekit");
return new turnDetector.EnglishModel(unlikelyThreshold);
}
/**
* Construct the Silero VAD instance for the session.
*
* Stricter-than-default thresholds so background noise isn't treated as speech
* (a higher activation threshold and a minimum speech duration reject short,
* quiet noise bursts).
*/
async function loadVad(config) {
const silero = await import("@livekit/agents-plugin-silero");
return silero.VAD.load({
activationThreshold: config?.activationThreshold ?? 0.6,
minSpeechDuration: config?.minSpeechDuration ?? 0.2,
minSilenceDuration: config?.minSilenceDuration ?? DEFAULT_MIN_SILENCE_DURATION,
});
}
/**
* Construct the STT plugin instance from configuration.
*
* Only defined options are passed — passing `undefined` would override the
* plugin's own defaults (e.g. its default model) with `undefined` and break it.
*/
async function buildStt(config) {
if (config.provider === "soniox") {
const soniox = await import("@livekit/agents-plugin-soniox");
const opts = {};
if (config.model !== undefined) {
opts.model = config.model;
}
if (config.language !== undefined) {
// Soft hint only: Soniox biases toward this language but can still
// auto-detect another (e.g. the user switching to Telugu mid-call).
// Do NOT set `languageHintsStrict` — forcing the hinted language makes
// the realtime STT stall/error on other-language audio and the session
// never recovers (no further transcripts, so no audio at all).
opts.languageHints = [config.language];
}
if (config.maxEndpointDelayMs !== undefined) {
opts.maxEndpointDelayMs = config.maxEndpointDelayMs;
}
return new soniox.STT(opts);
}
if (config.provider === "deepgram") {
const deepgram = await import("@livekit/agents-plugin-deepgram");
const opts = {};
if (config.language !== undefined) {
opts.language = config.language;
}
return new deepgram.STT(opts);
}
throw new Error(`Unsupported LiveKit STT provider "${config.provider}" (supported: soniox, deepgram)`);
}
/**
* Construct the TTS plugin instance from configuration.
*
* Only defined options are passed — passing `undefined` would override the
* plugin's own defaults (default voice/model) with `undefined` and break it.
*/
async function buildTts(config) {
if (config.provider === "cartesia") {
const cartesia = await import("@livekit/agents-plugin-cartesia");
const opts = {};
if (config.voice !== undefined) {
opts.voice = config.voice;
}
if (config.model !== undefined) {
opts.model = config.model;
}
return new cartesia.TTS(opts);
}
if (config.provider === "elevenlabs") {
const elevenlabs = await import("@livekit/agents-plugin-elevenlabs");
const opts = {};
if (config.voice !== undefined) {
opts.voiceId = config.voice;
}
if (config.model !== undefined) {
opts.modelID = config.model;
}
return new elevenlabs.TTS(opts);
}
throw new Error(`Unsupported LiveKit TTS provider "${config.provider}" (supported: cartesia, elevenlabs)`);
}
/**
* Define a LiveKit voice agent backed by NeuroLink.
*
* Place the result as the default export of the worker entry file:
*
* ```ts
* export default defineVoiceAgent({
* createNeuroLink: async () => buildConfiguredNeuroLink(),
* stt: { provider: "deepgram" },
* tts: { provider: "elevenlabs" },
* });
* ```
*/
export function defineVoiceAgent(config) {
const defaults = resolveBrainDefaults();
const provider = config.provider ?? defaults.provider;
const model = config.model ?? defaults.model;
const conversationPrefix = config.conversationIdPrefix ?? DEFAULT_CONVERSATION_PREFIX;
async function entry(ctx) {
const entryStartedAt = Date.now();
await ctx.connect();
logger.debug("voice.agent.roomJoined", {
room: ctx.room.name,
ms: Date.now() - entryStartedAt,
});
const { RoomEvent } = await import("@livekit/rtc-node");
ctx.room.on(RoomEvent.ParticipantDisconnected, () => {
if (ctx.room.remoteParticipants.size === 0) {
logger.info("voice.agent.participantLeft", {
room: ctx.room.name,
action: "shutdown",
});
ctx.shutdown("participant left");
}
});
let userStoppedSpeakingAt;
const neurolink = await config.createNeuroLink();
const brain = createVoiceBrain({
neurolink,
provider,
model,
systemPrompt: config.systemPrompt,
temperature: config.temperature,
maxTokens: config.maxTokens,
userId: config.userId,
});
const conversationId = `${conversationPrefix}-${ctx.room.name ?? ctx.job.id}`;
const { voice, llm } = await import("@livekit/agents");
const [vad, stt, tts, eouTurnDetector] = await Promise.all([
loadVad(config.vad),
buildStt(config.stt),
buildTts(config.tts),
loadEouTurnDetector(),
]);
const transcriptEventsEnabled = config.events?.enabled === true &&
typeof neurolink.getEventEmitter === "function";
const transcriptEmitter = transcriptEventsEnabled
? neurolink.getEventEmitter?.()
: undefined;
let userTranscriptBuffer = "";
let pendingPrefix = "";
function emitUserTranscriptSegment(segmentText, isFinal) {
if (transcriptEmitter === undefined) {
return;
}
const trimmed = segmentText.trim();
if (isFinal) {
userTranscriptBuffer =
userTranscriptBuffer.length > 0
? `${userTranscriptBuffer} ${trimmed}`
: trimmed;
transcriptEmitter.emit("voice:user-transcript", {
text: userTranscriptBuffer,
final: false,
});
return;
}
const live = userTranscriptBuffer.length > 0
? `${userTranscriptBuffer} ${trimmed}`
: trimmed;
transcriptEmitter.emit("voice:user-transcript", {
text: live,
final: false,
});
}
function commitUserTranscript(finalText, replacesPrevious = false) {
if (transcriptEmitter !== undefined) {
transcriptEmitter.emit("voice:user-transcript", {
text: finalText,
final: true,
replacesPrevious,
});
}
userTranscriptBuffer = "";
}
class NeuroLinkVoiceAgent extends voice.Agent {
async llmNode(chatCtx, _toolCtx, _modelSettings) {
const transcript = latestUserText(chatCtx);
if (transcript === undefined || transcript.trim().length === 0) {
userTranscriptBuffer = "";
return null;
}
const hadPrefix = pendingPrefix.length > 0;
const promptText = hadPrefix
? `${pendingPrefix} ${transcript}`
: transcript;
pendingPrefix = "";
commitUserTranscript(promptText, hadPrefix);
if (userStoppedSpeakingAt !== undefined) {
logger.debug("voice.agent.endpointingWaited", {
ms: Date.now() - userStoppedSpeakingAt,
});
}
return brainTurnStream(brain, promptText, conversationId, () => {
// Interrupted before producing any reply → carry this turn's text
// forward; the next turn merges it (prompt + UI).
pendingPrefix = promptText;
});
}
}
class PlaceholderLLM extends llm.LLM {
label() {
return "neurolink-placeholder";
}
chat() {
throw new Error("PlaceholderLLM.chat must not be called — llmNode overrides generation");
}
}
const turnHandling = {
interruption: {
minWords: config.interruption?.minWords ?? 2,
minDuration: config.interruption?.minDuration ?? 600,
},
};
if (eouTurnDetector !== undefined) {
turnHandling.turnDetection = eouTurnDetector;
logger.info("voice.agent.eouEnabled", { language: "english" });
}
else if (config.turn?.mode) {
turnHandling.turnDetection = config.turn.mode;
}
const endpointing = {};
endpointing.minDelay =
config.turn?.minEndpointingDelay ?? DEFAULT_MIN_ENDPOINTING_DELAY;
if (config.turn?.maxEndpointingDelay !== undefined) {
endpointing.maxDelay = config.turn.maxEndpointingDelay;
}
if (Object.keys(endpointing).length > 0) {
turnHandling.endpointing = endpointing;
}
const session = new voice.AgentSession({
vad,
stt,
tts,
llm: new PlaceholderLLM(),
turnHandling,
preemptiveGeneration: false,
});
const agent = new NeuroLinkVoiceAgent({
instructions: config.systemPrompt ?? "",
});
const inactivityTimeoutMs = Number(process.env.VOICE_INACTIVITY_TIMEOUT_MS ?? 600_000);
const inactivityEnabled = Number.isFinite(inactivityTimeoutMs) && inactivityTimeoutMs > 0;
let inactivityTimer;
let inactivityFired = false;
function clearInactivityTimer() {
if (inactivityTimer !== undefined) {
clearTimeout(inactivityTimer);
inactivityTimer = undefined;
}
}
function noteActivity() {
if (!inactivityEnabled || inactivityFired) {
return;
}
clearInactivityTimer();
inactivityTimer = setTimeout(() => {
inactivityFired = true;
logger.info("voice.agent.inactivityTimeout", {
room: ctx.room.name,
timeoutMs: inactivityTimeoutMs,
action: "shutdown",
});
ctx.shutdown("inactivity timeout");
}, inactivityTimeoutMs);
// The watchdog must not, by itself, keep the event loop alive.
inactivityTimer.unref?.();
}
ctx.addShutdownCallback(async () => {
clearInactivityTimer();
});
if (process.env.LK_REALTIME_CONNECT_MODE === "true") {
ctx.addShutdownCallback(async () => {
const parentPid = process.ppid;
setTimeout(() => {
try {
if (typeof parentPid === "number" && parentPid > 1) {
process.kill(parentPid, "SIGTERM");
}
}
catch {
// Parent already gone — fall through to the hard exit below.
}
process.exit(0);
}, 500).unref?.();
});
}
session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => {
noteActivity();
if (ev.oldState === "speaking" && ev.newState !== "speaking") {
userStoppedSpeakingAt = Date.now();
}
});
session.on(voice.AgentSessionEventTypes.AgentStateChanged, () => {
noteActivity();
});
session.on(voice.AgentSessionEventTypes.ConversationItemAdded, () => {
noteActivity();
});
if (transcriptEventsEnabled) {
session.on(voice.AgentSessionEventTypes.UserInputTranscribed, (ev) => {
emitUserTranscriptSegment(ev.transcript, ev.isFinal);
});
}
logger.info("voice.agent.sessionStarting", {
room: ctx.room.name,
provider,
model,
});
await session.start({ agent, room: ctx.room });
if (config.greeting !== undefined && config.greeting.trim().length > 0) {
const greetingStream = brainTurnStream(brain, config.greeting, conversationId);
session.say(greetingStream, {
addToChatCtx: true,
allowInterruptions: true,
});
}
noteActivity();
if (config.events?.enabled === true && neurolink.getEventEmitter) {
const bridge = await attachEventBridge({
room: ctx.room,
emitter: neurolink.getEventEmitter(),
options: config.events,
});
ctx.addShutdownCallback(async () => {
bridge.dispose();
});
}
}
return { entry };
}