@juspay/neurolink
Version:
Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applicatio
439 lines • 14.4 kB
JavaScript
/**
* Realtime Voice API Infrastructure
*
* Base handler and processor for realtime voice communication.
* Supports bidirectional audio streaming with providers like OpenAI and Gemini.
*
* @module voice/RealtimeVoiceAPI
*/
import { logger } from "../utils/logger.js";
import { RealtimeError } from "./errors.js";
import { DEFAULT_REALTIME_CONFIG, REALTIME_ERROR_CODES, } from "../types/index.js";
import { ErrorCategory, ErrorSeverity } from "../constants/enums.js";
/**
* Realtime Processor class for orchestrating realtime voice operations
*
* Provides a unified interface for realtime voice across multiple providers.
*
* @example
* ```typescript
* // Register a handler (typically done in providerRegistry.ts on startup)
* RealtimeProcessor.registerHandler('openai-realtime', openaiHandler);
*
* // Connect to a session — the first arg is the registered handler key,
* // and `config.provider` must match the same key.
* const session = await RealtimeProcessor.connect('openai-realtime', {
* provider: 'openai-realtime',
* voice: 'alloy',
* systemPrompt: 'You are a helpful assistant.'
* });
*
* // Send audio
* await RealtimeProcessor.sendAudio('openai-realtime', audioBuffer);
*
* // Disconnect
* await RealtimeProcessor.disconnect('openai-realtime');
* ```
*/
export class RealtimeProcessor {
/**
* Handler registry mapping provider names to Realtime handlers
*/
static handlers = new Map();
/**
* Active sessions by provider
*/
static sessions = new Map();
/**
* Register a Realtime handler for a specific provider
*
* @param providerName - Provider identifier (e.g., 'openai', 'gemini')
* @param handler - Realtime handler implementation
*/
static registerHandler(providerName, handler) {
if (!providerName) {
throw new Error("Provider name is required");
}
if (!handler) {
throw new Error("Handler is required");
}
const normalizedName = providerName.toLowerCase();
if (this.handlers.has(normalizedName)) {
logger.warn(`[RealtimeProcessor] Overwriting existing handler for provider: ${normalizedName}`);
}
this.handlers.set(normalizedName, handler);
logger.debug(`[RealtimeProcessor] Registered Realtime handler for provider: ${normalizedName}`);
}
/**
* Get a registered Realtime handler by provider name
*/
static getHandler(providerName) {
const normalizedName = providerName.toLowerCase();
return this.handlers.get(normalizedName);
}
/**
* Check if a provider is supported
*/
static supports(providerName) {
if (!providerName) {
return false;
}
const normalizedName = providerName.toLowerCase();
return this.handlers.has(normalizedName);
}
/**
* Get list of all registered providers
*/
static getProviders() {
return Array.from(this.handlers.keys());
}
/**
* Connect to a realtime session
*
* @param provider - Provider identifier
* @param config - Session configuration
* @param handlers - Event handlers
* @returns Session information
*/
static async connect(provider, config, handlers) {
const handler = this.getHandler(provider);
if (!handler) {
throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
}
if (!handler.isConfigured()) {
throw RealtimeError.providerNotConfigured(provider);
}
// Check for existing session
if (handler.isConnected()) {
throw RealtimeError.sessionAlreadyActive(provider);
}
// Merge with defaults
const mergedConfig = {
...DEFAULT_REALTIME_CONFIG,
...config,
};
// Register event handlers if provided
if (handlers) {
handler.on(handlers);
}
try {
logger.debug(`[RealtimeProcessor] Connecting to provider: ${provider}`);
const session = await handler.connect(mergedConfig);
this.sessions.set(provider.toLowerCase(), session);
logger.info(`[RealtimeProcessor] Connected to ${provider} session: ${session.id}`);
return session;
}
catch (err) {
if (handlers) {
handler.off();
}
if (err instanceof RealtimeError) {
throw err;
}
const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
throw RealtimeError.connectionFailed(errorMessage, provider, err instanceof Error ? err : undefined);
}
}
/**
* Disconnect from a realtime session
*
* @param provider - Provider identifier
*/
static async disconnect(provider) {
const handler = this.getHandler(provider);
if (!handler) {
throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
}
if (!handler.isConnected()) {
logger.warn(`[RealtimeProcessor] No active session for provider: ${provider}`);
return;
}
try {
await handler.disconnect();
this.sessions.delete(provider.toLowerCase());
handler.off();
logger.info(`[RealtimeProcessor] Disconnected from ${provider}`);
}
catch (err) {
if (err instanceof RealtimeError) {
throw err;
}
const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
throw RealtimeError.protocolError(`Disconnect failed: ${errorMessage}`, provider, err instanceof Error ? err : undefined);
}
}
/**
* Send audio to a realtime session
*
* @param provider - Provider identifier
* @param audio - Audio data
*/
static async sendAudio(provider, audio) {
const handler = this.getHandler(provider);
if (!handler) {
throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
}
if (!handler.isConnected()) {
throw RealtimeError.sessionNotActive(provider);
}
try {
await handler.sendAudio(audio);
}
catch (err) {
if (err instanceof RealtimeError) {
throw err;
}
const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
throw RealtimeError.audioStreamError(errorMessage, provider);
}
}
/**
* Send text to a realtime session
*
* @param provider - Provider identifier
* @param text - Text to send
*/
static async sendText(provider, text) {
const handler = this.getHandler(provider);
if (!handler) {
throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
}
if (!handler.isConnected()) {
throw RealtimeError.sessionNotActive(provider);
}
if (!handler.sendText) {
throw new RealtimeError({
code: REALTIME_ERROR_CODES.PROTOCOL_ERROR,
message: `Provider "${provider}" does not support text input`,
category: ErrorCategory.VALIDATION,
severity: ErrorSeverity.MEDIUM,
context: { provider },
});
}
// Normalize provider exceptions into RealtimeError so callers see a
// consistent error taxonomy across sendAudio/sendText/triggerResponse/
// cancelResponse — previously raw provider errors leaked from the
// text/control paths while sendAudio wrapped them (CodeRabbit review).
try {
await handler.sendText(text);
}
catch (err) {
if (err instanceof RealtimeError) {
throw err;
}
throw new RealtimeError({
code: REALTIME_ERROR_CODES.PROTOCOL_ERROR,
message: `sendText failed: ${err instanceof Error ? err.message : String(err)}`,
category: ErrorCategory.NETWORK,
severity: ErrorSeverity.MEDIUM,
retriable: true,
context: { provider },
originalError: err instanceof Error ? err : undefined,
});
}
}
/**
* Trigger a response from the model (manual turn detection)
*
* @param provider - Provider identifier
*/
static async triggerResponse(provider) {
const handler = this.getHandler(provider);
if (!handler) {
throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
}
if (!handler.isConnected()) {
throw RealtimeError.sessionNotActive(provider);
}
if (handler.triggerResponse) {
try {
await handler.triggerResponse();
}
catch (err) {
if (err instanceof RealtimeError) {
throw err;
}
throw new RealtimeError({
code: REALTIME_ERROR_CODES.PROTOCOL_ERROR,
message: `triggerResponse failed: ${err instanceof Error ? err.message : String(err)}`,
category: ErrorCategory.NETWORK,
severity: ErrorSeverity.MEDIUM,
retriable: true,
context: { provider },
originalError: err instanceof Error ? err : undefined,
});
}
}
}
/**
* Cancel the current response
*
* @param provider - Provider identifier
*/
static async cancelResponse(provider) {
const handler = this.getHandler(provider);
if (!handler) {
throw RealtimeError.providerNotSupported(provider, Array.from(this.handlers.keys()));
}
if (!handler.isConnected()) {
return; // Nothing to cancel
}
if (handler.cancelResponse) {
try {
await handler.cancelResponse();
}
catch (err) {
if (err instanceof RealtimeError) {
throw err;
}
throw new RealtimeError({
code: REALTIME_ERROR_CODES.PROTOCOL_ERROR,
message: `cancelResponse failed: ${err instanceof Error ? err.message : String(err)}`,
category: ErrorCategory.NETWORK,
severity: ErrorSeverity.MEDIUM,
retriable: true,
context: { provider },
originalError: err instanceof Error ? err : undefined,
});
}
}
}
/**
* Get current session for a provider
*
* @param provider - Provider identifier
* @returns Session or null
*/
static getSession(provider) {
const handler = this.getHandler(provider);
return handler?.getSession() ?? null;
}
/**
* Check if a provider has an active session
*
* @param provider - Provider identifier
*/
static isConnected(provider) {
const handler = this.getHandler(provider);
return handler?.isConnected() ?? false;
}
/**
* Get supported formats for a provider
*
* @param provider - Provider identifier
*/
static getSupportedFormats(provider) {
const handler = this.getHandler(provider);
return handler?.getSupportedFormats() ?? [];
}
/**
* Clear all handlers and sessions (for testing)
*/
static clearHandlers() {
// Disconnect all active sessions
for (const [provider] of this.sessions) {
const handler = this.handlers.get(provider);
if (handler?.isConnected()) {
handler.disconnect().catch(() => {
// Ignore errors during cleanup
});
}
}
this.handlers.clear();
this.sessions.clear();
logger.debug("[RealtimeProcessor] Cleared all handlers and sessions");
}
}
/**
* Base Realtime Handler with common functionality
*
* Providers can extend this class for common behavior.
*/
export class BaseRealtimeHandler {
session = null;
eventHandlers = null;
state = "disconnected";
isConnected() {
return this.state === "connected";
}
getSession() {
return this.session;
}
on(handlers) {
this.eventHandlers = handlers;
}
off() {
this.eventHandlers = null;
}
/**
* Emit state change event
*/
emitStateChange(newState) {
this.state = newState;
if (this.session) {
this.session.state = newState;
this.session.lastActivityAt = new Date();
}
this.eventHandlers?.onStateChange?.(newState);
}
/**
* Emit audio event
*/
emitAudio(chunk) {
this.eventHandlers?.onAudio?.(chunk);
}
/**
* Emit transcript event
*/
emitTranscript(text, isFinal) {
this.eventHandlers?.onTranscript?.(text, isFinal);
}
/**
* Emit text event
*/
emitText(text, isFinal) {
this.eventHandlers?.onText?.(text, isFinal);
}
/**
* Emit function call event
*/
async emitFunctionCall(name, args) {
if (this.eventHandlers?.onFunctionCall) {
return this.eventHandlers.onFunctionCall(name, args);
}
return undefined;
}
/**
* Emit error event
*/
emitError(error) {
this.eventHandlers?.onError?.(error);
}
/**
* Emit turn start event
*/
emitTurnStart() {
this.eventHandlers?.onTurnStart?.();
}
/**
* Emit turn end event
*/
emitTurnEnd() {
this.eventHandlers?.onTurnEnd?.();
}
/**
* Create a session object
*/
createSession(id, config) {
return {
id,
state: "connected",
provider: this.name,
model: config.model,
createdAt: new Date(),
lastActivityAt: new Date(),
config,
};
}
}
//# sourceMappingURL=RealtimeVoiceAPI.js.map