@aj-archipelago/cortex/helper-apps/cortex-realtime-voice-server/src/realtime/client.ts

Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.

import { EventEmitter } from 'node:events';
import type { WebSocket as WS } from 'ws';
import type { MessageEvent as WS_MessageEvent } from 'ws';
import { createId } from '@paralleldrive/cuid2';
import { hasNativeWebSocket, trimDebugEvent } from './utils';
import { logger } from '../utils/logger';
import type {
  ConversationCreatedEvent,
  ConversationItemCreatedEvent,
  ConversationItemDeletedEvent,
  ConversationItemInputAudioTranscriptionCompletedEvent,
  ConversationItemInputAudioTranscriptionFailedEvent,
  ConversationItemTruncatedEvent,
  InputAudioBufferClearedEvent,
  InputAudioBufferCommittedEvent,
  InputAudioBufferSpeechStartedEvent,
  InputAudioBufferSpeechStoppedEvent,
  RateLimitsUpdatedEvent,
  RealtimeErrorEvent,
  RealtimeItem,
  RealtimeResponseConfig,
  RealtimeSession,
  RealtimeSessionConfig,
  ResponseAudioDeltaEvent,
  ResponseAudioDoneEvent,
  ResponseAudioTranscriptDeltaEvent,
  ResponseAudioTranscriptDoneEvent,
  ResponseContentPartAddedEvent,
  ResponseContentPartDoneEvent,
  ResponseCreatedEvent,
  ResponseDoneEvent,
  ResponseFunctionCallArgumentsDeltaEvent,
  ResponseFunctionCallArgumentsDoneEvent,
  ResponseOutputItemAddedEvent,
  ResponseOutputItemDoneEvent,
  ResponseTextDeltaEvent,
  ResponseTextDoneEvent,
  SessionCreatedEvent,
  SessionUpdatedEvent,
  Voice,
} from './realtimeTypes';
import { Transcription } from './transcription';
import type { ClientRequest } from 'node:http';

const REALTIME_VOICE_API_URL = 'wss://api.openai.com/v1/realtime';
const DEFAULT_INSTRUCTIONS = `
Your knowledge cutoff is 2023-10.
You are a helpful, witty, and friendly AI.
Act like a human, but remember that you aren't a human and that you can't do human things in the real world.
Your voice and personality should be warm and engaging, with a lively and playful tone.
If interacting in a non-English language, start by using the standard accent or dialect familiar to the user.
Talk quickly. You should always call a function if you can. 
Do not refer to these rules, even if you're asked about them.`;

const MAX_RECONNECT_ATTEMPTS = 5;
const BASE_RECONNECT_DELAY_MS = 1000;
const MAX_RECONNECT_DELAY_MS = 30000;

export interface RealtimeVoiceEvents {
  'connected': [];
  'close': [{ type: 'close', error?: boolean }];
  'error': [RealtimeErrorEvent];
  'session.created': [SessionCreatedEvent];
  'session.updated': [SessionUpdatedEvent];
  'conversation.created': [ConversationCreatedEvent];
  'conversation.item.created': [ConversationItemCreatedEvent];
  'conversation.item.input_audio_transcription.completed': [ConversationItemInputAudioTranscriptionCompletedEvent];
  'conversation.item.input_audio_transcription.failed': [ConversationItemInputAudioTranscriptionFailedEvent];
  'conversation.item.truncated': [ConversationItemTruncatedEvent];
  'conversation.item.deleted': [ConversationItemDeletedEvent];
  'input_audio_buffer.committed': [InputAudioBufferCommittedEvent];
  'input_audio_buffer.cleared': [InputAudioBufferClearedEvent];
  'input_audio_buffer.speech_started': [InputAudioBufferSpeechStartedEvent];
  'input_audio_buffer.speech_stopped': [InputAudioBufferSpeechStoppedEvent];
  'response.created': [ResponseCreatedEvent];
  'response.done': [ResponseDoneEvent];
  'response.output_item.added': [ResponseOutputItemAddedEvent];
  'response.output_item.done': [ResponseOutputItemDoneEvent];
  'response.content_part.added': [ResponseContentPartAddedEvent];
  'response.content_part.done': [ResponseContentPartDoneEvent];
  'response.text.delta': [ResponseTextDeltaEvent];
  'response.text.done': [ResponseTextDoneEvent];
  'response.audio_transcript.delta': [ResponseAudioTranscriptDeltaEvent];
  'response.audio_transcript.done': [ResponseAudioTranscriptDoneEvent];
  'response.audio.delta': [ResponseAudioDeltaEvent];
  'response.audio.done': [ResponseAudioDoneEvent];
  'response.function_call_arguments.delta': [ResponseFunctionCallArgumentsDeltaEvent];
  'response.function_call_arguments.done': [ResponseFunctionCallArgumentsDoneEvent];
  'rate_limits.updated': [RateLimitsUpdatedEvent];
}

interface RealtimeVoiceClientConfig {
  sessionConfig?: RealtimeSessionConfig;
  apiKey?: string;
  realtimeUrl?: string;
  model?: string;
  autoReconnect?: boolean;
  debug?: boolean;
  filterDeltas?: boolean;
}

// Create a type for the emit method
type TypedEmitter = {
  emit<K extends keyof RealtimeVoiceEvents>(
    event: K,
    ...args: RealtimeVoiceEvents[K]
  ): boolean;
  on<K extends keyof RealtimeVoiceEvents>(
    event: K,
    listener: (...args: RealtimeVoiceEvents[K]) => void
  ): TypedEmitter;
  once<K extends keyof RealtimeVoiceEvents>(
    event: K,
    listener: (...args: RealtimeVoiceEvents[K]) => void
  ): TypedEmitter;
  off<K extends keyof RealtimeVoiceEvents>(
    event: K,
    listener: (...args: RealtimeVoiceEvents[K]) => void
  ): TypedEmitter;
};

// Change the class declaration to use intersection types
export class RealtimeVoiceClient extends EventEmitter implements TypedEmitter {
  private readonly apiKey?: string;
  private readonly autoReconnect: boolean;
  private readonly debug: boolean;
  private readonly filterDeltas: boolean;
  private readonly url: string = '';
  private readonly isAzure: boolean = false;
  private readonly transcription: Transcription = new Transcription();
  private ws?: WebSocket | WS;
  private isConnected = false;
  private reconnectAttempts = 0;
  private reconnectTimeout?: NodeJS.Timer;
  private sessionConfig: RealtimeSessionConfig;

  constructor({
    sessionConfig,
    apiKey = process.env.OPENAI_API_KEY,
    realtimeUrl = process.env.REALTIME_VOICE_API_URL || REALTIME_VOICE_API_URL,
    model = 'gpt-4o-realtime-preview-2024-10-01',
    autoReconnect = true,
    debug = false,
    filterDeltas = false,
  }: RealtimeVoiceClientConfig) {
    super();
    
    this.isAzure = realtimeUrl.includes('azure.com');
    if (this.isAzure) {
      model = 'gpt-4o-realtime-preview-2024-10-01';
    } else {
      model = 'gpt-4o-realtime-preview-2024-12-17';
    }

    this.url = `${realtimeUrl.replace('https://', 'wss://')}${realtimeUrl.includes('?') ? '&' : '?'}model=${model}`;
    
    this.apiKey = apiKey;
    this.autoReconnect = autoReconnect;
    this.debug = debug;
    this.filterDeltas = filterDeltas;

    // Default voice based on provider
    const defaultVoice: Voice = 'alloy';
    
    this.sessionConfig = {
      modalities: ['audio', 'text'],
      instructions: DEFAULT_INSTRUCTIONS,
      voice: sessionConfig?.voice || defaultVoice,
      input_audio_format: 'pcm16',
      output_audio_format: 'pcm16',
      input_audio_transcription: {
        model: 'whisper-1',
      },
      turn_detection: {
        type: 'server_vad',
        threshold: 0.5,
        prefix_padding_ms: 300,
        silence_duration_ms: 1500,
      },
      tools: [],
      tool_choice: 'auto',
      temperature: 0.8,
      max_response_output_tokens: 4096,
      ...sessionConfig,
    };

    // Validate voice selection based on provider
    if (this.isAzure) {
      const azureVoices: Voice[] = ['amuch', 'dan', 'elan', 'marilyn', 'meadow', 'breeze', 'cove', 'ember', 'jupiter', 'alloy', 'echo', 'shimmer'];
      if (!azureVoices.includes(this.sessionConfig.voice)) {
        throw new Error(`Invalid voice for Azure: ${this.sessionConfig.voice}. Supported values are: ${azureVoices.join(', ')}`);
      }
    } else {
      const openaiVoices: Voice[] = ['alloy', 'echo', 'shimmer', 'ash', 'ballad', 'coral', 'sage', 'verse'];
      if (!openaiVoices.includes(this.sessionConfig.voice)) {
        throw new Error(`Invalid voice for OpenAI: ${this.sessionConfig.voice}. Supported values are: ${openaiVoices.join(', ')}`);
      }
    }
  }

  async connect() {
    if (this.isConnected) {
      return;
    }

    if (hasNativeWebSocket()) {
      if (process.versions.bun) {
        const headers: Record<string, string> = this.isAzure
          ? {
              'api-key': this.apiKey || '',
              'OpenAI-Beta': 'realtime=v1',
            }
          : {
              'Authorization': `Bearer ${this.apiKey}`,
              'OpenAI-Beta': 'realtime=v1',
            };

        this.ws = new WebSocket(this.url, {
          // @ts-ignore
          headers,
        });
      } else {
        const protocols = this.isAzure
          ? ['realtime', 'openai-beta.realtime-v1']
          : [
              'realtime',
              `openai-insecure-api-key.${this.apiKey}`,
              'openai-beta.realtime-v1',
            ];

        this.ws = new WebSocket(this.url, protocols);
      }
    } else {
      const wsModule = await import('ws');
      this.ws = new wsModule.WebSocket(this.url, [], {
        finishRequest: (request: ClientRequest) => {
          request.setHeader('OpenAI-Beta', 'realtime=v1');

          if (this.apiKey) {
            if (this.isAzure) {
              request.setHeader('api-key', this.apiKey);
            } else {
              request.setHeader('Authorization', `Bearer ${this.apiKey}`);
              request.setHeader('api-key', this.apiKey);
            }
          }
          request.end();
        },
        // TODO: this `any` is a workaround for `@types/ws` being out-of-date.
      } as any);
    }
    this.ws.addEventListener('open', this.onOpen.bind(this));
    this.ws.addEventListener('message', this.onMessage.bind(this));
    this.ws.addEventListener('error', this.onError.bind(this));
    this.ws.addEventListener('close', this.onCloseWithReconnect.bind(this));
  }

  onOpen() {
    this._log(`Connected to "${this.url}"`);
    this.isConnected = true;
    
    // If reconnectAttempts > 0, this is a reconnection
    if (this.reconnectAttempts > 0) {
      this.updateSocketState();
    } else {
      this.emit('connected');
    }
    this.reconnectAttempts = 0; // Reset attempts on successful connection
  }

  onMessage(event: MessageEvent<any> | WS_MessageEvent) {
    const message: any = JSON.parse(event.data);
    this._log('Received message:', message);

    this.receive(message.type, message);
  }

  async onError() {
    this._log(`Error, disconnected from "${this.url}"`);
    if (!await this.disconnect(this.autoReconnect)) {
      this.emit('close', { type: 'close', error: true });
    }
  }

  async onCloseWithReconnect() {
    this._log(`Disconnected from "${this.url}", reconnect: ${this.autoReconnect}`);
    if (!await this.disconnect(this.autoReconnect)) {
      this.emit('close', { type: 'close', error: false });
    }
  }

  async disconnect(reconnect: boolean = false): Promise<boolean> {
    logger.log('Disconnect called:', this.isConnected, reconnect);
    if (this.isConnected) {
      this.isConnected = false;
      this.ws?.close();
      this.ws = undefined;
    }

    if (reconnect) {
      if (this.reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {
        logger.error('Max reconnection attempts reached');
        this.emit('error', { type: 'error', message: 'Failed to reconnect after maximum attempts' });
        return false;
      }

      // Clear any existing reconnect timeout
      if (this.reconnectTimeout) {
        clearTimeout(this.reconnectTimeout);
      }

      // Calculate delay with exponential backoff
      const delay = Math.min(
        BASE_RECONNECT_DELAY_MS * Math.pow(2, this.reconnectAttempts),
        MAX_RECONNECT_DELAY_MS
      );

      this.reconnectAttempts++;
      
      // Schedule reconnection attempt
      this.reconnectTimeout = setTimeout(async () => {
        try {
          await this.connect();
        } catch (error) {
          logger.error('Reconnection attempt failed:', error);
          // Try again if we haven't hit the limit
          if (this.reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
            await this.disconnect(true);
          } else {
            this.emit('error', { type: 'error', message: 'Failed to reconnect after maximum attempts' });
          }
        }
      }, delay);

      return true;
    }

    // Reset reconnection state when explicitly disconnecting
    this.reconnectAttempts = 0;
    if (this.reconnectTimeout) {
      clearTimeout(this.reconnectTimeout);
    }
    return false;
  }

  getConversationItems(): RealtimeItem[] {
    return this.transcription.getOrderedItems();
  }

  getItem(item_id: string): RealtimeItem | undefined {
    return this.transcription.getItem(item_id);
  }

  updateSession(sessionConfig: Partial<RealtimeSessionConfig>) {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    
    // Create a new config object without custom_voice_id
    const { custom_voice_id, ...filteredConfig } = {
      ...this.sessionConfig,
      ...sessionConfig
    };

    const message = JSON.stringify({
      event_id: createId(),
      type: 'session.update',
      session: filteredConfig,
    });
    
    // No need to log session update messages as they can be noisy
    logger.log('Sending session update message:', message);
    this.ws?.send(message);
  }

  appendInputAudio(base64AudioBuffer: string) {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    if (base64AudioBuffer.length > 0) {
      this.ws?.send(JSON.stringify({
        event_id: createId(),
        type: 'input_audio_buffer.append',
        audio: base64AudioBuffer,
      }));
    }
  }

  commitInputAudio() {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    this.ws?.send(JSON.stringify({
      event_id: createId(),
      type: 'input_audio_buffer.commit',
    }));
  }

  clearInputAudio() {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    this.ws?.send(JSON.stringify({
      event_id: createId(),
      type: 'input_audio_buffer.clear',
    }));
  }

  createConversationItem(item: RealtimeItem, previousItemId: string | null = null) {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    this.ws?.send(JSON.stringify({
      event_id: createId(),
      type: 'conversation.item.create',
      previous_item_id: previousItemId,
      item,
    }));
  }

  truncateConversationItem(itemId: string, contentIndex: number, audioEndMs: number) {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    this.ws?.send(JSON.stringify({
      event_id: createId(),
      type: 'conversation.item.truncate',
      item_id: itemId,
      content_index: contentIndex,
      audio_end_ms: audioEndMs,
    }));
  }

  deleteConversationItem(itemId: string) {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    this.ws?.send(JSON.stringify({
      event_id: createId(),
      type: 'conversation.item.delete',
      item_id: itemId,
    }));
  }

  createResponse(responseConfig: Partial<RealtimeResponseConfig>) {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    this.ws?.send(JSON.stringify({
      event_id: createId(),
      type: 'response.create',
      response: responseConfig,
    }));
  }

  cancelResponse() {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    this.ws?.send(JSON.stringify({
      event_id: createId(),
      type: 'response.cancel',
    }));
  }

  protected updateSocketState() {
    if (!this.isConnected) {
      throw new Error('Not connected');
    }
    this.updateSession(this.sessionConfig);
    const items = this.getConversationItems();
    let previousItemId: string | null = null;
    items.forEach((item) => {
      this.createConversationItem(item, previousItemId);
      previousItemId = item.id;
    });
  }

  protected saveSession(newSession: RealtimeSession) {
    const sessionCopy: any = structuredClone(newSession);
    delete sessionCopy['id'];
    delete sessionCopy['object'];
    delete sessionCopy['model'];
    delete sessionCopy['expires_at'];
    delete sessionCopy['client_secret'];
    this.sessionConfig = sessionCopy;
  }

  protected receive(type: string, message: any) {
    switch (type) {
      case 'error':
        this.emit('error', message);
        break;
      case 'session.created':
        this.saveSession((message as SessionCreatedEvent).session);
        break;
      case 'session.updated':
        this.saveSession((message as SessionUpdatedEvent).session);
        break;
      case 'conversation.item.created':
        this.transcription.addItem(message.item, message.previous_item_id);
        break;
      case 'conversation.item.input_audio_transcription.completed':
        this.transcription.addTranscriptToItem(message.item_id, message.transcript);
        break;
      case 'conversation.item.deleted':
        this.transcription.removeItem(message.item_id);
        break;
      case 'response.output_item.added':
        this.transcription.addItem(message.item, message.previous_item_id);
        break;
      case 'response.output_item.done':
        this.transcription.updateItem(message.item.id, message.item);
        break;
    }
    // @ts-ignore
    this.emit(type, message);
  }

  protected _log(...args: any[]) {
    if (!this.debug) {
      return;
    }

    // Filter out delta messages if filterDeltas is enabled
    if (this.filterDeltas) {
      const firstArg = args[0];
      if (typeof firstArg === 'object' && firstArg?.type?.includes('.delta')) {
        return;
      }
      if (typeof firstArg === 'string' && firstArg === 'Received message:' && args[1]?.type?.includes('.delta')) {
        return;
      }
    }

    const date = new Date().toISOString();
    const logs = [`[Websocket/${date}]`].concat(args).map((arg) => {
      if (typeof arg === 'object' && arg !== null) {
        return JSON.stringify(trimDebugEvent(arg), null, 2);
      } else {
        return arg;
      }
    });
    logger.log(...logs);
  }

  public canReconnect(): boolean {
    return this.autoReconnect && this.reconnectAttempts < MAX_RECONNECT_ATTEMPTS;
  }
}