@openai/agents-realtime
Version:
The OpenAI Agents SDK is a lightweight yet powerful framework for building multi-agent workflows. This package contains the logic for building realtime voice agents on the server or in the browser.
148 lines (147 loc) • 6.99 kB
TypeScript
import { RuntimeEventEmitter } from '@openai/agents-core';
import type { MessageEvent as WebSocketMessageEvent } from 'ws';
import { RealtimeClientMessage, RealtimeSessionConfig, RealtimeTracingConfig, RealtimeTurnDetectionConfigAsIs, RealtimeUserInput } from './clientMessages';
import { RealtimeItem } from './items';
import { ApiKey, RealtimeTransportLayer, RealtimeTransportLayerConnectOptions } from './transportLayer';
import { RealtimeTransportEventTypes, TransportToolCallEvent } from './transportLayerEvents';
import { EventEmitterDelegate } from '@openai/agents-core/utils';
/**
* The models that are supported by the OpenAI Realtime API.
*/
export type OpenAIRealtimeModels = 'gpt-4o-realtime-preview' | 'gpt-4o-mini-realtime-preview' | 'gpt-4o-realtime-preview-2025-06-03' | 'gpt-4o-realtime-preview-2024-12-17' | 'gpt-4o-realtime-preview-2024-10-01' | 'gpt-4o-mini-realtime-preview-2024-12-17' | (string & {});
/**
* The default model that is used during the connection if no model is provided.
*/
export declare const DEFAULT_OPENAI_REALTIME_MODEL: OpenAIRealtimeModels;
/**
* The default session config that gets send over during session connection unless overridden
* by the user.
*/
export declare const DEFAULT_OPENAI_REALTIME_SESSION_CONFIG: Partial<RealtimeSessionConfig>;
/**
* The options for the OpenAI Realtime transport layer.
*/
export type OpenAIRealtimeBaseOptions = {
/**
* The model to used during the connection.
*/
model?: OpenAIRealtimeModels;
/**
* The API key to use for the connection.
*/
apiKey?: ApiKey;
};
/**
* The events that are emitted by the OpenAI Realtime transport layer.
*/
export type OpenAIRealtimeEventTypes = {
/**
* Triggered when the connection is established.
*/
connected: [];
/**
* Triggered when the connection is closed.
*/
disconnected: [];
} & RealtimeTransportEventTypes;
export declare abstract class OpenAIRealtimeBase extends EventEmitterDelegate<OpenAIRealtimeEventTypes> implements RealtimeTransportLayer {
#private;
protected eventEmitter: RuntimeEventEmitter<OpenAIRealtimeEventTypes>;
constructor(options?: OpenAIRealtimeBaseOptions);
/**
* The current model that is being used by the transport layer.
*/
get currentModel(): OpenAIRealtimeModels;
/**
* The current model that is being used by the transport layer.
* **Note**: The model cannot be changed mid conversation.
*/
set currentModel(model: OpenAIRealtimeModels);
abstract get status(): 'connected' | 'disconnected' | 'connecting' | 'disconnecting';
abstract connect(options: RealtimeTransportLayerConnectOptions): Promise<void>;
abstract sendEvent(event: RealtimeClientMessage): void;
abstract mute(muted: boolean): void;
abstract close(): void;
abstract interrupt(): void;
abstract readonly muted: boolean | null;
protected get _rawSessionConfig(): Record<string, any> | null;
protected _getApiKey(options: RealtimeTransportLayerConnectOptions): Promise<string>;
protected _onMessage(event: MessageEvent | WebSocketMessageEvent): void;
protected _onError(error: any): void;
protected _onOpen(): void;
protected _onClose(): void;
/**
* Send a message to the Realtime API. This will create a new item in the conversation and
* trigger a response.
*
* @param message - The message to send.
* @param otherEventData - Additional event data to send.
*/
sendMessage(message: RealtimeUserInput, otherEventData: Record<string, any>): void;
protected _getMergedSessionConfig(config: Partial<RealtimeSessionConfig>): {
instructions: string | undefined;
model: string;
voice: string | undefined;
speed: number | undefined;
modalities: ("text" | "audio")[] | undefined;
input_audio_format: import("./clientMessages").RealtimeAudioFormat | undefined;
output_audio_format: import("./clientMessages").RealtimeAudioFormat | undefined;
input_audio_transcription: import("./clientMessages").RealtimeInputAudioTranscriptionConfig | undefined;
input_audio_noise_reduction: import("./clientMessages").RealtimeInputAudioNoiseReductionConfig | null | undefined;
turn_detection: RealtimeTurnDetectionConfigAsIs | (import("./clientMessages").RealtimeTurnDetectionConfigCamelCase & Record<string, any>) | undefined;
tool_choice: import("@openai/agents-core").ModelSettingsToolChoice | undefined;
tools: {
strict: undefined;
type: "function";
name: string;
description: string;
parameters: import("@openai/agents-core/types").JsonObjectSchema<any>;
}[] | undefined;
};
private static buildTurnDetectionConfig;
/**
* Sets the internal tracing config. This is used to track the tracing config that has been set
* during the session.create event.
*/
set _tracingConfig(tracingConfig: RealtimeTracingConfig | null);
/**
* Sets the tracing config for the session. This will send the tracing config to the Realtime API.
*
* @param tracingConfig - The tracing config to set. We don't support 'auto' here as the SDK will always configure a Workflow Name unless it exists
*/
protected _updateTracingConfig(tracingConfig: RealtimeTracingConfig): void;
/**
* Updates the session config. This will merge it with the current session config with the default
* values and send it to the Realtime API.
*
* @param config - The session config to update.
*/
updateSessionConfig(config: Partial<RealtimeSessionConfig>): void;
/**
* Send the output of a function call to the Realtime API.
*
* @param toolCall - The tool call to send the output for.
* @param output - The output of the function call.
* @param startResponse - Whether to start a new response after sending the output.
*/
sendFunctionCallOutput(toolCall: TransportToolCallEvent, output: string, startResponse?: boolean): void;
/**
* Send an audio buffer to the Realtime API. If `{ commit: true }` is passed, the audio buffer
* will be committed and the model will start processing it. This is necessary if you have
* disabled turn detection / voice activity detection (VAD).
*
* @param audio - The audio buffer to send.
* @param options - The options for the audio buffer.
*/
sendAudio(audio: ArrayBuffer, { commit }?: {
commit?: boolean;
}): void;
/**
* Reset the history of the conversation. This will create a diff between the old and new history
* and send the necessary events to the Realtime API to update the history.
*
* @param oldHistory - The old history of the conversation.
* @param newHistory - The new history of the conversation.
*/
resetHistory(oldHistory: RealtimeItem[], newHistory: RealtimeItem[]): void;
}