@assistant-ui/react/dist/adapters/OpenAIRealtimeVoiceAdapter.js

Open-source TypeScript/React library for building production-grade AI chat experiences

import { createVoiceSession } from "@assistant-ui/core";
// src/adapters/OpenAIRealtimeVoiceAdapter.ts
const asRecord = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : void 0;
/**
* Connects browser microphone audio directly to OpenAI Realtime over WebRTC.
* Configure audio output, input transcription, and automatic turn detection on
* the server. Each connection starts a new conversation; existing thread history
* and assistant-ui tools are not sent to the voice model.
*/
var OpenAIRealtimeVoiceAdapter = class {
	options;
	constructor(options) {
		this.options = options;
	}
	connect(options) {
		return createVoiceSession(options, async (session) => {
			if (typeof RTCPeerConnection === "undefined" || typeof document === "undefined" || typeof navigator === "undefined" || !navigator.mediaDevices?.getUserMedia) throw new Error("OpenAI Realtime voice requires a browser with WebRTC and microphone access.");
			const controller = new AbortController();
			const peer = new RTCPeerConnection();
			const channel = peer.createDataChannel("oai-events");
			const audio = document.createElement("audio");
			audio.autoplay = true;
			let microphone;
			let muted = false;
			let closed = false;
			let generating = false;
			let speaking = false;
			let transcribing = false;
			let volumeTimer;
			let connectionTimer;
			let assistantTranscript;
			const transcriptQueue = [];
			const pendingTranscripts = /* @__PURE__ */ new Map();
			const flushTranscripts = () => {
				while (transcriptQueue[0]?.ready) {
					const item = transcriptQueue[0];
					item.ready = false;
					if (item.value?.isFinal !== false) transcriptQueue.shift();
					if (item.value) session.emitTranscript(item.value);
				}
			};
			const finishUserTranscript = (id, value) => {
				const item = typeof id === "string" ? pendingTranscripts.get(id) : void 0;
				if (item) {
					item.ready = true;
					item.value = value;
					pendingTranscripts.delete(id);
				} else if (value) transcriptQueue.push({
					ready: true,
					value
				});
				flushTranscripts();
			};
			const pending = /* @__PURE__ */ new Map();
			const disconnect = () => {
				if (closed) return;
				closed = true;
				controller.abort();
				clearInterval(volumeTimer);
				clearTimeout(connectionTimer);
				for (const id of pending.keys()) settle(id, /* @__PURE__ */ new Error("OpenAI Realtime session disconnected before accepting the message."));
				pendingTranscripts.clear();
				transcriptQueue.length = 0;
				channel.onopen = channel.onmessage = channel.onclose = channel.onerror = null;
				peer.ontrack = peer.onconnectionstatechange = null;
				for (const track of microphone?.getTracks() ?? []) track.stop();
				audio.pause();
				audio.srcObject = null;
				channel.close();
				peer.close();
			};
			const end = (reason, error) => {
				session.end(reason, error);
				disconnect();
			};
			const send = (event) => {
				if (closed || channel.readyState !== "open") throw new Error("OpenAI Realtime session is not connected.");
				channel.send(JSON.stringify(event));
			};
			const request = (id, event) => new Promise((resolve, reject) => {
				const timer = setTimeout(() => {
					reject(/* @__PURE__ */ new Error("OpenAI Realtime did not acknowledge the message."));
				}, 1e4);
				pending.set(id, {
					resolve,
					reject,
					timer
				});
				send({
					...event,
					event_id: id
				});
			}).finally(() => settle(id));
			const settle = (id, error) => {
				if (typeof id !== "string") return false;
				const message = pending.get(id);
				if (!message) return false;
				clearTimeout(message.timer);
				pending.delete(id);
				if (error) message.reject(error);
				else message.resolve();
				return true;
			};
			const setSpeaking = (value) => {
				speaking = value;
				session.emitMode(value ? "speaking" : "listening");
			};
			channel.onmessage = ({ data }) => {
				if (closed) return;
				let event;
				try {
					event = asRecord(JSON.parse(data));
				} catch {
					return;
				}
				if (!event) return;
				switch (event.type) {
					case "session.created":
					case "session.updated": {
						const audio = asRecord(asRecord(event.session)?.audio);
						const input = asRecord(audio?.input);
						transcribing = asRecord(input?.transcription) !== void 0;
						break;
					}
					case "input_audio_buffer.committed":
						if (transcribing && typeof event.item_id === "string" && !pendingTranscripts.has(event.item_id)) {
							const item = {
								ready: false,
								value: void 0
							};
							pendingTranscripts.set(event.item_id, item);
							transcriptQueue.push(item);
						}
						break;
					case "conversation.item.input_audio_transcription.completed":
						if (typeof event.transcript === "string") finishUserTranscript(event.item_id, {
							role: "user",
							text: event.transcript,
							isFinal: true
						});
						break;
					case "response.output_audio_transcript.delta":
					case "response.output_audio_transcript.done": {
						if (typeof event.item_id !== "string") break;
						const final = event.type === "response.output_audio_transcript.done";
						const text = final ? event.transcript : event.delta;
						if (typeof text !== "string") break;
						if (assistantTranscript?.id !== event.item_id) {
							assistantTranscript = {
								id: event.item_id,
								ready: false,
								value: void 0
							};
							transcriptQueue.push(assistantTranscript);
						}
						assistantTranscript.ready = true;
						assistantTranscript.value = {
							role: "assistant",
							text: final ? text : (assistantTranscript.value?.text ?? "") + text,
							isFinal: final
						};
						flushTranscripts();
						break;
					}
					case "response.created":
						generating = true;
						settle(asRecord(asRecord(event.response)?.metadata)?.aui_request_id);
						break;
					case "response.done": {
						generating = false;
						const response = asRecord(event.response);
						if (assistantTranscript?.value?.isFinal === false) {
							assistantTranscript.value = {
								...assistantTranscript.value,
								isFinal: true
							};
							assistantTranscript.ready = true;
							flushTranscripts();
						}
						if (response?.status === "failed") console.warn("OpenAI Realtime response failed.", response.status_details);
						break;
					}
					case "output_audio_buffer.started":
						setSpeaking(true);
						break;
					case "output_audio_buffer.stopped":
					case "output_audio_buffer.cleared":
						setSpeaking(false);
						break;
					case "conversation.item.added":
						settle(asRecord(event.item)?.id);
						break;
					case "conversation.item.input_audio_transcription.failed":
						finishUserTranscript(event.item_id);
						console.warn("OpenAI Realtime transcription failed.", event.error);
						break;
					case "error": {
						const error = asRecord(event.error);
						if (error?.code === "response_cancel_not_active") break;
						if (!settle(error?.event_id, new Error(typeof error?.message === "string" ? error.message : "OpenAI Realtime rejected the message."))) console.warn("OpenAI Realtime error.", event.error);
						break;
					}
				}
			};
			channel.onopen = () => {
				if (closed) return;
				clearTimeout(connectionTimer);
				session.setStatus({ type: "running" });
				if (closed) return;
				session.emitMode("listening");
				if (closed) return;
				volumeTimer = setInterval(() => {
					peer.getStats().then((report) => {
						if (closed) return;
						let volume = 0;
						report.forEach((stat) => {
							if ((stat.type === "inbound-rtp" || !muted && stat.type === "media-source") && typeof stat.audioLevel === "number") volume = Math.max(volume, stat.audioLevel);
						});
						session.emitVolume(Math.min(1, volume));
					}).catch(() => {});
				}, 100);
			};
			channel.onclose = () => end("finished");
			channel.onerror = () => end("error", /* @__PURE__ */ new Error("OpenAI Realtime data channel failed."));
			peer.onconnectionstatechange = () => {
				if (peer.connectionState === "failed") end("error", /* @__PURE__ */ new Error("OpenAI Realtime connection failed."));
				else if (peer.connectionState === "closed") end("finished");
			};
			peer.ontrack = ({ track }) => {
				if (closed || track.kind !== "audio") return;
				audio.srcObject = new MediaStream([track]);
				audio.play().catch((error) => end("error", error));
			};
			const start = async () => {
				const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
				if (closed || session.isDisposed()) {
					for (const track of stream.getTracks()) track.stop();
					return;
				}
				microphone = stream;
				for (const track of stream.getAudioTracks()) {
					track.enabled = !muted;
					peer.addTrack(track, stream);
				}
				connectionTimer = setTimeout(() => end("error", /* @__PURE__ */ new Error("OpenAI Realtime connection timed out.")), 3e4);
				const offer = await peer.createOffer();
				if (closed) return;
				await peer.setLocalDescription(offer);
				if (closed) return;
				const answer = await this.options.exchangeSdp(offer.sdp, { signal: controller.signal });
				if (closed) return;
				await peer.setRemoteDescription({
					type: "answer",
					sdp: answer
				});
			};
			start().catch((error) => end("error", error));
			return {
				disconnect,
				mute: () => {
					muted = true;
					for (const track of microphone?.getAudioTracks() ?? []) track.enabled = false;
				},
				unmute: () => {
					muted = false;
					for (const track of microphone?.getAudioTracks() ?? []) track.enabled = true;
				},
				sendText: async (text) => {
					if (generating) send({ type: "response.cancel" });
					if (speaking) send({ type: "output_audio_buffer.clear" });
					const id = crypto.randomUUID().replaceAll("-", "");
					await request(id, {
						type: "conversation.item.create",
						item: {
							id,
							type: "message",
							role: "user",
							content: [{
								type: "input_text",
								text
							}]
						}
					});
					const responseId = crypto.randomUUID().replaceAll("-", "");
					await request(responseId, {
						type: "response.create",
						response: { metadata: { aui_request_id: responseId } }
					});
				}
			};
		});
	}
};
export { OpenAIRealtimeVoiceAdapter };

//# sourceMappingURL=OpenAIRealtimeVoiceAdapter.js.map