@assistant-ui/react
Version:
Open-source TypeScript/React library for building production-grade AI chat experiences
762 lines (723 loc) • 26.8 kB
text/typescript
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { OpenAIRealtimeVoiceAdapter } from "./OpenAIRealtimeVoiceAdapter";
const deferred = <T>() => {
let resolve!: (value: T) => void;
let reject!: (reason: unknown) => void;
const promise = new Promise<T>((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
};
class Channel {
readyState = "connecting";
onopen: (() => void) | null = null;
onclose: (() => void) | null = null;
onerror: (() => void) | null = null;
onmessage: ((event: { data: string }) => void) | null = null;
send = vi.fn<(data: string) => void>();
close = vi.fn(() => {
this.readyState = "closed";
this.onclose?.();
});
open() {
this.readyState = "open";
this.onopen?.();
}
emit(event: unknown) {
this.onmessage?.({ data: JSON.stringify(event) });
}
events() {
return this.send.mock.calls.map(([data]) => JSON.parse(data));
}
}
class Peer {
static instances: Peer[] = [];
channel = new Channel();
connectionState = "new";
onconnectionstatechange: (() => void) | null = null;
ontrack: ((event: { track: { kind: string } }) => void) | null = null;
createDataChannel = vi.fn(() => this.channel);
addTrack = vi.fn();
createOffer = vi.fn(async () => ({ type: "offer", sdp: "browser-offer" }));
setLocalDescription = vi.fn(async () => {});
setRemoteDescription = vi.fn(async () => {});
getStats = vi.fn(
async () =>
new Map([
["input", { type: "media-source", audioLevel: 0.7 }],
["output", { type: "inbound-rtp", audioLevel: 0.4 }],
]),
);
close = vi.fn(() => {
this.connectionState = "closed";
this.onconnectionstatechange?.();
});
constructor() {
Peer.instances.push(this);
}
}
const track = { enabled: true, stop: vi.fn() };
const stream = { getTracks: () => [track], getAudioTracks: () => [track] };
const getUserMedia = vi.fn(async () => stream);
const audio = {
autoplay: false,
srcObject: null as unknown,
play: vi.fn(async () => {}),
pause: vi.fn(),
};
const exchangeSdp = vi.fn<
(sdp: string, options: { signal: AbortSignal }) => Promise<string>
>(async () => "server-answer");
const sessions: ReturnType<OpenAIRealtimeVoiceAdapter["connect"]>[] = [];
const start = (abortSignal?: AbortSignal) => {
const session = new OpenAIRealtimeVoiceAdapter({ exchangeSdp }).connect(
abortSignal ? { abortSignal } : {},
);
sessions.push(session);
const peer = Peer.instances.at(-1)!;
return { session, peer, channel: peer.channel };
};
const connected = async () => {
const result = start();
await vi.waitFor(() =>
expect(result.peer.setRemoteDescription).toHaveBeenCalled(),
);
result.channel.open();
result.channel.emit({
type: "session.created",
session: {
audio: { input: { transcription: { model: "gpt-4o-mini-transcribe" } } },
},
});
return result;
};
beforeEach(() => {
Peer.instances = [];
track.enabled = true;
getUserMedia.mockResolvedValue(stream);
exchangeSdp.mockResolvedValue("server-answer");
audio.play.mockResolvedValue();
vi.stubGlobal("RTCPeerConnection", Peer);
vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } });
vi.stubGlobal("document", { createElement: () => audio });
vi.stubGlobal(
"MediaStream",
class {
tracks: unknown[];
constructor(tracks: unknown[]) {
this.tracks = tracks;
}
},
);
});
afterEach(() => {
for (const session of sessions.splice(0)) session.disconnect();
vi.unstubAllGlobals();
vi.useRealTimers();
});
describe("OpenAIRealtimeVoiceAdapter", () => {
it("exchanges SDP and waits for the data channel before becoming usable", async () => {
const { session, peer, channel } = start();
await vi.waitFor(() => expect(exchangeSdp).toHaveBeenCalled());
expect(exchangeSdp).toHaveBeenCalledWith("browser-offer", {
signal: expect.any(AbortSignal),
});
expect(peer.setRemoteDescription).toHaveBeenCalledWith({
type: "answer",
sdp: "server-answer",
});
expect(peer.addTrack).toHaveBeenCalledWith(track, stream);
expect(session.status).toEqual({ type: "starting" });
channel.open();
expect(session.status).toEqual({ type: "running" });
expect(session.sendText).toBeTypeOf("function");
});
it("streams cumulative assistant transcripts and records completed user audio", async () => {
const { session, channel } = await connected();
const onTranscript = vi.fn();
session.onTranscript(onTranscript);
channel.emit({
type: "conversation.item.input_audio_transcription.completed",
transcript: "Hello",
});
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "a",
delta: "Good ",
});
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "a",
delta: "morning",
});
channel.emit({
type: "response.output_audio_transcript.done",
item_id: "a",
transcript: "Good morning!",
});
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "b",
delta: "Next",
});
expect(onTranscript.mock.calls.map(([item]) => item)).toEqual([
{ role: "user", text: "Hello", isFinal: true },
{ role: "assistant", text: "Good ", isFinal: false },
{ role: "assistant", text: "Good morning", isFinal: false },
{ role: "assistant", text: "Good morning!", isFinal: true },
{ role: "assistant", text: "Next", isFinal: false },
]);
});
it("uses audio playback events instead of generation completion for speaking state", async () => {
const { session, channel } = await connected();
const onMode = vi.fn();
session.onModeChange(onMode);
channel.emit({ type: "output_audio_buffer.started" });
channel.emit({ type: "response.done", response: { status: "completed" } });
expect(onMode.mock.calls).toEqual([["speaking"]]);
channel.emit({ type: "input_audio_buffer.speech_started" });
expect(onMode.mock.calls).toEqual([["speaking"]]);
channel.emit({ type: "output_audio_buffer.cleared" });
expect(onMode).toHaveBeenLastCalledWith("listening");
});
it("holds replies until preceding spoken turns are transcribed in conversation order", async () => {
const { session, channel } = await connected();
const onTranscript = vi.fn();
session.onTranscript(onTranscript);
for (const id of ["first", "second"]) {
channel.emit({ type: "input_audio_buffer.committed", item_id: id });
channel.emit({
type: "response.output_audio_transcript.done",
item_id: `${id}-reply`,
transcript: `Reply to ${id}`,
});
}
channel.emit({
type: "conversation.item.input_audio_transcription.completed",
item_id: "second",
transcript: "Second question",
});
expect(onTranscript).not.toHaveBeenCalled();
channel.emit({
type: "conversation.item.input_audio_transcription.completed",
item_id: "first",
transcript: "First question",
});
expect(onTranscript.mock.calls.map(([item]) => item)).toEqual([
{ role: "user", text: "First question", isFinal: true },
{ role: "assistant", text: "Reply to first", isFinal: true },
{ role: "user", text: "Second question", isFinal: true },
{ role: "assistant", text: "Reply to second", isFinal: true },
]);
});
it("keeps a streaming reply together when the user transcript arrives after its first delta", async () => {
const { session, channel } = await connected();
const onTranscript = vi.fn();
session.onTranscript(onTranscript);
channel.emit({ type: "input_audio_buffer.committed", item_id: "user" });
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "reply",
delta: "Good ",
});
expect(onTranscript).not.toHaveBeenCalled();
channel.emit({
type: "conversation.item.input_audio_transcription.completed",
item_id: "user",
transcript: "Hello",
});
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "reply",
delta: "morning",
});
channel.emit({
type: "response.output_audio_transcript.done",
item_id: "reply",
transcript: "Good morning",
});
expect(onTranscript.mock.calls.map(([item]) => item)).toEqual([
{ role: "user", text: "Hello", isFinal: true },
{ role: "assistant", text: "Good ", isFinal: false },
{ role: "assistant", text: "Good morning", isFinal: false },
{ role: "assistant", text: "Good morning", isFinal: true },
]);
});
it("releases replies after transcription failure and keeps the call usable", async () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
try {
const { session, channel } = await connected();
const onTranscript = vi.fn();
session.onTranscript(onTranscript);
channel.emit({ type: "input_audio_buffer.committed", item_id: "user" });
channel.emit({
type: "response.output_audio_transcript.done",
item_id: "reply",
transcript: "Please repeat that",
});
channel.emit({
type: "conversation.item.input_audio_transcription.failed",
item_id: "user",
error: { message: "Could not transcribe" },
});
channel.emit({ type: "response.done", response: { status: "failed" } });
channel.emit({
type: "error",
error: {
code: "conversation_already_has_active_response",
message: "Already responding",
},
});
expect(session.status.type).toBe("running");
expect(onTranscript).toHaveBeenCalledWith({
role: "assistant",
text: "Please repeat that",
isFinal: true,
});
expect(track.stop).not.toHaveBeenCalled();
expect(warn).toHaveBeenCalled();
} finally {
warn.mockRestore();
}
});
it.each(["session.created", "session.updated"])(
"streams replies with transcription disabled by %s",
async (type) => {
const { session, channel } = await connected();
const onTranscript = vi.fn();
session.onTranscript(onTranscript);
channel.emit({
type,
session: { audio: { input: { transcription: null } } },
});
channel.emit({ type: "input_audio_buffer.committed", item_id: "user" });
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "reply",
delta: "Hello",
});
channel.emit({
type: "response.output_audio_transcript.done",
item_id: "reply",
transcript: "Hello!",
});
expect(onTranscript.mock.calls.map(([item]) => item)).toEqual([
{ role: "assistant", text: "Hello", isFinal: false },
{ role: "assistant", text: "Hello!", isFinal: true },
]);
},
);
it("releases later turns when a failed response omits the final transcript event", async () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
try {
const { session, channel } = await connected();
const onTranscript = vi.fn();
session.onTranscript(onTranscript);
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "reply",
delta: "Partial",
});
channel.emit({ type: "input_audio_buffer.committed", item_id: "user" });
channel.emit({
type: "conversation.item.input_audio_transcription.completed",
item_id: "user",
transcript: "Next question",
});
channel.emit({ type: "response.done", response: { status: "failed" } });
channel.emit({
type: "response.output_audio_transcript.done",
item_id: "next-reply",
transcript: "Next reply",
});
channel.emit({
type: "response.done",
response: { status: "completed" },
});
expect(onTranscript.mock.calls.map(([item]) => item)).toEqual([
{ role: "assistant", text: "Partial", isFinal: false },
{ role: "assistant", text: "Partial", isFinal: true },
{ role: "user", text: "Next question", isFinal: true },
{ role: "assistant", text: "Next reply", isFinal: true },
]);
expect(session.status.type).toBe("running");
} finally {
warn.mockRestore();
}
});
it("cleans up a typed request when the data channel rejects the send", async () => {
vi.useFakeTimers();
const { session, channel } = await connected();
channel.send.mockImplementation(() => {
throw new Error("Channel unavailable");
});
await expect(session.sendText!("Hello")).rejects.toThrow(
"Channel unavailable",
);
expect(vi.getTimerCount()).toBe(1);
session.disconnect();
expect(vi.getTimerCount()).toBe(0);
});
it.each([true, false])(
"keeps an uninterrupted assistant message together when the next user transcript finishes early: %s",
async (early) => {
const { session, channel } = await connected();
const onTranscript = vi.fn();
session.onTranscript(onTranscript);
channel.emit({
type: "session.updated",
session: {
audio: {
input: {
transcription: { model: "gpt-4o-mini-transcribe" },
turn_detection: { type: "server_vad", interrupt_response: false },
},
},
},
});
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "reply",
delta: "Good ",
});
channel.emit({
type: "input_audio_buffer.committed",
item_id: "next-user",
});
const user = {
type: "conversation.item.input_audio_transcription.completed",
item_id: "next-user",
transcript: "Next question",
};
if (early) channel.emit(user);
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "reply",
delta: "morning",
});
channel.emit({
type: "response.output_audio_transcript.done",
item_id: "reply",
transcript: "Good morning",
});
if (!early) channel.emit(user);
expect(onTranscript.mock.calls.map(([item]) => item)).toEqual([
{ role: "assistant", text: "Good ", isFinal: false },
{ role: "assistant", text: "Good morning", isFinal: false },
{ role: "assistant", text: "Good morning", isFinal: true },
{ role: "user", text: "Next question", isFinal: true },
]);
},
);
it("waits for both typed item and response acceptance without echoing the transcript", async () => {
const { session, channel } = await connected();
const onTranscript = vi.fn();
const accepted = vi.fn();
session.onTranscript(onTranscript);
channel.emit({ type: "response.created" });
channel.emit({
type: "response.output_audio_transcript.delta",
item_id: "old-reply",
delta: "Previous reply",
});
channel.emit({ type: "output_audio_buffer.started" });
const sent = Promise.resolve(session.sendText!("Change the topic"));
void sent.then(accepted);
const events = channel.events();
expect(events.map((event) => event.type)).toEqual([
"response.cancel",
"output_audio_buffer.clear",
"conversation.item.create",
]);
expect(events[2].item).toMatchObject({
role: "user",
content: [{ type: "input_text", text: "Change the topic" }],
});
channel.emit({ type: "response.done", response: { status: "cancelled" } });
expect(onTranscript.mock.calls.map(([item]) => item)).toEqual([
{ role: "assistant", text: "Previous reply", isFinal: false },
{ role: "assistant", text: "Previous reply", isFinal: true },
]);
expect(accepted).not.toHaveBeenCalled();
channel.emit({
type: "conversation.item.added",
item: { id: "unrelated" },
});
expect(accepted).not.toHaveBeenCalled();
channel.emit({
type: "conversation.item.added",
item: { id: events[2].item.id },
});
await vi.waitFor(() =>
expect(channel.events().at(-1).type).toBe("response.create"),
);
expect(accepted).not.toHaveBeenCalled();
channel.emit({
type: "response.created",
response: { metadata: { aui_request_id: "unrelated" } },
});
await Promise.resolve();
expect(accepted).not.toHaveBeenCalled();
const response = channel.events().at(-1);
expect(response.response.metadata.aui_request_id).toBe(response.event_id);
channel.emit({
type: "response.created",
response: { metadata: response.response.metadata },
});
await sent;
expect(onTranscript.mock.calls.map(([item]) => item)).toEqual([
{ role: "assistant", text: "Previous reply", isFinal: false },
{ role: "assistant", text: "Previous reply", isFinal: true },
]);
});
it("rejects a typed send whose response request fails after the item was accepted", async () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
try {
const { session, channel } = await connected();
const rejected = expect(session.sendText!("Hello")).rejects.toThrow(
"Already responding",
);
const item = channel.events().at(-1);
channel.emit({
type: "conversation.item.added",
item: { id: item.item.id },
});
await vi.waitFor(() =>
expect(channel.events().at(-1).type).toBe("response.create"),
);
channel.emit({
type: "error",
error: {
event_id: channel.events().at(-1).event_id,
code: "conversation_already_has_active_response",
message: "Already responding",
},
});
await rejected;
expect(session.status.type).toBe("running");
expect(track.stop).not.toHaveBeenCalled();
} finally {
warn.mockRestore();
}
});
it("rejects a pending text send on disconnect", async () => {
const { session } = await connected();
const sent = session.sendText!("Hello");
const rejected = expect(sent).rejects.toThrow(
"disconnected before accepting",
);
session.disconnect();
await rejected;
});
it.each(["disconnect", "timeout"])(
"rejects a typed send on %s while waiting for response acceptance",
async (failure) => {
const { session, channel } = await connected();
vi.useFakeTimers();
const rejected = expect(session.sendText!("Hello")).rejects.toThrow(
failure === "disconnect"
? "disconnected before accepting"
: "did not acknowledge",
);
channel.emit({
type: "conversation.item.added",
item: { id: channel.events().at(-1).item.id },
});
await vi.waitFor(() =>
expect(channel.events().at(-1).type).toBe("response.create"),
);
if (failure === "disconnect") session.disconnect();
else await vi.advanceTimersByTimeAsync(10_000);
await rejected;
session.disconnect();
expect(vi.getTimerCount()).toBe(0);
},
);
it("rejects an unacknowledged text send instead of committing it to history", async () => {
const { session } = await connected();
vi.useFakeTimers();
const rejected = expect(session.sendText!("Hello")).rejects.toThrow(
"did not acknowledge",
);
await vi.advanceTimersByTimeAsync(10_000);
await rejected;
});
it("stops microphone capture, playback, timers, and WebRTC exactly once", async () => {
vi.useFakeTimers();
const { session, peer, channel } = await connected();
peer.ontrack?.({ track: { kind: "audio" } });
expect(audio.play).toHaveBeenCalledOnce();
session.disconnect();
session.disconnect();
expect(track.stop).toHaveBeenCalledOnce();
expect(audio.pause).toHaveBeenCalledOnce();
expect(audio.srcObject).toBeNull();
expect(channel.close).toHaveBeenCalledOnce();
expect(peer.close).toHaveBeenCalledOnce();
expect(vi.getTimerCount()).toBe(0);
});
it("mutes the microphone and excludes it from volume measurements", async () => {
vi.useFakeTimers();
const { session } = await connected();
const onVolume = vi.fn();
session.onVolumeChange(onVolume);
await vi.advanceTimersByTimeAsync(100);
expect(onVolume).toHaveBeenLastCalledWith(0.7);
session.mute();
expect(track.enabled).toBe(false);
await vi.advanceTimersByTimeAsync(100);
expect(onVolume).toHaveBeenLastCalledWith(0.4);
session.unmute();
expect(track.enabled).toBe(true);
});
it("applies a mute requested while microphone permission is pending", async () => {
const permission = deferred<typeof stream>();
getUserMedia.mockReturnValue(permission.promise);
const { session, peer } = start();
session.mute();
permission.resolve(stream);
await vi.waitFor(() =>
expect(peer.setRemoteDescription).toHaveBeenCalled(),
);
expect(track.enabled).toBe(false);
});
it("disposes immediately while microphone permission is pending and stops a late stream", async () => {
const permission = deferred<typeof stream>();
getUserMedia.mockReturnValue(permission.promise);
const { session, peer } = start();
session.disconnect();
await vi.waitFor(() => expect(peer.close).toHaveBeenCalledOnce());
permission.resolve(stream);
await vi.waitFor(() => expect(track.stop).toHaveBeenCalledOnce());
expect(exchangeSdp).not.toHaveBeenCalled();
});
it("aborts an in-flight SDP exchange without installing a late answer", async () => {
const answer = deferred<string>();
exchangeSdp.mockReturnValue(answer.promise);
const controller = new AbortController();
const { session, peer } = start(controller.signal);
await vi.waitFor(() => expect(exchangeSdp).toHaveBeenCalled());
const [, options] = exchangeSdp.mock.calls[0]!;
controller.abort();
expect(options.signal.aborted).toBe(true);
expect(track.stop).toHaveBeenCalledOnce();
answer.resolve("late-answer");
await Promise.resolve();
expect(peer.setRemoteDescription).not.toHaveBeenCalled();
expect(session.status).toEqual({ type: "ended", reason: "cancelled" });
});
it("cleans up on permission rejection", async () => {
const error = new Error("Permission denied");
getUserMedia.mockRejectedValue(error);
const { session, peer } = start();
await vi.waitFor(() =>
expect(session.status).toEqual({ type: "ended", reason: "error", error }),
);
expect(peer.close).toHaveBeenCalledOnce();
expect(exchangeSdp).not.toHaveBeenCalled();
});
it("cleans up on signaling rejection", async () => {
const error = new Error("Unauthorized");
exchangeSdp.mockRejectedValue(error);
const { session, peer } = start();
await vi.waitFor(() =>
expect(session.status).toEqual({ type: "ended", reason: "error", error }),
);
expect(track.stop).toHaveBeenCalledOnce();
expect(peer.close).toHaveBeenCalledOnce();
});
it("times out a connection whose channel never opens", async () => {
vi.useFakeTimers();
const { session, peer } = start();
await vi.waitFor(() =>
expect(peer.setRemoteDescription).toHaveBeenCalled(),
);
await vi.advanceTimersByTimeAsync(30_000);
expect(session.status).toMatchObject({ type: "ended", reason: "error" });
expect(track.stop).toHaveBeenCalledOnce();
expect(peer.close).toHaveBeenCalledOnce();
});
it.each(["failed", "closed"])(
"cleans up when the peer becomes %s",
async (state) => {
const { session, peer } = await connected();
peer.connectionState = state;
peer.onconnectionstatechange?.();
expect(session.status).toMatchObject({
type: "ended",
reason: state === "failed" ? "error" : "finished",
});
expect(track.stop).toHaveBeenCalledOnce();
},
);
it("rejects the affected message on a provider error without ending the call", async () => {
const { session, channel } = await connected();
const rejected = expect(session.sendText!("Hello")).rejects.toThrow();
const id = channel.events().at(-1).event_id;
channel.emit({
type: "error",
error: { event_id: id, message: "Invalid item" },
});
await rejected;
expect(session.status.type).toBe("running");
expect(track.stop).not.toHaveBeenCalled();
});
it("ignores malformed and unrelated events", async () => {
const { session, channel } = await connected();
const onTranscript = vi.fn();
session.onTranscript(onTranscript);
channel.onmessage?.({ data: "not-json" });
for (const event of [
null,
[],
{},
{ type: "session.updated" },
{ type: "response.output_audio_transcript.delta" },
{ type: "response.output_audio_transcript.done", item_id: "a" },
{ type: "conversation.item.input_audio_transcription.completed" },
])
channel.emit(event);
expect(onTranscript).not.toHaveBeenCalled();
expect(session.status.type).toBe("running");
});
it("reports unsupported environments through session status", async () => {
vi.stubGlobal("RTCPeerConnection", undefined);
const session = new OpenAIRealtimeVoiceAdapter({ exchangeSdp }).connect({});
await vi.waitFor(() =>
expect(session.status).toMatchObject({ type: "ended", reason: "error" }),
);
expect(getUserMedia).not.toHaveBeenCalled();
});
it("does not restart the volume meter when a status listener disconnects", async () => {
vi.useFakeTimers();
const { session, peer, channel } = start();
await vi.waitFor(() =>
expect(peer.setRemoteDescription).toHaveBeenCalled(),
);
session.onStatusChange((status) => {
if (status.type === "running") session.disconnect();
});
channel.open();
expect(session.status).toEqual({ type: "ended", reason: "cancelled" });
expect(vi.getTimerCount()).toBe(0);
});
it("ends the session when remote audio cannot be played", async () => {
const { session, peer } = await connected();
const error = new Error("Autoplay blocked");
audio.play.mockRejectedValue(error);
peer.ontrack?.({ track: { kind: "audio" } });
await vi.waitFor(() =>
expect(session.status).toEqual({ type: "ended", reason: "error", error }),
);
expect(track.stop).toHaveBeenCalledOnce();
});
it("does not allocate media for an already-aborted connection", () => {
const session = new OpenAIRealtimeVoiceAdapter({ exchangeSdp }).connect({
abortSignal: AbortSignal.abort(),
});
expect(session.status).toEqual({ type: "ended", reason: "cancelled" });
expect(Peer.instances).toHaveLength(0);
expect(getUserMedia).not.toHaveBeenCalled();
});
});