@trillet-ai/web-sdk
Version:
Trillet Web SDK for real-time audio communication with AI agents
239 lines (238 loc) • 10.2 kB
JavaScript
import { EventEmitter } from 'eventemitter3';
import { RemoteAudioTrack, Room, RoomEvent, Track, createAudioAnalyser, LocalAudioTrack } from 'livekit-client';
const decoder = new TextDecoder();
// Internal SDK class
export class TrilletWebSDK extends EventEmitter {
constructor() {
super();
this.isConnected = false;
// Audio state tracking
this.isAssistantSpeaking = false;
}
async initializeCall(config) {
var _a, _b, _c, _d;
try {
this.room = new Room({
audioCaptureDefaults: {
autoGainControl: true,
echoCancellation: true,
noiseSuppression: true,
channelCount: 1,
deviceId: (_a = config.audioSettings) === null || _a === void 0 ? void 0 : _a.inputDeviceId,
sampleRate: (_b = config.audioSettings) === null || _b === void 0 ? void 0 : _b.sampleRate,
},
audioOutput: {
deviceId: (_c = config.audioSettings) === null || _c === void 0 ? void 0 : _c.outputDeviceId,
},
});
this.setupEventHandlers(config);
this.room.prepareConnection(config.wsUrl, config.token);
await this.room.connect(config.wsUrl, config.token);
console.log('Connected to Trillet room:', this.room.name);
if (config.mode !== 'text') {
await this.room.localParticipant.setMicrophoneEnabled(true);
// Apply Krisp noise filter if enabled
if ((_d = config.features) === null || _d === void 0 ? void 0 : _d.enableKrispNoiseFilter) {
await this.setupKrispNoiseFilter();
}
}
this.isConnected = true;
this.emit('connected');
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Failed to initialize call';
this.emit('error', errorMessage);
console.error('Call initialization failed:', error);
this.endCall();
throw error; // Re-throw to let TrilletAgent handle it
}
}
async enableAudioPlayback() {
var _a;
await ((_a = this.room) === null || _a === void 0 ? void 0 : _a.startAudio());
}
endCall() {
var _a;
if (!this.isConnected)
return;
this.isConnected = false;
this.emit('disconnected');
(_a = this.room) === null || _a === void 0 ? void 0 : _a.disconnect();
this.isAssistantSpeaking = false;
if (this.audioAnalyzer) {
this.audioAnalyzer.cleanup();
this.audioAnalyzer = undefined;
}
if (this.audioFrameCapture) {
cancelAnimationFrame(this.audioFrameCapture);
this.audioFrameCapture = undefined;
}
// Cleanup Krisp processor
if (this.krispProcessor) {
this.krispProcessor = undefined;
}
this.room = undefined;
}
toggleMicrophone(enabled) {
if (this.isConnected && this.room) {
this.room.localParticipant.setMicrophoneEnabled(enabled);
}
}
async setupKrispNoiseFilter() {
var _a;
if (!this.room)
return;
try {
// Dynamically import Krisp noise filter
const { KrispNoiseFilter, isKrispNoiseFilterSupported } = await import('@livekit/krisp-noise-filter');
// Check browser compatibility
if (!isKrispNoiseFilterSupported()) {
console.warn('Krisp noise filter is not supported on this browser');
return;
}
// Get the local microphone track
const microphoneTrack = (_a = this.room.localParticipant.getTrackPublication(Track.Source.Microphone)) === null || _a === void 0 ? void 0 : _a.track;
if (microphoneTrack && microphoneTrack instanceof LocalAudioTrack) {
// Initialize the Krisp processor
this.krispProcessor = KrispNoiseFilter();
await microphoneTrack.setProcessor(this.krispProcessor);
// Enable the noise filter
await this.krispProcessor.setEnabled(true);
console.log('Krisp noise filter enabled successfully');
}
else {
console.warn('Could not find local microphone track to apply Krisp filter');
}
}
catch (error) {
console.error('Failed to setup Krisp noise filter:', error);
}
}
captureAudio() {
if (!this.isConnected || !this.audioAnalyzer)
return;
const bufferSize = this.audioAnalyzer.analyser.fftSize;
const audioData = new Float32Array(bufferSize);
this.audioAnalyzer.analyser.getFloatTimeDomainData(audioData);
this.emit('audioData', audioData);
this.audioFrameCapture = requestAnimationFrame(() => this.captureAudio());
}
setupEventHandlers(config) {
let wasAgentSpeaking = false;
if (!this.room)
return;
// Room events
this.room.on(RoomEvent.ParticipantDisconnected, (participant) => {
if ((participant === null || participant === void 0 ? void 0 : participant.identity) === 'assistant') {
this.endCall();
}
});
this.room.on(RoomEvent.Disconnected, () => {
if (this.isConnected)
this.endCall();
});
this.room.registerTextStreamHandler('lk.transcription', async (reader, info) => {
var _a, _b;
const text = await reader.readAll();
const attributes = ((_a = reader.info) === null || _a === void 0 ? void 0 : _a.attributes) || {};
const isFinal = attributes['lk.transcription_final'] === 'true';
if (!info)
return;
const participant = (_b = this.room) === null || _b === void 0 ? void 0 : _b.getParticipantByIdentity(info.identity);
if (!participant)
return;
this.emit('transcript', { text, isFinal }, participant);
});
// Audio events
this.room.on(RoomEvent.TrackSubscribed, (track, publication) => {
var _a;
if (track.kind === Track.Kind.Audio) {
if (publication.trackName === 'assistant_audio' && track instanceof RemoteAudioTrack && ((_a = config.features) === null || _a === void 0 ? void 0 : _a.enableRawAudio)) {
const analyzer = createAudioAnalyser(track);
this.audioAnalyzer = {
calculateVolume: analyzer.calculateVolume,
analyser: analyzer.analyser,
cleanup: analyzer.cleanup,
};
this.audioFrameCapture = requestAnimationFrame(() => this.captureAudio());
}
track.attach();
}
});
// Data events
this.room.on(RoomEvent.DataReceived, (payload, participant) => {
if (!(participant === null || participant === void 0 ? void 0 : participant.identity.startsWith('agent')))
return;
try {
const data = JSON.parse(decoder.decode(payload));
this.handleDataEvent(data, participant);
}
catch (error) {
console.error('Failed to process received data:', error);
}
});
this.room.on(RoomEvent.ActiveSpeakersChanged, (speakers) => {
const isAgentSpeaking = speakers.some((p) => p.identity.startsWith('agent'));
if (isAgentSpeaking && !wasAgentSpeaking) {
this.emit('assistantStartedSpeaking');
}
else if (!isAgentSpeaking && wasAgentSpeaking) {
this.emit('assistantStoppedSpeaking');
}
wasAgentSpeaking = isAgentSpeaking;
});
// Handle RPC events if available
// Note: LiveKit may not have specific RPC events, but we can monitor participant events
this.room.on(RoomEvent.ParticipantConnected, (participant) => {
console.log('Participant connected:', participant.identity);
if (participant.identity.startsWith('agent')) {
console.log('Agent connected, RPC communication available');
}
});
}
handleDataEvent(event, participant) {
switch (event.type) {
case 'status':
this.emit('status', event);
break;
case 'metadata':
this.emit('metadata', event);
break;
case 'transcript':
// this is for agent-generated transcripts, which we are overriding with lk.transcription
// this.emit('transcript', event.data, participant);
break;
case 'llm_call':
this.emit('toolStarted', { name: event.name, args: event.arguments });
break;
case 'llm_call_response':
this.emit('toolCompleted', {
name: event.details.api_name,
result: event.details.api_response,
isError: event.details.status >= 400,
});
break;
case 'error':
this.emit('toolCompleted', {
name: event.name,
result: event.details,
isError: true,
});
break;
case 'assistant_speaking_started':
this.isAssistantSpeaking = true;
this.emit('assistantStartedSpeaking');
break;
case 'assistant_speaking_ended':
this.isAssistantSpeaking = false;
this.emit('assistantStoppedSpeaking');
break;
case 'agent_message':
this.emit('message', event);
break;
}
}
getRoom() {
return this.room;
}
}