@airsurfer09/web-handsfree
Version:
A React package for integrating Convai's AI-powered voice assistants with LiveKit for real-time audio/video conversations
348 lines • 15.1 kB
JavaScript
import { useState, useCallback, useMemo, useEffect } from "react";
import { Room, RoomEvent } from "livekit-client";
import { useUserTextMessageSender } from "./useUserTextMessageSender";
import { useTriggerMessageSender } from "./useTriggerMessageSender";
import { useTemplateKeysUpdater } from "./useTemplateKeysUpdater";
import { useDynamicInfoUpdater } from "./useDynamicInfoUpdater";
import { useMessageHandler } from "./useMessageHandler";
import { useAudioControls } from "./useAudioControls";
import { useVideoControls } from "./useVideoControls";
import { useScreenShare } from "./useScreenShare";
import { logger } from "../utils/logger";
import { useTtsToggle } from "./useTtsToggle";
/**
* Main hook for managing Convai client connection and functionality.
*
* Provides a complete interface for connecting to Convai's AI-powered voice assistants,
* managing real-time audio/video conversations, and handling various message types.
*
* @returns {ConvaiClient & { activity: string; chatMessages: ChatMessage[] }} Complete client interface with connection state, methods, and message history
*
* @example
* ```tsx
* function App() {
* const convaiClient = useConvaiClient();
*
* const connect = async () => {
* await convaiClient.connect({
* apiKey: 'your-api-key',
* characterId: 'your-character-id',
* enableVideo: true,
* enableAudio: true
* });
* };
*
* return (
* <div>
* <button onClick={connect}>Connect</button>
* <p>Status: {convaiClient.state.isConnected ? 'Connected' : 'Disconnected'}</p>
* </div>
* );
* }
* ```
*/
export const useConvaiClient = () => {
const [room] = useState(new Room());
const [isConnected, setIsConnected] = useState(false);
const [isConnecting, setIsConnecting] = useState(false);
const [isBotReady, setIsBotReady] = useState(false); // Bot is ready to receive messages
const [activity, setActivity] = useState("Idle");
const [chatMessages, setChatMessages] = useState([]);
const [userTranscription, setUserTranscription] = useState("");
const [isBotResponding, setIsBotResponding] = useState(false);
const [participantSid, setParticipantSid] = useState("");
const [isSpeaking, setIsSpeaking] = useState(false);
const [characterSessionId, setCharacterSessionId] = useState("-1");
// Default to Noisy Mode for optimal interruption handling
const [audioSettings, setAudioSettings] = useState({
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
sampleRate: 48000,
channelCount: 1,
});
// Initialize message sender hooks
const { sendUserTextMessage } = useUserTextMessageSender(room, participantSid);
const { sendTriggerMessage } = useTriggerMessageSender(room, participantSid);
const { updateTemplateKeys } = useTemplateKeysUpdater(room, participantSid);
const { updateDynamicInfo } = useDynamicInfoUpdater(room, participantSid);
const { toggleTts } = useTtsToggle(room);
// Initialize message handler
const { setupMessageListener } = useMessageHandler(room, setChatMessages, setUserTranscription, setIsBotResponding, isBotResponding, setIsSpeaking, setIsConnected, // Pass callback to update connection state when bot-ready is received
setActivity, // Pass callback to update activity state when bot-ready is received
setIsBotReady);
// Initialize media controls
const audioControls = useAudioControls(room);
const videoControls = useVideoControls(room);
const screenShareControls = useScreenShare(room);
// Data message listener effect
useEffect(() => {
const cleanup = setupMessageListener();
return cleanup;
}, [setupMessageListener]);
// Listen for room disconnection events (PC manager closed, network issues, etc.)
useEffect(() => {
if (!room)
return;
const handleDisconnected = () => {
logger.log("Room disconnected (PC manager closed or connection lost)");
setIsConnected(false);
setIsBotReady(false); // Reset bot-ready state on disconnect
setActivity("Disconnected");
setChatMessages([]);
setIsBotResponding(false);
};
const handleConnectionStateChanged = (state) => {
logger.log("Room connection state changed:", state);
if (state === "disconnected") {
setIsConnected(false);
setActivity("Disconnected");
setChatMessages([]);
setIsBotResponding(false);
}
else if (state === "connected") {
setIsConnected(true);
setIsBotReady(false); // Room connected but bot not ready yet
setActivity("Connected");
}
else if (state === "connecting") {
setIsConnecting(true);
setIsBotReady(false); // Reset bot-ready when reconnecting
setActivity("Connecting");
}
};
// Listen for disconnection events
room.on(RoomEvent.Disconnected, handleDisconnected);
room.on(RoomEvent.ConnectionStateChanged, handleConnectionStateChanged);
return () => {
room.off(RoomEvent.Disconnected, handleDisconnected);
room.off(RoomEvent.ConnectionStateChanged, handleConnectionStateChanged);
};
}, [room]);
// Create client state
const state = useMemo(() => ({
isConnected,
isConnecting,
isListening: false,
isThinking: false,
isSpeaking,
agentState: "disconnected",
}), [isConnected, isConnecting, isSpeaking]);
// Connect function
const connect = useCallback(async (config) => {
if (!config.apiKey ||
!config.characterId ||
!config.url) {
throw new Error("apiKey, characterId, and url are required");
}
setIsConnecting(true);
setActivity("Connecting...");
try {
// Prepare request body with required parameters
const requestBody = {
character_id: config.characterId,
transport: "livekit",
connection_type: "video",
llm_provider: "dynamic",
default_tts_enabled: true, // Set TTS enabled by default
...(characterSessionId &&
characterSessionId !== "-1" && {
character_session_id: characterSessionId,
}), // Resume with existing session if available (skip if -1)
...(config.actionConfig && { action_config: config.actionConfig }),
};
logger.log("Connecting to Convai with config:", {
url: config.url,
characterId: config.characterId,
llmProvider: "dynamic",
hasActionConfig: !!config.actionConfig,
characterSessionId: characterSessionId === "-1" ? "new session" : characterSessionId,
});
// Call Core Service API
const response = await fetch(`${config.url}/connect`, {
method: "POST",
headers: {
"x-api-key": config.apiKey,
"Content-Type": "application/json",
},
body: JSON.stringify(requestBody),
});
if (!response.ok) {
const errorText = await response.text();
logger.error("API Error Response:", {
status: response.status,
statusText: response.statusText,
body: errorText,
});
let errorMessage = `HTTP error! status: ${response.status}`;
try {
const errorData = JSON.parse(errorText);
if (errorData.message) {
errorMessage = errorData.message;
}
else if (errorData.error) {
errorMessage = errorData.error;
}
else if (errorData.detail) {
errorMessage = errorData.detail;
}
}
catch (e) {
// If we can't parse the error, use the raw text
if (errorText) {
errorMessage = `${errorMessage}: ${errorText}`;
}
}
throw new Error(errorMessage);
}
const connectionData = await response.json();
logger.log("Connection data received:", connectionData);
// Capture character_session_id from connection response
if (connectionData.character_session_id) {
setCharacterSessionId(connectionData.character_session_id);
logger.log("Character Session ID:", connectionData.character_session_id);
}
// Update audio settings if provided
if (config.audioSettings) {
setAudioSettings(config.audioSettings);
}
// Connect to LiveKit room
await room.connect(connectionData.room_url, connectionData.token, {
rtcConfig: {
iceTransportPolicy: "relay",
},
});
// Enable mic/camera based on config
if (config.enableAudio !== false) {
const currentSettings = config.audioSettings || audioSettings;
await room.localParticipant.setMicrophoneEnabled(true, {
echoCancellation: currentSettings.echoCancellation,
noiseSuppression: currentSettings.noiseSuppression,
autoGainControl: currentSettings.autoGainControl,
sampleRate: currentSettings.sampleRate,
channelCount: currentSettings.channelCount,
});
}
if (config.enableVideo !== false) {
await room.localParticipant.setCameraEnabled(true);
}
// Capture participant SID
setParticipantSid(room.localParticipant.sid);
setIsConnected(true);
setActivity("Connected");
logger.log("Connected to room:", connectionData.room_name);
logger.log("Session ID:", connectionData.session_id);
logger.log("Participant SID:", room.localParticipant.sid);
}
catch (error) {
logger.error("Connection failed:", error);
setIsConnected(false);
setActivity("Connection failed");
throw error;
}
finally {
setIsConnecting(false);
}
}, [room, characterSessionId]);
// Disconnect function
const disconnect = useCallback(async () => {
if (room && room.state !== "disconnected") {
try {
await room.disconnect();
setIsConnected(false);
setIsBotReady(false); // Reset bot-ready state on disconnect
setActivity("Disconnected");
setChatMessages([]);
setIsBotResponding(false);
// Keep characterSessionId for reconnection - only clear on explicit reset
logger.log("Disconnected from room");
}
catch (error) {
// Handle disconnect errors gracefully
// ConnectionError "Client initiated disconnect" is expected in some cases
if (error instanceof Error && error.message.includes("disconnect")) {
logger.debug("Disconnect completed (expected):", error.message);
}
else {
logger.error("Disconnect error:", error);
}
// Still update state even if disconnect throws
setIsConnected(false);
setIsBotReady(false); // Reset bot-ready state on disconnect
setActivity("Disconnected");
setChatMessages([]);
setIsBotResponding(false);
// Keep characterSessionId for reconnection - only clear on explicit reset
}
}
}, [room]);
// Reset session function - clears character_session_id for a fresh session
const resetSession = useCallback(() => {
setCharacterSessionId("-1");
logger.log("Character session reset - next connection will create a new session");
}, []);
// Send RTVI trigger
const sendRTVI = useCallback((triggerName, message) => {
if (room && room.localParticipant) {
const data = {
type: "rtvi-trigger",
trigger: triggerName,
message: message || "",
};
room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(data)), {
reliable: true,
});
}
}, [room]);
// Update audio settings dynamically
const updateAudioSettings = useCallback(async (newSettings) => {
const updatedSettings = { ...audioSettings, ...newSettings };
setAudioSettings(updatedSettings);
// If connected and audio is enabled, restart the microphone with new settings
if (room && room.state === "connected" && audioControls.isAudioEnabled) {
try {
logger.log("Updating audio settings:", updatedSettings);
// Disable and re-enable microphone with new settings
await room.localParticipant.setMicrophoneEnabled(false);
await room.localParticipant.setMicrophoneEnabled(true, {
echoCancellation: updatedSettings.echoCancellation,
noiseSuppression: updatedSettings.noiseSuppression,
autoGainControl: updatedSettings.autoGainControl,
sampleRate: updatedSettings.sampleRate,
channelCount: updatedSettings.channelCount,
});
logger.log("Audio settings updated successfully");
}
catch (error) {
logger.error("Failed to update audio settings:", error);
throw error;
}
}
}, [room, audioSettings, audioControls.isAudioEnabled]);
// Return client object with activity and chat messages
return {
state,
connect,
disconnect,
resetSession,
room,
videoTrack: null,
audioTrack: null,
sendUserTextMessage,
sendTriggerMessage,
updateTemplateKeys,
updateDynamicInfo,
activity,
chatMessages,
userTranscription,
characterSessionId,
isBotReady,
audioControls,
videoControls,
screenShareControls,
toggleTts,
audioSettings,
updateAudioSettings,
};
};
//# sourceMappingURL=useConvaiClient.js.map