@airsurfer09/web-handsfree/dist/hooks/useMessageHandler.js

A React package for integrating Convai's AI-powered voice assistants with LiveKit for real-time audio/video conversations

import { useCallback } from "react";
import { RoomEvent } from "livekit-client";
import { logger } from "../utils/logger";
/**
 * Hook for handling incoming messages from Convai.
 *
 * Sets up data message listeners to process various message types
 * from the AI assistant and updates the chat message history.
 *
 * @param {Room} room - LiveKit room instance
 * @param {Function} setChatMessages - Function to update chat messages state
 * @returns {Function} Cleanup function to remove message listeners
 *
 * @example
 * ```tsx
 * function ChatComponent() {
 *   const [chatMessages, setChatMessages] = useState([]);
 *   const cleanup = useMessageHandler(room, setChatMessages);
 *
 *   useEffect(() => {
 *     return cleanup;
 *   }, []);
 * }
 * ```
 */
export const useMessageHandler = (room, setChatMessages, onUserTranscription, setIsBotResponding, isBotResponding, setIsSpeaking, onConnectionStateChange, setActivity, setIsBotReady) => {
    const handleDataReceived = useCallback((payload, participant) => {
        try {
            // Decode bytes to string
            const decoder = new TextDecoder();
            const messageString = decoder.decode(payload);
            // Parse JSON
            const messageData = JSON.parse(messageString);
            logger.log("📨 Data message received:", messageData);
            // Extract and categorize messages for chat display
            const timestamp = new Date().toISOString();
            const messageId = `${messageData.type}-${Date.now()}-${Math.random()}`;
            // Debug: Log all incoming messages to see what's being received
            logger.log("%c📨 INCOMING MESSAGE%c", "background: #673ab7; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", {
                type: messageData.type,
                data: messageData.data,
                message: messageData.message,
                text: messageData.text,
                content: messageData.content,
                fullMessage: messageData,
            });
            // Handle different message types
            switch (messageData.type) {
                // User text messages - Skip these, only allow user-transcription
                case "user_text_message":
                case "user-llm-text":
                case "text-input":
                case "user-input":
                case "text-message":
                case "chat-message":
                case "input-text":
                case "message":
                    // Skip user text messages - only allow user-transcription
                    logger.log("%c🚫 SKIPPED USER TEXT%c %c" +
                        (messageData.data?.text ||
                            messageData.message ||
                            messageData.text ||
                            messageData.content ||
                            "unknown"), "background: #ff5722; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #ff5722; font-weight: bold;", "color: #f7f1e3;");
                    break;
                // Bot Ready - Bot is ready, connection should be true
                case "bot-ready":
                    logger.log("%c✅ BOT READY%c", "background: #4caf50; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;");
                    // Set connection state to true when bot is ready
                    if (onConnectionStateChange) {
                        onConnectionStateChange(true);
                    }
                    // Set bot-ready state to true - bot can now receive messages
                    if (setIsBotReady) {
                        setIsBotReady(true);
                    }
                    // Update activity state
                    if (setActivity) {
                        setActivity("Connected");
                    }
                    break;
                // Bot LLM Started - Begin streaming response
                case "bot-llm-started":
                    logger.log("%c🤖 BOT RESPONSE STARTED%c", "background: #4caf50; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;");
                    if (setIsBotResponding) {
                        setIsBotResponding(true);
                    }
                    // Add initial streaming message to chat
                    const streamingMessage = {
                        id: messageId,
                        type: "bot-llm-text",
                        content: "",
                        timestamp: timestamp,
                        isFinal: true,
                    };
                    setChatMessages((prev) => [...prev, streamingMessage]);
                    break;
                // Bot LLM Text Messages - Streaming chunks
                case "bot-llm-text":
                    if (messageData.data?.text) {
                        const newChunk = messageData.data.text;
                        logger.log("%c🤖 BOT CHUNK%c %c" + newChunk, "background: #45b7d1; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #45b7d1; font-weight: bold;", "color: #f7f1e3;");
                        // Update the streaming message in chat
                        setChatMessages((prevMessages) => {
                            const lastMessage = prevMessages[prevMessages.length - 1];
                            if (lastMessage && lastMessage.isFinal) {
                                return [
                                    ...prevMessages.slice(0, -1),
                                    { ...lastMessage, content: lastMessage.content + newChunk }
                                ];
                            }
                            else {
                                // If no streaming message exists yet, create one
                                const streamingMessage = {
                                    id: `${messageData.type}-${Date.now()}-${Math.random()}`,
                                    type: "bot-llm-text",
                                    content: newChunk,
                                    timestamp: new Date().toISOString(),
                                    isFinal: true,
                                };
                                return [...prevMessages, streamingMessage];
                            }
                        });
                    }
                    break;
                // Bot LLM Stopped - Complete streaming response
                case "bot-llm-stopped":
                    // Get the most up-to-date content from the current message in chat
                    setChatMessages((prevMessages) => {
                        const lastMessage = prevMessages[prevMessages.length - 1];
                        if (lastMessage && lastMessage.isFinal) {
                            const finalContent = lastMessage.content;
                            logger.log("%c🤖 BOT RESPONSE COMPLETED%c %c" + finalContent, "background: #ff9800; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #ff9800; font-weight: bold;", "color: #f7f1e3;");
                            return [
                                ...prevMessages.slice(0, -1),
                                { ...lastMessage, content: finalContent, isFinal: false }
                            ];
                        }
                        return prevMessages;
                    });
                    // Clear streaming state
                    if (setIsBotResponding) {
                        setIsBotResponding(false);
                    }
                    break;
                // User Transcription Messages
                case "user-transcription":
                    if (messageData.data?.text) {
                        const messageContent = messageData.data.text;
                        if (messageData.data?.final) {
                            // Final transcription - add to chat messages
                            // Check if this transcription already exists to prevent duplicates
                            setChatMessages((prev) => {
                                const alreadyExists = prev.some((msg) => msg.type === "user-transcription" &&
                                    msg.content.toLowerCase().trim() ===
                                        messageContent.toLowerCase().trim());
                                if (alreadyExists) {
                                    logger.log("%c🔄 DUPLICATE SKIPPED%c UserTranscription: %c" +
                                        messageContent, "background: #ff6b6b; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #ff6b6b; font-weight: bold;", "color: #ffa726; font-weight: bold;");
                                    return prev;
                                }
                                const userMessage = {
                                    id: messageId,
                                    type: "user-transcription",
                                    content: messageContent,
                                    timestamp: timestamp,
                                };
                                logger.log("%c🎤 USER TRANSCRIPTION (FINAL)%c %c" + messageContent, "background: #ffa726; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #ffa726; font-weight: bold;", "color: #f7f1e3;");
                                return [...prev, userMessage];
                            });
                            // Clear live transcription when final
                            if (onUserTranscription) {
                                onUserTranscription('');
                            }
                        }
                        else {
                            // Non-final transcription - show in input field
                            if (onUserTranscription) {
                                onUserTranscription(messageContent);
                                logger.log("%c🎤 LIVE TRANSCRIPTION%c %c" + messageContent, "background: #4caf50; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #4caf50; font-weight: bold;", "color: #f7f1e3;");
                            }
                        }
                    }
                    break;
                // Bot Emotion Messages
                case "bot-emotion":
                    if (messageData.data?.emotion) {
                        const emotionMessage = {
                            id: messageId,
                            type: "bot-emotion",
                            content: `${messageData.data.emotion} (scale: ${messageData.data.scale || 1})`,
                            timestamp,
                        };
                        setChatMessages((prev) => [...prev, emotionMessage]);
                        logger.log("%c😊 CONVAI EMOTION%c %c" +
                            messageData.data.emotion +
                            "%c (scale: %c" +
                            (messageData.data.scale || 1) +
                            "%c)", "background: #e91e63; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #e91e63; font-weight: bold;", "color: #f7f1e3; font-weight: bold;", "color: #e91e63;", "color: #ffeb3b; font-weight: bold;", "color: #e91e63;");
                    }
                    break;
                // Bot Started Speaking
                case "bot-started-speaking":
                    logger.log("%c🔊 BOT STARTED SPEAKING%c", "background: #10b981; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;");
                    if (setIsSpeaking) {
                        setIsSpeaking(true);
                    }
                    break;
                // Bot Stopped Speaking
                case "bot-stopped-speaking":
                    logger.log("%c🔇 BOT STOPPED SPEAKING%c", "background: #6b7280; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;");
                    if (setIsSpeaking) {
                        setIsSpeaking(false);
                    }
                    break;
                // Action Response Messages
                case "action-response":
                    if (messageData.data?.actions) {
                        const actionMessage = {
                            id: messageId,
                            type: "action",
                            content: `Actions: ${messageData.data.actions.join(", ")}`,
                            timestamp,
                        };
                        setChatMessages((prev) => [...prev, actionMessage]);
                        logger.log("%c🎭 ACTION RESPONSE%c %c" +
                            messageData.data.actions.join(", "), "background: #9c27b0; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #9c27b0; font-weight: bold;", "color: #f7f1e3;");
                    }
                    break;
                // Behavior Tree Response Messages
                case "behavior-tree-response":
                    if (messageData.data?.narrative_section_id) {
                        const behaviorMessage = {
                            id: messageId,
                            type: "behavior-tree",
                            content: `Narrative Section: ${messageData.data.narrative_section_id}`,
                            timestamp,
                        };
                        setChatMessages((prev) => [...prev, behaviorMessage]);
                        logger.log("%c🌳 BEHAVIOR TREE%c %c" +
                            messageData.data.narrative_section_id, "background: #4caf50; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #4caf50; font-weight: bold;", "color: #f7f1e3;");
                        logger.log("%c🌳 BT DETAILS%c", "background: #4caf50; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", {
                            narrativeSection: messageData.data.narrative_section_id,
                            btCode: messageData.data.bt_code,
                            btConstants: messageData.data.bt_constants,
                        });
                    }
                    break;
                // Moderation Response Messages
                case "moderation-response":
                    logger.log("%c🛡️ MODERATION%c %c" + (messageData.data?.result || "unknown"), "background: #ff9800; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #ff9800; font-weight: bold;", "color: #f7f1e3;");
                    logger.log("%c🛡️ MODERATION DETAILS%c", "background: #ff9800; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", {
                        result: messageData.data?.result,
                        userInput: messageData.data?.user_input,
                        reason: messageData.data?.reason,
                    });
                    break;
                // Trigger Messages
                case "trigger-message":
                    logger.log("%c🎯 TRIGGER MESSAGE%c", "background: #2196f3; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", messageData.data);
                    break;
                // Template Keys Updates
                case "update-template-keys":
                    logger.log("%c🔑 TEMPLATE KEYS%c", "background: #607d8b; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", messageData.data);
                    break;
                // Dynamic Info Updates
                case "update-dynamic-info":
                    logger.log("%c🔄 DYNAMIC INFO%c", "background: #795548; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", messageData.data);
                    break;
            }
            // Log any unhandled message types
            if (![
                "user_text_message",
                "user-llm-text",
                "text-input",
                "user-input",
                "text-message",
                "chat-message",
                "input-text",
                "message",
                "bot-llm-text",
                "bot-llm-started",
                "bot-llm-stopped",
                "user-transcription",
                "bot-emotion",
                "bot-started-speaking",
                "bot-stopped-speaking",
                "action-response",
                "behavior-tree-response",
                "moderation-response",
                "trigger-message",
                "update-template-keys",
                "update-dynamic-info",
            ].includes(messageData.type)) {
                logger.log("%c🔍 UNHANDLED MESSAGE TYPE%c %c" + messageData.type, "background: #f44336; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", "color: #f44336; font-weight: bold;", "color: #f7f1e3;");
                logger.log("%c🔍 UNHANDLED DATA%c", "background: #f44336; color: white; padding: 2px 6px; border-radius: 3px; font-weight: bold;", messageData);
            }
        }
        catch (error) {
            logger.error("Failed to parse data message:", error);
        }
    }, [setChatMessages, onUserTranscription, setIsBotResponding, isBotResponding, setIsSpeaking, onConnectionStateChange, setActivity]);
    const setupMessageListener = useCallback(() => {
        if (!room)
            return;
        // Listen for data received events
        room.on(RoomEvent.DataReceived, handleDataReceived);
        // Return cleanup function
        return () => {
            room.off(RoomEvent.DataReceived, handleDataReceived);
        };
    }, [room, handleDataReceived]);
    return {
        setupMessageListener,
    };
};
//# sourceMappingURL=useMessageHandler.js.map