UNPKG

@marcosremar/cabecao

Version:

Modern React 3D avatar component with chat and lip-sync capabilities

2,025 lines (1,935 loc) 168 kB
/* eslint-disable */ 'use strict'; Object.defineProperty(exports, '__esModule', { value: true }); var react = require('react'); var drei = require('@react-three/drei'); var fiber = require('@react-three/fiber'); var leva = require('leva'); var socket_ioClient = require('socket.io-client'); var jsxRuntime = require('react/jsx-runtime'); var THREE = require('three'); var vadReact = require('@ricky0123/vad-react'); function _interopNamespaceDefault(e) { var n = Object.create(null); if (e) { Object.keys(e).forEach(function (k) { if (k !== 'default') { var d = Object.getOwnPropertyDescriptor(e, k); Object.defineProperty(n, k, d.get ? d : { enumerable: true, get: function () { return e[k]; } }); } }); } n.default = e; return Object.freeze(n); } var THREE__namespace = /*#__PURE__*/_interopNamespaceDefault(THREE); /** * Hook para gerenciar fila de reprodução de áudio * Segue princípios SOLID e DRY */ const useAudioQueue = () => { // Estado da fila const [queue, setQueue] = react.useState([]); const [currentMessage, setCurrentMessage] = react.useState(null); const [isProcessing, setIsProcessing] = react.useState(false); // Referências para controle const processingRef = react.useRef(false); const queueRef = react.useRef([]); // Sincronizar ref com state para evitar closures react.useEffect(() => { queueRef.current = queue; }, [queue]); /** * Adiciona mensagem à fila */ const addToQueue = react.useCallback(message => { console.log('🎵 [AudioQueue] Adicionando à fila:', { text: message.text?.substring(0, 30), sequenceNumber: message.sequenceNumber, queueSizeBefore: queueRef.current.length }); setQueue(prev => { const newQueue = [...prev, message]; console.log(`📊 [AudioQueue] Tamanho da fila após adicionar: ${newQueue.length}`); return newQueue; }); }, []); /** * Remove mensagem da fila após reprodução */ const removeFromQueue = react.useCallback(audioId => { console.log('🗑️ [AudioQueue] Removendo áudio reproduzido:', audioId?.substring(0, 50)); setQueue(prev => { const newQueue = prev.filter(msg => msg.audio !== audioId); console.log(`📊 [AudioQueue] Tamanho da fila após remover: ${newQueue.length}`); return newQueue; }); // Limpar mensagem atual setCurrentMessage(null); setIsProcessing(false); processingRef.current = false; }, []); /** * Processa próxima mensagem da fila */ const processNext = react.useCallback(() => { // Evitar processamento paralelo if (processingRef.current || queueRef.current.length === 0) { return; } const next = queueRef.current[0]; if (!next) return; console.log('▶️ [AudioQueue] Processando próxima mensagem:', { text: next.text?.substring(0, 30), queueSize: queueRef.current.length }); processingRef.current = true; setIsProcessing(true); setCurrentMessage(next); }, []); /** * Monitora fila e processa automaticamente */ react.useEffect(() => { // Se não está processando e tem itens na fila if (!processingRef.current && queue.length > 0 && !currentMessage) { console.log('🔄 [AudioQueue] Fila não vazia, iniciando processamento'); processNext(); } }, [queue, currentMessage, processNext]); /** * Limpa a fila */ const clearQueue = react.useCallback(() => { console.log('🧹 [AudioQueue] Limpando fila'); setQueue([]); setCurrentMessage(null); setIsProcessing(false); processingRef.current = false; }, []); /** * Retorna informações da fila */ const getQueueInfo = react.useCallback(() => ({ size: queue.length, isProcessing: processingRef.current, currentMessage, messages: queue.map(m => ({ text: m.text, sequenceNumber: m.sequenceNumber })) }), [queue, currentMessage]); return { // Estado queue, currentMessage, isProcessing, // Ações addToQueue, removeFromQueue, clearQueue, processNext, // Informações getQueueInfo, queueSize: queue.length }; }; /** * Hook para gerenciar buffer de chunks ordenados * Garante que chunks sejam processados na ordem correta */ const useChunkBuffer = () => { const [buffer, setBuffer] = react.useState([]); const [nextExpectedSequence, setNextExpectedSequence] = react.useState(0); /** * Reseta o buffer para nova sequência */ const resetBuffer = react.useCallback(() => { console.log('🔄 [ChunkBuffer] Resetando buffer'); setBuffer([]); setNextExpectedSequence(0); }, []); /** * Adiciona chunk ao buffer e retorna chunks prontos para processar */ const addChunk = react.useCallback(chunk => { console.log(`📦 [ChunkBuffer] Recebido chunk ${chunk.sequenceNumber}`, { expected: nextExpectedSequence, isInOrder: chunk.sequenceNumber === nextExpectedSequence }); // Se é o primeiro chunk, reseta if (chunk.sequenceNumber === 0) { resetBuffer(); } let chunksToProcess = []; // Se é o chunk esperado if (chunk.sequenceNumber === nextExpectedSequence) { chunksToProcess.push(chunk); let nextSeq = nextExpectedSequence + 1; // Verifica se há chunks subsequentes no buffer setBuffer(prevBuffer => { const remaining = []; const sorted = [...prevBuffer].sort((a, b) => a.sequenceNumber - b.sequenceNumber); for (const bufferedChunk of sorted) { if (bufferedChunk.sequenceNumber === nextSeq) { chunksToProcess.push(bufferedChunk); nextSeq++; } else if (bufferedChunk.sequenceNumber > nextSeq) { remaining.push(bufferedChunk); } // Ignora chunks com sequência menor (duplicados ou atrasados) } return remaining; }); setNextExpectedSequence(nextSeq); console.log(`✅ [ChunkBuffer] ${chunksToProcess.length} chunks prontos para processar`); } else if (chunk.sequenceNumber > nextExpectedSequence) { // Chunk fora de ordem - adiciona ao buffer setBuffer(prev => [...prev, chunk]); console.log(`⏸️ [ChunkBuffer] Chunk ${chunk.sequenceNumber} armazenado no buffer`); } else { // Chunk duplicado ou atrasado - ignora console.log(`⚠️ [ChunkBuffer] Ignorando chunk ${chunk.sequenceNumber} (duplicado ou atrasado)`); } return chunksToProcess; }, [nextExpectedSequence, resetBuffer]); /** * Retorna informações do buffer */ const getBufferInfo = react.useCallback(() => ({ size: buffer.length, nextExpected: nextExpectedSequence, bufferedSequences: buffer.map(c => c.sequenceNumber).sort((a, b) => a - b) }), [buffer, nextExpectedSequence]); return { addChunk, resetBuffer, getBufferInfo, bufferSize: buffer.length, nextExpectedSequence }; }; const ChatContext = /*#__PURE__*/react.createContext(); const ChatProvider = ({ children, apiUrl, r2Url, wsUrl }) => { // URLs de configuração const backendUrl = apiUrl || typeof window !== 'undefined' && undefined?.VITE_API_URL || "http://localhost:4001"; const websocketUrl = wsUrl || typeof window !== 'undefined' && undefined?.VITE_WS_URL || "http://localhost:4001"; // Socket ref const socketRef = react.useRef(null); const [isSocketConnected, setIsSocketConnected] = react.useState(false); // Estados de processamento const [isProcessing, setIsProcessing] = react.useState(false); const [loading, setLoading] = react.useState(false); const startTimeRef = react.useRef(null); // Ref para armazenar a função processIncomingChunk const processIncomingChunkRef = react.useRef(null); // Hooks customizados para gerenciamento de fila e buffer const audioQueue = useAudioQueue(); const chunkBuffer = useChunkBuffer(); // Estados de áudio const [isAudioPlaying, setIsAudioPlaying] = react.useState(false); const [audioLoadingState, setAudioLoadingState] = react.useState('idle'); const [audioPlaybackError, setAudioPlaybackErrorState] = react.useState(null); const audioPlayerRef = react.useRef(null); // Estados do avatar const [isAvatarSpeaking, setIsAvatarSpeaking] = react.useState(false); const [vadBlockedUntil, setVadBlockedUntil] = react.useState(0); // Estados UI const [cameraZoomed, setCameraZoomed] = react.useState(true); const [audioInteractionNeeded, setAudioInteractionNeeded] = react.useState(() => { const audioEnabled = localStorage.getItem('audioEnabled'); return audioEnabled !== 'true'; }); // Estado da transcrição const [userTranscription, setUserTranscription] = react.useState(null); /** * Processa chunk recebido do WebSocket */ const processIncomingChunk = react.useCallback(chunk => { console.log(`📨 [CHUNK ${chunk.chunkNumber}/${chunk.totalChunks}] Recebido:`, { text: chunk.text?.substring(0, 30) + '...', sequenceNumber: chunk.sequenceNumber, isLastChunk: chunk.isLastChunk }); // Ignora chunks sem áudio válido if (!chunk.audio || chunk.text === '🤔') { console.warn('⚠️ Chunk sem áudio válido, ignorando'); return; } // Cria mensagem formatada const message = { text: chunk.text, audio: chunk.audio, lipsync: { metadata: { version: 1 }, mouthCues: (chunk.visemes || []).map(viseme => ({ value: viseme.v, start: viseme.start / 1000, end: viseme.end / 1000 })) }, facialExpression: chunk.facialExpression || 'default', animation: chunk.animation || 'Talking_4', ...(chunk.emotes && { emotes: chunk.emotes }), ...(chunk.gesture && localStorage.getItem('gesturesEnabled') === 'true' && { gesture: chunk.gesture }), sequenceNumber: chunk.sequenceNumber, isLastChunk: chunk.isLastChunk }; // Adiciona ao buffer e processa chunks em ordem const chunksToProcess = chunkBuffer.addChunk(message); // Adiciona chunks ordenados à fila de áudio chunksToProcess.forEach(orderedMessage => { audioQueue.addToQueue(orderedMessage); }); // Se é o último chunk, reseta o buffer if (chunk.isLastChunk) { setTimeout(() => { chunkBuffer.resetBuffer(); }, 100); } }, [chunkBuffer, audioQueue]); // Atualiza ref sempre que a função mudar react.useEffect(() => { processIncomingChunkRef.current = processIncomingChunk; }, [processIncomingChunk]); /** * Inicializa conexão WebSocket */ react.useEffect(() => { if (!socketRef.current) { console.log('🔌 Conectando ao servidor WebSocket...'); socketRef.current = socket_ioClient.io(websocketUrl, { transports: ['websocket'], reconnection: true, reconnectionAttempts: 5, reconnectionDelay: 1000 }); // Eventos de conexão socketRef.current.on('connect', () => { console.log('✅ Conectado ao servidor'); setIsSocketConnected(true); }); socketRef.current.on('disconnect', reason => { console.log('❌ Desconectado:', reason); setIsSocketConnected(false); setIsProcessing(false); }); socketRef.current.on('connect_error', error => { console.error('❌ Erro de conexão:', error.message); setIsSocketConnected(false); }); // Handler de chunks de áudio socketRef.current.on('audio-chunk', chunk => { if (processIncomingChunkRef.current) { processIncomingChunkRef.current(chunk); } }); // Handler de conclusão socketRef.current.on('chat-complete', data => { console.log(`✅ Processamento completo: ${data.chunksCount} chunks`); setIsProcessing(false); setLoading(false); }); // Handler de erro socketRef.current.on('chat-error', error => { console.error('❌ Erro no processamento:', error); setIsProcessing(false); setLoading(false); }); // Handler de transcrição socketRef.current.on('transcription', data => { console.log('📝 Transcrição recebida:', data.text); setUserTranscription({ text: data.text, timestamp: data.timestamp }); // Não limpar automaticamente - só será substituída pela próxima transcrição }); } return () => { if (socketRef.current) { socketRef.current.off('connect'); socketRef.current.off('disconnect'); socketRef.current.off('connect_error'); socketRef.current.off('audio-chunk'); socketRef.current.off('chat-complete'); socketRef.current.off('chat-error'); socketRef.current.off('transcription'); if (socketRef.current.connected) { socketRef.current.disconnect(); } socketRef.current = null; } }; }, [websocketUrl]); // Remove processIncomingChunk from dependencies /** * Envia dados de áudio via WebSocket */ const sendAudioData = async audioArray => { if (!socketRef.current || !isSocketConnected) { console.error('❌ WebSocket não está conectado'); return; } if (isProcessing) { console.warn('⚠️ Já existe um processamento em andamento'); return; } setIsProcessing(true); setLoading(true); startTimeRef.current = Date.now(); // Limpa fila e buffer antes de novo processamento audioQueue.clearQueue(); chunkBuffer.resetBuffer(); console.log('📤 Enviando áudio via WebSocket...'); socketRef.current.emit('chat', { audio: audioArray, sampleRate: 16000 }); }; /** * Callback quando mensagem termina de tocar */ const onMessagePlayed = playedAudioIdentifier => { console.log('🎵 [Audio] Reprodução concluída'); // Atualiza estados setIsAudioPlaying(false); setAudioLoadingState('ended'); setIsAvatarSpeaking(false); // Remove da fila audioQueue.removeFromQueue(playedAudioIdentifier); // Bloqueia VAD temporariamente const blockUntil = Date.now() + 500; setVadBlockedUntil(blockUntil); }; /** * Gerencia erros de reprodução */ const setAudioPlaybackError = error => { setAudioPlaybackErrorState(error); }; const clearAudioPlaybackError = () => { setAudioPlaybackErrorState(null); }; const audioInteractionCompleted = () => { setAudioInteractionNeeded(false); localStorage.setItem('audioEnabled', 'true'); }; // Inicializa player de áudio compartilhado react.useEffect(() => { if (!audioPlayerRef.current) { audioPlayerRef.current = new Audio(); console.log("Player de áudio inicializado"); } return () => { if (audioPlayerRef.current) { audioPlayerRef.current.pause(); audioPlayerRef.current.src = ""; audioPlayerRef.current = null; } }; }, []); return /*#__PURE__*/jsxRuntime.jsx(ChatContext.Provider, { value: { // Funções principais sendAudioData, onMessagePlayed, // Estado da mensagem atual message: audioQueue.currentMessage, // Estados de controle loading, isSocketConnected, isProcessing, // Estados de áudio isAudioPlaying, setIsAudioPlaying, audioLoadingState, setAudioLoadingState, audioPlaybackError, setAudioPlaybackError, clearAudioPlaybackError, audioInteractionNeeded, audioInteractionCompleted, audioPlayerRef, // Estados do avatar isAvatarSpeaking, vadBlockedUntil, // Estados UI cameraZoomed, setCameraZoomed, // URLs r2Url, websocketUrl, backendUrl, // Informações da fila (para debug) queueInfo: audioQueue.getQueueInfo(), bufferInfo: chunkBuffer.getBufferInfo(), // Transcrição do usuário userTranscription }, children: children }); }; const useChat = () => { const context = react.useContext(ChatContext); if (!context) { throw new Error("useChat must be used within a ChatProvider"); } return context; }; /** * Gesture templates based on TalkingHead project * Each gesture contains rotation values for avatar bones */ const gestureTemplates = { handup: { // Values from TalkingHead (in radians) 'LeftShoulder.rotation': { x: 1.75, y: 0.3, z: -1.4 }, 'LeftArm.rotation': { x: 1.6, y: -0.5, z: 1.1 }, 'LeftForeArm.rotation': { x: -0.815, y: -0.2, z: 1.575 }, 'LeftHand.rotation': { x: -0.529, y: -0.2, z: 0.022 }, // Finger rotations for open hand 'LeftHandThumb1.rotation': { x: 0.745, y: -0.526, z: 0.604 }, 'LeftHandThumb2.rotation': { x: -0.107, y: -0.01, z: -0.142 }, 'LeftHandThumb3.rotation': { x: 0, y: 0.001, z: 0 }, 'LeftHandIndex1.rotation': { x: -0.126, y: -0.035, z: -0.087 }, 'LeftHandIndex2.rotation': { x: 0.255, y: 0.007, z: -0.085 }, 'LeftHandIndex3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandMiddle1.rotation': { x: -0.019, y: -0.128, z: -0.082 }, 'LeftHandMiddle2.rotation': { x: 0.233, y: 0.019, z: -0.074 }, 'LeftHandMiddle3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandRing1.rotation': { x: 0.005, y: -0.241, z: -0.122 }, 'LeftHandRing2.rotation': { x: 0.261, y: 0.021, z: -0.076 }, 'LeftHandRing3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandPinky1.rotation': { x: 0.059, y: -0.336, z: -0.2 }, 'LeftHandPinky2.rotation': { x: 0.153, y: 0.019, z: 0.001 }, 'LeftHandPinky3.rotation': { x: 0, y: 0, z: 0 } }, index: { 'LeftShoulder.rotation': { x: 1.75, y: 0.3, z: -1.4 }, 'LeftArm.rotation': { x: 1.6, y: -0.5, z: 1.1 }, 'LeftForeArm.rotation': { x: -0.815, y: -0.2, z: 1.575 }, 'LeftHand.rotation': { x: -0.276, y: -0.506, z: -0.208 }, // Index finger extended 'LeftHandIndex1.rotation': { x: 0, y: -0.105, z: 0.225 }, 'LeftHandIndex2.rotation': { x: 0.256, y: -0.103, z: -0.213 }, 'LeftHandIndex3.rotation': { x: 0, y: 0, z: 0 }, // Other fingers folded 'LeftHandThumb1.rotation': { x: 0.579, y: 0.228, z: 0.363 }, 'LeftHandThumb2.rotation': { x: -0.027, y: -0.04, z: -0.662 }, 'LeftHandThumb3.rotation': { x: 0, y: 0.001, z: 0 }, 'LeftHandMiddle1.rotation': { x: 1.453, y: 0.07, z: 0.021 }, 'LeftHandMiddle2.rotation': { x: 1.599, y: 0.062, z: 0.07 }, 'LeftHandMiddle3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandRing1.rotation': { x: 1.528, y: -0.073, z: 0.052 }, 'LeftHandRing2.rotation': { x: 1.386, y: 0.044, z: 0.053 }, 'LeftHandRing3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandPinky1.rotation': { x: 1.65, y: -0.204, z: 0.031 }, 'LeftHandPinky2.rotation': { x: 1.302, y: 0.071, z: 0.085 }, 'LeftHandPinky3.rotation': { x: 0, y: 0, z: 0 } }, ok: { 'LeftShoulder.rotation': { x: 1.75, y: 0.3, z: -1.4 }, 'LeftArm.rotation': { x: 1.6, y: -0.5, z: 1.1 }, 'LeftForeArm.rotation': { x: -0.415, y: -0.2, z: 1.575 }, 'LeftHand.rotation': { x: -0.476, y: -0.506, z: -0.208 }, // Thumb and index forming circle 'LeftHandThumb1.rotation': { x: 0.703, y: 0.445, z: 0.899 }, 'LeftHandThumb2.rotation': { x: -0.312, y: -0.04, z: -0.938 }, 'LeftHandThumb3.rotation': { x: -0.37, y: 0.024, z: -0.393 }, 'LeftHandIndex1.rotation': { x: 0.8, y: -0.086, z: -0.091 }, 'LeftHandIndex2.rotation': { x: 1.123, y: -0.046, z: -0.074 }, 'LeftHandIndex3.rotation': { x: 0.562, y: -0.013, z: -0.043 }, // Other fingers extended 'LeftHandMiddle1.rotation': { x: -0.019, y: -0.128, z: -0.082 }, 'LeftHandMiddle2.rotation': { x: 0.233, y: 0.019, z: -0.074 }, 'LeftHandMiddle3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandRing1.rotation': { x: 0.005, y: -0.241, z: -0.122 }, 'LeftHandRing2.rotation': { x: 0.261, y: 0.021, z: -0.076 }, 'LeftHandRing3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandPinky1.rotation': { x: 0.059, y: -0.336, z: -0.2 }, 'LeftHandPinky2.rotation': { x: 0.153, y: 0.019, z: 0.001 }, 'LeftHandPinky3.rotation': { x: 0, y: 0, z: 0 } }, thumbup: { 'LeftShoulder.rotation': { x: 1.75, y: 0.3, z: -1.4 }, 'LeftArm.rotation': { x: 1.6, y: -0.5, z: 1.1 }, 'LeftForeArm.rotation': { x: -0.415, y: 0.206, z: 1.575 }, 'LeftHand.rotation': { x: -0.276, y: -0.506, z: -0.208 }, // Thumb up 'LeftHandThumb1.rotation': { x: 0.208, y: -0.189, z: 0.685 }, 'LeftHandThumb2.rotation': { x: 0.129, y: -0.285, z: -0.163 }, 'LeftHandThumb3.rotation': { x: -0.047, y: 0.068, z: 0.401 }, // Other fingers folded 'LeftHandIndex1.rotation': { x: 1.412, y: -0.102, z: -0.152 }, 'LeftHandIndex2.rotation': { x: 1.903, y: -0.16, z: -0.114 }, 'LeftHandIndex3.rotation': { x: 0.535, y: -0.017, z: -0.062 }, 'LeftHandMiddle1.rotation': { x: 1.424, y: -0.103, z: -0.12 }, 'LeftHandMiddle2.rotation': { x: 1.919, y: -0.162, z: -0.114 }, 'LeftHandMiddle3.rotation': { x: 0.44, y: -0.012, z: -0.051 }, 'LeftHandRing1.rotation': { x: 1.619, y: -0.127, z: -0.053 }, 'LeftHandRing2.rotation': { x: 1.898, y: -0.16, z: -0.115 }, 'LeftHandRing3.rotation': { x: 0.262, y: -4e-3, z: -0.031 }, 'LeftHandPinky1.rotation': { x: 1.661, y: -0.131, z: -0.016 }, 'LeftHandPinky2.rotation': { x: 1.715, y: -0.067, z: -0.13 }, 'LeftHandPinky3.rotation': { x: 0.627, y: -0.023, z: -0.071 } }, thumbdown: { // Thumbs down gesture - values from TalkingHead 'LeftShoulder.rotation': { x: 1.75, y: 0.3, z: -1.4 }, // Using middle of array ranges 'LeftArm.rotation': { x: 1.6, y: -0.5, z: 1.1 }, // Using middle of array ranges 'LeftForeArm.rotation': { x: -2.015, y: 0.406, z: 1.575 }, // Key rotation for thumbdown 'LeftHand.rotation': { x: -0.176, y: -0.206, z: -0.208 }, // Thumb pointing down 'LeftHandThumb1.rotation': { x: 0.208, y: -0.189, z: 0.685 }, 'LeftHandThumb2.rotation': { x: 0.129, y: -0.285, z: -0.163 }, 'LeftHandThumb3.rotation': { x: -0.047, y: 0.068, z: 0.401 }, // Fingers folded 'LeftHandIndex1.rotation': { x: 1.412, y: -0.102, z: -0.152 }, 'LeftHandIndex2.rotation': { x: 1.903, y: -0.16, z: -0.114 }, 'LeftHandIndex3.rotation': { x: 0.535, y: -0.017, z: -0.062 }, 'LeftHandMiddle1.rotation': { x: 1.424, y: -0.103, z: -0.12 }, 'LeftHandMiddle2.rotation': { x: 1.919, y: -0.162, z: -0.114 }, 'LeftHandMiddle3.rotation': { x: 0.44, y: -0.012, z: -0.051 }, 'LeftHandRing1.rotation': { x: 1.619, y: -0.127, z: -0.053 }, 'LeftHandRing2.rotation': { x: 1.898, y: -0.16, z: -0.115 }, 'LeftHandRing3.rotation': { x: 0.262, y: -4e-3, z: -0.031 }, 'LeftHandPinky1.rotation': { x: 1.661, y: -0.131, z: -0.016 }, 'LeftHandPinky2.rotation': { x: 1.715, y: -0.067, z: -0.13 }, 'LeftHandPinky3.rotation': { x: 0.627, y: -0.023, z: -0.071 } }, salute: { 'LeftShoulder.rotation': { x: 1.696, y: -0.17, z: -1.763 }, 'LeftArm.rotation': { x: 0.886, y: -0.295, z: 0.889 }, 'LeftForeArm.rotation': { x: 0, y: 0, z: 2.18 }, 'LeftHand.rotation': { x: 0.027, y: -0.31, z: 0.348 }, // Fingers in salute position 'LeftHandThumb1.rotation': { x: 1.394, y: -0.882, z: 0.967 }, 'LeftHandThumb2.rotation': { x: -0.394, y: 0.246, z: 0.112 }, 'LeftHandThumb3.rotation': { x: -0.029, y: -2e-3, z: -7e-3 }, 'LeftHandIndex1.rotation': { x: 0.251, y: -8e-3, z: -0.084 }, 'LeftHandIndex2.rotation': { x: 0.009, y: -7e-3, z: -6e-3 }, 'LeftHandIndex3.rotation': { x: -0.046, y: 0, z: 0 }, 'LeftHandMiddle1.rotation': { x: 0.125, y: -9e-3, z: -0.061 }, 'LeftHandMiddle2.rotation': { x: 0.094, y: -5e-3, z: -1e-3 }, 'LeftHandMiddle3.rotation': { x: 0.092, y: 0, z: 0 }, 'LeftHandRing1.rotation': { x: 0.214, y: -0.011, z: 0.026 }, 'LeftHandRing2.rotation': { x: 0.103, y: -4e-3, z: -6e-3 }, 'LeftHandRing3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandPinky1.rotation': { x: 0.269, y: 0.005, z: 0.05 }, 'LeftHandPinky2.rotation': { x: 0.074, y: -2e-3, z: 0.003 }, 'LeftHandPinky3.rotation': { x: 0, y: 0, z: 0 } }, side: { // Side gesture - exact values from TalkingHead 'LeftShoulder.rotation': { x: 1.755, y: -0.035, z: -1.63 }, 'LeftArm.rotation': { x: 1.263, y: -0.955, z: 1.024 }, 'LeftForeArm.rotation': { x: 0, y: 0, z: 0.8 }, 'LeftHand.rotation': { x: -0.36, y: -1.353, z: -0.184 }, // Finger positions 'LeftHandThumb1.rotation': { x: 0.137, y: -0.049, z: 0.863 }, 'LeftHandThumb2.rotation': { x: -0.293, y: 0.153, z: -0.193 }, 'LeftHandThumb3.rotation': { x: -0.271, y: -0.17, z: 0.18 }, 'LeftHandIndex1.rotation': { x: -0.018, y: 0.007, z: 0.28 }, 'LeftHandIndex2.rotation': { x: 0.247, y: -3e-3, z: -0.025 }, 'LeftHandIndex3.rotation': { x: 0.13, y: -1e-3, z: -0.013 }, 'LeftHandMiddle1.rotation': { x: 0.333, y: -0.015, z: 0.182 }, 'LeftHandMiddle2.rotation': { x: 0.313, y: -5e-3, z: -0.032 }, 'LeftHandMiddle3.rotation': { x: 0.294, y: -4e-3, z: -0.03 }, 'LeftHandRing1.rotation': { x: 0.456, y: -0.028, z: -0.092 }, 'LeftHandRing2.rotation': { x: 0.53, y: -0.014, z: -0.052 }, 'LeftHandRing3.rotation': { x: 0.478, y: -0.012, z: -0.047 }, 'LeftHandPinky1.rotation': { x: 0.647, y: -0.049, z: -0.184 }, 'LeftHandPinky2.rotation': { x: 0.29, y: -4e-3, z: -0.029 }, 'LeftHandPinky3.rotation': { x: 0.501, y: -0.013, z: -0.049 } }, shrug: { // Shrug gesture - values from TalkingHead 'Neck.rotation': { x: 0, y: 0, z: 0 }, // TalkingHead uses random values, we use neutral 'Head.rotation': { x: 0, y: 0, z: 0 }, // TalkingHead uses random values, we use neutral // Right side 'RightShoulder.rotation': { x: 1.732, y: -0.058, z: 1.407 }, 'RightArm.rotation': { x: 1.305, y: 0.46, z: 0.118 }, 'RightForeArm.rotation': { x: 1.0, y: -0.4, z: -1.637 }, // Using middle of random range 'RightHand.rotation': { x: -0.048, y: 0.165, z: -0.39 }, // Left side 'LeftShoulder.rotation': { x: 1.713, y: 0.141, z: -1.433 }, 'LeftArm.rotation': { x: 1.136, y: -0.422, z: -0.416 }, 'LeftForeArm.rotation': { x: 1.42, y: 0.123, z: 1.506 }, 'LeftHand.rotation': { x: 0.073, y: -0.138, z: 0.064 }, // Right hand fingers 'RightHandThumb1.rotation': { x: 1.467, y: 0.599, z: -1.315 }, 'RightHandThumb2.rotation': { x: -0.255, y: -0.123, z: 0.119 }, 'RightHandThumb3.rotation': { x: 0, y: -2e-3, z: 0 }, 'RightHandIndex1.rotation': { x: -0.293, y: -0.066, z: -0.112 }, 'RightHandIndex2.rotation': { x: 0.181, y: 0.007, z: 0.069 }, 'RightHandIndex3.rotation': { x: 0, y: 0, z: 0 }, 'RightHandMiddle1.rotation': { x: -0.063, y: -0.041, z: 0.032 }, 'RightHandMiddle2.rotation': { x: 0.149, y: 0.005, z: 0.05 }, 'RightHandMiddle3.rotation': { x: 0, y: 0, z: 0 }, 'RightHandRing1.rotation': { x: 0.152, y: -0.03, z: 0.132 }, 'RightHandRing2.rotation': { x: 0.194, y: 0.007, z: 0.058 }, 'RightHandRing3.rotation': { x: 0, y: 0, z: 0 }, 'RightHandPinky1.rotation': { x: 0.306, y: -0.015, z: 0.257 }, 'RightHandPinky2.rotation': { x: 0.15, y: -3e-3, z: -3e-3 }, 'RightHandPinky3.rotation': { x: 0, y: 0, z: 0 }, // Left hand fingers 'LeftHandThumb1.rotation': { x: 1.467, y: -0.599, z: 1.314 }, 'LeftHandThumb2.rotation': { x: -0.255, y: 0.123, z: -0.119 }, 'LeftHandThumb3.rotation': { x: 0, y: 0.001, z: 0 }, 'LeftHandIndex1.rotation': { x: -0.293, y: 0.066, z: 0.112 }, 'LeftHandIndex2.rotation': { x: 0.181, y: -7e-3, z: -0.069 }, 'LeftHandIndex3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandMiddle1.rotation': { x: -0.062, y: 0.041, z: -0.032 }, 'LeftHandMiddle2.rotation': { x: 0.149, y: -5e-3, z: -0.05 }, 'LeftHandMiddle3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandRing1.rotation': { x: 0.152, y: 0.03, z: -0.132 }, 'LeftHandRing2.rotation': { x: 0.194, y: -7e-3, z: -0.058 }, 'LeftHandRing3.rotation': { x: 0, y: 0, z: 0 }, 'LeftHandPinky1.rotation': { x: 0.306, y: 0.015, z: -0.257 }, 'LeftHandPinky2.rotation': { x: 0.15, y: 0.003, z: 0.003 }, 'LeftHandPinky3.rotation': { x: 0, y: 0, z: 0 } }, namaste: { // Prayer position - values from TalkingHead // Right side 'RightShoulder.rotation': { x: 1.758, y: 0.099, z: 1.604 }, 'RightArm.rotation': { x: 0.862, y: -0.292, z: -0.932 }, 'RightForeArm.rotation': { x: 0.083, y: 0.066, z: -1.791 }, 'RightHand.rotation': { x: -0.52, y: -1e-3, z: -0.176 }, // Left side 'LeftShoulder.rotation': { x: 1.711, y: -2e-3, z: -1.625 }, 'LeftArm.rotation': { x: 0.683, y: 0.334, z: 0.977 }, 'LeftForeArm.rotation': { x: 0.086, y: -0.066, z: 1.843 }, 'LeftHand.rotation': { x: -0.595, y: -0.229, z: 0.096 }, // Right hand fingers 'RightHandThumb1.rotation': { x: 0.227, y: 0.418, z: -0.776 }, 'RightHandThumb2.rotation': { x: -0.011, y: -3e-3, z: 0.171 }, 'RightHandThumb3.rotation': { x: -0.041, y: -1e-3, z: -0.013 }, 'RightHandIndex1.rotation': { x: -0.236, y: 0.003, z: -0.028 }, 'RightHandIndex2.rotation': { x: 0.004, y: 0, z: 0.001 }, 'RightHandIndex3.rotation': { x: 0.002, y: 0, z: 0 }, 'RightHandMiddle1.rotation': { x: -0.236, y: 0.003, z: -0.028 }, 'RightHandMiddle2.rotation': { x: 0.004, y: 0, z: 0.001 }, 'RightHandMiddle3.rotation': { x: 0.002, y: 0, z: 0 }, 'RightHandRing1.rotation': { x: -0.236, y: 0.003, z: -0.028 }, 'RightHandRing2.rotation': { x: 0.004, y: 0, z: 0.001 }, 'RightHandRing3.rotation': { x: 0.002, y: 0, z: 0 }, 'RightHandPinky1.rotation': { x: -0.236, y: 0.003, z: -0.028 }, 'RightHandPinky2.rotation': { x: 0.004, y: 0, z: 0.001 }, 'RightHandPinky3.rotation': { x: 0.002, y: 0, z: 0 }, // Left hand fingers 'LeftHandThumb1.rotation': { x: 0.404, y: -0.05, z: 0.537 }, 'LeftHandThumb2.rotation': { x: -0.02, y: 0.004, z: -0.154 }, 'LeftHandThumb3.rotation': { x: -0.049, y: 0.002, z: -0.019 }, 'LeftHandIndex1.rotation': { x: -0.113, y: -1e-3, z: 0.014 }, 'LeftHandIndex2.rotation': { x: 0.003, y: 0, z: 0 }, 'LeftHandIndex3.rotation': { x: 0.002, y: 0, z: 0 }, 'LeftHandMiddle1.rotation': { x: -0.113, y: -1e-3, z: 0.014 }, 'LeftHandMiddle2.rotation': { x: 0.003, y: 0, z: 0 }, 'LeftHandMiddle3.rotation': { x: 0.002, y: 0, z: 0 }, 'LeftHandRing1.rotation': { x: -0.113, y: -1e-3, z: 0.014 }, 'LeftHandRing2.rotation': { x: 0.004, y: 0, z: 0 }, 'LeftHandRing3.rotation': { x: 0.002, y: 0, z: 0 }, 'LeftHandPinky1.rotation': { x: -0.122, y: -1e-3, z: -0.057 }, 'LeftHandPinky2.rotation': { x: 0.012, y: 0.001, z: 0.07 }, 'LeftHandPinky3.rotation': { x: 0.002, y: 0, z: 0 } } }; /** * Mirror a gesture (swap left/right and invert appropriate axes) */ function mirrorGesture(gesture) { const mirrored = {}; Object.entries(gesture).forEach(([boneKey, rotation]) => { let mirroredBoneKey = boneKey; if (boneKey.includes('Left')) { mirroredBoneKey = boneKey.replace('Left', 'Right'); } else if (boneKey.includes('Right')) { mirroredBoneKey = boneKey.replace('Right', 'Left'); } mirrored[mirroredBoneKey] = { x: rotation.x, y: -rotation.y, z: -rotation.z }; }); return mirrored; } /** * Get all unique bone names from gesture templates */ function getAllGestureBones() { const bones = new Set(); Object.values(gestureTemplates).forEach(gesture => { Object.keys(gesture).forEach(boneKey => { const boneName = boneKey.replace('.rotation', ''); bones.add(boneName); // Also add mirrored version if (boneName.includes('Left')) { bones.add(boneName.replace('Left', 'Right')); } else if (boneName.includes('Right')) { bones.add(boneName.replace('Right', 'Left')); } }); }); return Array.from(bones); } /** * Sigmoid factory similar to TalkingHead * Creates smooth S-curve for natural movement */ const sigmoidFactory = (k = 5) => { // Sigmoid curve creates smooth acceleration and deceleration // k=5 gives a natural curve for gesture transitions const base = t => 1 / (1 + Math.exp(-k * t)) - 0.5; const corr = 0.5 / base(1); return t => corr * base(2 * Math.max(Math.min(t, 1), 0) - 1) + 0.5; }; /** * Hook to manage gesture animations for the avatar */ function useGestures(avatarRef, onGestureStateChange, options = {}) { // console.log('🎯 Using gesture system with TalkingHead values'); const gestureTimeoutRef = react.useRef(null); const currentGestureRef = react.useRef(null); const originalRotationsRef = react.useRef({}); /** * Bone name mapping for different model formats */ const boneMapping = { // Standard to mixamorig 'LeftShoulder': ['mixamorigLeftShoulder', 'mixamorig:LeftShoulder', 'left_shoulder', 'Left_Shoulder'], 'LeftArm': ['mixamorigLeftArm', 'mixamorig:LeftArm', 'left_arm', 'Left_Arm', 'LeftUpperArm'], 'LeftForeArm': ['mixamorigLeftForeArm', 'mixamorig:LeftForeArm', 'left_forearm', 'Left_ForeArm', 'LeftLowerArm'], 'LeftHand': ['mixamorigLeftHand', 'mixamorig:LeftHand', 'left_hand', 'Left_Hand'], 'RightShoulder': ['mixamorigRightShoulder', 'mixamorig:RightShoulder', 'right_shoulder', 'Right_Shoulder'], 'RightArm': ['mixamorigRightArm', 'mixamorig:RightArm', 'right_arm', 'Right_Arm', 'RightUpperArm'], 'RightForeArm': ['mixamorigRightForeArm', 'mixamorig:RightForeArm', 'right_forearm', 'Right_ForeArm', 'RightLowerArm'], 'RightHand': ['mixamorigRightHand', 'mixamorig:RightHand', 'right_hand', 'Right_Hand'], 'Head': ['mixamorigHead', 'mixamorig:Head', 'head'], 'Neck': ['mixamorigNeck', 'mixamorig:Neck', 'neck'], // Add finger mappings 'LeftHandThumb1': ['mixamorigLeftHandThumb1', 'mixamorig:LeftHandThumb1', 'left_thumb_01'], 'LeftHandThumb2': ['mixamorigLeftHandThumb2', 'mixamorig:LeftHandThumb2', 'left_thumb_02'], 'LeftHandThumb3': ['mixamorigLeftHandThumb3', 'mixamorig:LeftHandThumb3', 'left_thumb_03'], 'LeftHandIndex1': ['mixamorigLeftHandIndex1', 'mixamorig:LeftHandIndex1', 'left_index_01'], 'LeftHandIndex2': ['mixamorigLeftHandIndex2', 'mixamorig:LeftHandIndex2', 'left_index_02'], 'LeftHandIndex3': ['mixamorigLeftHandIndex3', 'mixamorig:LeftHandIndex3', 'left_index_03'], 'LeftHandMiddle1': ['mixamorigLeftHandMiddle1', 'mixamorig:LeftHandMiddle1', 'left_middle_01'], 'LeftHandMiddle2': ['mixamorigLeftHandMiddle2', 'mixamorig:LeftHandMiddle2', 'left_middle_02'], 'LeftHandMiddle3': ['mixamorigLeftHandMiddle3', 'mixamorig:LeftHandMiddle3', 'left_middle_03'], 'LeftHandRing1': ['mixamorigLeftHandRing1', 'mixamorig:LeftHandRing1', 'left_ring_01'], 'LeftHandRing2': ['mixamorigLeftHandRing2', 'mixamorig:LeftHandRing2', 'left_ring_02'], 'LeftHandRing3': ['mixamorigLeftHandRing3', 'mixamorig:LeftHandRing3', 'left_ring_03'], 'LeftHandPinky1': ['mixamorigLeftHandPinky1', 'mixamorig:LeftHandPinky1', 'left_pinky_01'], 'LeftHandPinky2': ['mixamorigLeftHandPinky2', 'mixamorig:LeftHandPinky2', 'left_pinky_02'], 'LeftHandPinky3': ['mixamorigLeftHandPinky3', 'mixamorig:LeftHandPinky3', 'left_pinky_03'], // Right hand fingers 'RightHandThumb1': ['mixamorigRightHandThumb1', 'mixamorig:RightHandThumb1', 'right_thumb_01'], 'RightHandThumb2': ['mixamorigRightHandThumb2', 'mixamorig:RightHandThumb2', 'right_thumb_02'], 'RightHandThumb3': ['mixamorigRightHandThumb3', 'mixamorig:RightHandThumb3', 'right_thumb_03'], 'RightHandIndex1': ['mixamorigRightHandIndex1', 'mixamorig:RightHandIndex1', 'right_index_01'], 'RightHandIndex2': ['mixamorigRightHandIndex2', 'mixamorig:RightHandIndex2', 'right_index_02'], 'RightHandIndex3': ['mixamorigRightHandIndex3', 'mixamorig:RightHandIndex3', 'right_index_03'], 'RightHandMiddle1': ['mixamorigRightHandMiddle1', 'mixamorig:RightHandMiddle1', 'right_middle_01'], 'RightHandMiddle2': ['mixamorigRightHandMiddle2', 'mixamorig:RightHandMiddle2', 'right_middle_02'], 'RightHandMiddle3': ['mixamorigRightHandMiddle3', 'mixamorig:RightHandMiddle3', 'right_middle_03'], 'RightHandRing1': ['mixamorigRightHandRing1', 'mixamorig:RightHandRing1', 'right_ring_01'], 'RightHandRing2': ['mixamorigRightHandRing2', 'mixamorig:RightHandRing2', 'right_ring_02'], 'RightHandRing3': ['mixamorigRightHandRing3', 'mixamorig:RightHandRing3', 'right_ring_03'], 'RightHandPinky1': ['mixamorigRightHandPinky1', 'mixamorig:RightHandPinky1', 'right_pinky_01'], 'RightHandPinky2': ['mixamorigRightHandPinky2', 'mixamorig:RightHandPinky2', 'right_pinky_02'], 'RightHandPinky3': ['mixamorigRightHandPinky3', 'mixamorig:RightHandPinky3', 'right_pinky_03'] }; /** * Find bone by name in the avatar hierarchy with automatic mapping */ const findBone = react.useCallback(boneName => { if (!avatarRef.current) return null; // First try exact name let bone = null; avatarRef.current.traverse(child => { if (child.isBone && child.name === boneName) { bone = child; } }); // If not found, try mapped names if (!bone && boneMapping[boneName]) { const possibleNames = boneMapping[boneName]; for (const mappedName of possibleNames) { avatarRef.current.traverse(child => { if (child.isBone && child.name === mappedName) { bone = child; } }); if (bone) { console.log(`🔄 Mapped bone ${boneName} -> ${mappedName}`); break; } } } return bone; }, [avatarRef]); /** * Store original bone rotations (rest pose) */ const storeOriginalRotations = react.useCallback(() => { const bones = getAllGestureBones(); let storedCount = 0; bones.forEach(boneName => { const bone = findBone(boneName); if (bone) { // Store the rest pose rotation originalRotationsRef.current[boneName] = bone.quaternion.clone(); storedCount++; console.log(`💾 Stored rest pose for ${boneName}:`, { x: bone.quaternion.x.toFixed(3), y: bone.quaternion.y.toFixed(3), z: bone.quaternion.z.toFixed(3), w: bone.quaternion.w.toFixed(3) }); } }); console.log(`💾 Total stored rest poses: ${storedCount}`); }, [findBone]); /** * Restore bones to original position */ const restoreBones = react.useCallback((duration = 500) => { const startTime = Date.now(); const easing = sigmoidFactory(5); // Store current positions const currentRotations = {}; const bones = getAllGestureBones(); bones.forEach(boneName => { const bone = findBone(boneName); if (bone) { currentRotations[boneName] = bone.quaternion.clone(); } }); const animate = () => { const elapsed = Date.now() - startTime; const progress = Math.min(elapsed / duration, 1); const alpha = easing(progress); bones.forEach(boneName => { const bone = findBone(boneName); if (bone && originalRotationsRef.current[boneName] && currentRotations[boneName]) { // Create temporary quaternion for interpolation const tempQuat = new THREE__namespace.Quaternion(); tempQuat.copy(currentRotations[boneName]); tempQuat.slerp(originalRotationsRef.current[boneName], alpha); // Apply the interpolated quaternion bone.quaternion.copy(tempQuat); } }); if (progress < 1) { requestAnimationFrame(animate); } else { console.log('✅ Restored to original position'); } }; animate(); }, [findBone]); /** * Apply gesture rotations to bones */ const applyGesture = react.useCallback((gesture, duration = 500, gestureName = '') => { console.log(`⏱️ Starting gesture animation with duration: ${duration}ms`); const startTime = Date.now(); const targetRotations = {}; const affectedBones = {}; // Create sigmoid easing function const easing = sigmoidFactory(5); // Store current rotations as start point const startRotations = {}; // Debug: Log gesture input console.log('🎯 Applying gesture with bones:', Object.keys(gesture)); // Convert euler angles to quaternions Object.entries(gesture).forEach(([boneKey, rotation]) => { // Extract bone name (remove .rotation suffix) const boneName = boneKey.replace('.rotation', ''); const bone = findBone(boneName); if (bone) { // Store bone reference affectedBones[boneName] = bone; // Store starting rotation startRotations[boneName] = bone.quaternion.clone(); // Debug: Log original rotation values console.log(`🦴 ${boneName} (${boneKey}) input rotation (radians):`, rotation); console.log(`🦴 ${boneName} input rotation (degrees):`, { x: THREE__namespace.MathUtils.radToDeg(rotation.x).toFixed(1), y: THREE__namespace.MathUtils.radToDeg(rotation.y).toFixed(1), z: THREE__namespace.MathUtils.radToDeg(rotation.z).toFixed(1) }); // Use rotation values directly (no processing) const x = rotation.x; const y = rotation.y; const z = rotation.z; // Create gesture rotation (TalkingHead uses XYZ order) const euler = new THREE__namespace.Euler(x, y, z, 'XYZ'); const gestureQuat = new THREE__namespace.Quaternion().setFromEuler(euler).normalize(); // Debug: Log quaternion console.log(`🎲 ${boneName} gesture quaternion:`, { x: gestureQuat.x.toFixed(3), y: gestureQuat.y.toFixed(3), z: gestureQuat.z.toFixed(3), w: gestureQuat.w.toFixed(3) }); // Apply gesture rotation targetRotations[boneName] = new THREE__namespace.Quaternion(); // TalkingHead applies gestures as absolute rotations, not relative // The gesture values already include the rest pose targetRotations[boneName].copy(gestureQuat); console.log(`🎯 ${boneName} gesture applied as absolute rotation`); } else { console.warn(`❌ Bone not found: ${boneName} (from ${boneKey})`); } }); // Special adjustment for namaste gesture (from TalkingHead) if (gestureName === 'namaste') { console.log('🙏 Namaste gesture - checking adjustment need'); // Log what bones we have before adjustment console.log('🙏 Bones found for namaste:', Object.keys(targetRotations)); { console.log('🙏 Namaste adjustment skipped (testing without it)'); } } console.log(`📊 Total bones to animate: ${Object.keys(targetRotations).length}`); const animate = () => { const elapsed = Date.now() - startTime; const progress = Math.min(elapsed / duration, 1); // Apply easing const alpha = easing(progress); Object.entries(targetRotations).forEach(([boneName, targetQuat]) => { const bone = affectedBones[boneName]; if (bone && startRotations[boneName]) { // Log first frame only if (elapsed < 50 && boneName === 'LeftArm') { console.log(`🎬 Animation frame - ${boneName}:`, { progress: progress.toFixed(2), alpha: alpha.toFixed(2), currentQuat: { x: bone.quaternion.x.toFixed(3), y: bone.quaternion.y.toFixed(3), z: bone.quaternion.z.toFixed(3), w: bone.quaternion.w.toFixed(3) } }); } // Create a temporary quaternion for interpolation const tempQuat = new THREE__namespace.Quaternion(); tempQuat.copy(startRotations[boneName]); tempQuat.slerp(targetQuat, alpha); // Apply the interpolated quaternion bone.quaternion.copy(tempQuat); } }); if (progress < 1) { requestAnimationFrame(animate); } else { console.log('✅ Gesture animation completed'); } }; animate(); }, [findBone]); /** * Play a gesture animation */ const playGesture = react.useCallback((gestureName, duration = 2, mirror = false, transitionMs = 500) => { console.log(`🤚 Playing gesture: ${gestureName} for ${duration}s`); console.log(`🤚 Avatar ref status:`, { hasRef: !!avatarRef.current, refType: avatarRef.current?.type, refChildren: avatarRef.current?.children?.length }); // Clear any existing gesture timeout if (gestureTimeoutRef.current) { clearTimeout(gestureTimeoutRef.current); } // Get gesture template let gesture = gestureTemplates[gestureName]; if (!gesture) { console.warn(`Gesture "${gestureName}" not found`); return; } // Mirror if requested if (mirror) { gesture = mirrorGesture(gesture); } // Store current gesture currentGestureRef.current = gestureName; // Notify that gesture is starting if (onGestureStateChange) { onGestureStateChange(true); } // Apply the gesture console.log(`🎯 Applying gesture with transitionMs: ${transitionMs}ms`); applyGesture(gesture, transitionMs, gestureName); // Schedule restoration const totalDuration = duration * 1000; console.log(`⏰ Scheduling gesture end in ${totalDuration}ms (${duration} seconds)`); gestureTimeou