@marcosremar/cabecao
Version:
Modern React 3D avatar component with chat and lip-sync capabilities
2,025 lines (1,935 loc) • 168 kB
JavaScript
/* eslint-disable */
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
var react = require('react');
var drei = require('@react-three/drei');
var fiber = require('@react-three/fiber');
var leva = require('leva');
var socket_ioClient = require('socket.io-client');
var jsxRuntime = require('react/jsx-runtime');
var THREE = require('three');
var vadReact = require('@ricky0123/vad-react');
function _interopNamespaceDefault(e) {
var n = Object.create(null);
if (e) {
Object.keys(e).forEach(function (k) {
if (k !== 'default') {
var d = Object.getOwnPropertyDescriptor(e, k);
Object.defineProperty(n, k, d.get ? d : {
enumerable: true,
get: function () { return e[k]; }
});
}
});
}
n.default = e;
return Object.freeze(n);
}
var THREE__namespace = /*#__PURE__*/_interopNamespaceDefault(THREE);
/**
* Hook para gerenciar fila de reprodução de áudio
* Segue princípios SOLID e DRY
*/
const useAudioQueue = () => {
// Estado da fila
const [queue, setQueue] = react.useState([]);
const [currentMessage, setCurrentMessage] = react.useState(null);
const [isProcessing, setIsProcessing] = react.useState(false);
// Referências para controle
const processingRef = react.useRef(false);
const queueRef = react.useRef([]);
// Sincronizar ref com state para evitar closures
react.useEffect(() => {
queueRef.current = queue;
}, [queue]);
/**
* Adiciona mensagem à fila
*/
const addToQueue = react.useCallback(message => {
console.log('🎵 [AudioQueue] Adicionando à fila:', {
text: message.text?.substring(0, 30),
sequenceNumber: message.sequenceNumber,
queueSizeBefore: queueRef.current.length
});
setQueue(prev => {
const newQueue = [...prev, message];
console.log(`📊 [AudioQueue] Tamanho da fila após adicionar: ${newQueue.length}`);
return newQueue;
});
}, []);
/**
* Remove mensagem da fila após reprodução
*/
const removeFromQueue = react.useCallback(audioId => {
console.log('🗑️ [AudioQueue] Removendo áudio reproduzido:', audioId?.substring(0, 50));
setQueue(prev => {
const newQueue = prev.filter(msg => msg.audio !== audioId);
console.log(`📊 [AudioQueue] Tamanho da fila após remover: ${newQueue.length}`);
return newQueue;
});
// Limpar mensagem atual
setCurrentMessage(null);
setIsProcessing(false);
processingRef.current = false;
}, []);
/**
* Processa próxima mensagem da fila
*/
const processNext = react.useCallback(() => {
// Evitar processamento paralelo
if (processingRef.current || queueRef.current.length === 0) {
return;
}
const next = queueRef.current[0];
if (!next) return;
console.log('▶️ [AudioQueue] Processando próxima mensagem:', {
text: next.text?.substring(0, 30),
queueSize: queueRef.current.length
});
processingRef.current = true;
setIsProcessing(true);
setCurrentMessage(next);
}, []);
/**
* Monitora fila e processa automaticamente
*/
react.useEffect(() => {
// Se não está processando e tem itens na fila
if (!processingRef.current && queue.length > 0 && !currentMessage) {
console.log('🔄 [AudioQueue] Fila não vazia, iniciando processamento');
processNext();
}
}, [queue, currentMessage, processNext]);
/**
* Limpa a fila
*/
const clearQueue = react.useCallback(() => {
console.log('🧹 [AudioQueue] Limpando fila');
setQueue([]);
setCurrentMessage(null);
setIsProcessing(false);
processingRef.current = false;
}, []);
/**
* Retorna informações da fila
*/
const getQueueInfo = react.useCallback(() => ({
size: queue.length,
isProcessing: processingRef.current,
currentMessage,
messages: queue.map(m => ({
text: m.text,
sequenceNumber: m.sequenceNumber
}))
}), [queue, currentMessage]);
return {
// Estado
queue,
currentMessage,
isProcessing,
// Ações
addToQueue,
removeFromQueue,
clearQueue,
processNext,
// Informações
getQueueInfo,
queueSize: queue.length
};
};
/**
* Hook para gerenciar buffer de chunks ordenados
* Garante que chunks sejam processados na ordem correta
*/
const useChunkBuffer = () => {
const [buffer, setBuffer] = react.useState([]);
const [nextExpectedSequence, setNextExpectedSequence] = react.useState(0);
/**
* Reseta o buffer para nova sequência
*/
const resetBuffer = react.useCallback(() => {
console.log('🔄 [ChunkBuffer] Resetando buffer');
setBuffer([]);
setNextExpectedSequence(0);
}, []);
/**
* Adiciona chunk ao buffer e retorna chunks prontos para processar
*/
const addChunk = react.useCallback(chunk => {
console.log(`📦 [ChunkBuffer] Recebido chunk ${chunk.sequenceNumber}`, {
expected: nextExpectedSequence,
isInOrder: chunk.sequenceNumber === nextExpectedSequence
});
// Se é o primeiro chunk, reseta
if (chunk.sequenceNumber === 0) {
resetBuffer();
}
let chunksToProcess = [];
// Se é o chunk esperado
if (chunk.sequenceNumber === nextExpectedSequence) {
chunksToProcess.push(chunk);
let nextSeq = nextExpectedSequence + 1;
// Verifica se há chunks subsequentes no buffer
setBuffer(prevBuffer => {
const remaining = [];
const sorted = [...prevBuffer].sort((a, b) => a.sequenceNumber - b.sequenceNumber);
for (const bufferedChunk of sorted) {
if (bufferedChunk.sequenceNumber === nextSeq) {
chunksToProcess.push(bufferedChunk);
nextSeq++;
} else if (bufferedChunk.sequenceNumber > nextSeq) {
remaining.push(bufferedChunk);
}
// Ignora chunks com sequência menor (duplicados ou atrasados)
}
return remaining;
});
setNextExpectedSequence(nextSeq);
console.log(`✅ [ChunkBuffer] ${chunksToProcess.length} chunks prontos para processar`);
} else if (chunk.sequenceNumber > nextExpectedSequence) {
// Chunk fora de ordem - adiciona ao buffer
setBuffer(prev => [...prev, chunk]);
console.log(`⏸️ [ChunkBuffer] Chunk ${chunk.sequenceNumber} armazenado no buffer`);
} else {
// Chunk duplicado ou atrasado - ignora
console.log(`⚠️ [ChunkBuffer] Ignorando chunk ${chunk.sequenceNumber} (duplicado ou atrasado)`);
}
return chunksToProcess;
}, [nextExpectedSequence, resetBuffer]);
/**
* Retorna informações do buffer
*/
const getBufferInfo = react.useCallback(() => ({
size: buffer.length,
nextExpected: nextExpectedSequence,
bufferedSequences: buffer.map(c => c.sequenceNumber).sort((a, b) => a - b)
}), [buffer, nextExpectedSequence]);
return {
addChunk,
resetBuffer,
getBufferInfo,
bufferSize: buffer.length,
nextExpectedSequence
};
};
const ChatContext = /*#__PURE__*/react.createContext();
const ChatProvider = ({
children,
apiUrl,
r2Url,
wsUrl
}) => {
// URLs de configuração
const backendUrl = apiUrl || typeof window !== 'undefined' && undefined?.VITE_API_URL || "http://localhost:4001";
const websocketUrl = wsUrl || typeof window !== 'undefined' && undefined?.VITE_WS_URL || "http://localhost:4001";
// Socket ref
const socketRef = react.useRef(null);
const [isSocketConnected, setIsSocketConnected] = react.useState(false);
// Estados de processamento
const [isProcessing, setIsProcessing] = react.useState(false);
const [loading, setLoading] = react.useState(false);
const startTimeRef = react.useRef(null);
// Ref para armazenar a função processIncomingChunk
const processIncomingChunkRef = react.useRef(null);
// Hooks customizados para gerenciamento de fila e buffer
const audioQueue = useAudioQueue();
const chunkBuffer = useChunkBuffer();
// Estados de áudio
const [isAudioPlaying, setIsAudioPlaying] = react.useState(false);
const [audioLoadingState, setAudioLoadingState] = react.useState('idle');
const [audioPlaybackError, setAudioPlaybackErrorState] = react.useState(null);
const audioPlayerRef = react.useRef(null);
// Estados do avatar
const [isAvatarSpeaking, setIsAvatarSpeaking] = react.useState(false);
const [vadBlockedUntil, setVadBlockedUntil] = react.useState(0);
// Estados UI
const [cameraZoomed, setCameraZoomed] = react.useState(true);
const [audioInteractionNeeded, setAudioInteractionNeeded] = react.useState(() => {
const audioEnabled = localStorage.getItem('audioEnabled');
return audioEnabled !== 'true';
});
// Estado da transcrição
const [userTranscription, setUserTranscription] = react.useState(null);
/**
* Processa chunk recebido do WebSocket
*/
const processIncomingChunk = react.useCallback(chunk => {
console.log(`📨 [CHUNK ${chunk.chunkNumber}/${chunk.totalChunks}] Recebido:`, {
text: chunk.text?.substring(0, 30) + '...',
sequenceNumber: chunk.sequenceNumber,
isLastChunk: chunk.isLastChunk
});
// Ignora chunks sem áudio válido
if (!chunk.audio || chunk.text === '🤔') {
console.warn('⚠️ Chunk sem áudio válido, ignorando');
return;
}
// Cria mensagem formatada
const message = {
text: chunk.text,
audio: chunk.audio,
lipsync: {
metadata: {
version: 1
},
mouthCues: (chunk.visemes || []).map(viseme => ({
value: viseme.v,
start: viseme.start / 1000,
end: viseme.end / 1000
}))
},
facialExpression: chunk.facialExpression || 'default',
animation: chunk.animation || 'Talking_4',
...(chunk.emotes && {
emotes: chunk.emotes
}),
...(chunk.gesture && localStorage.getItem('gesturesEnabled') === 'true' && {
gesture: chunk.gesture
}),
sequenceNumber: chunk.sequenceNumber,
isLastChunk: chunk.isLastChunk
};
// Adiciona ao buffer e processa chunks em ordem
const chunksToProcess = chunkBuffer.addChunk(message);
// Adiciona chunks ordenados à fila de áudio
chunksToProcess.forEach(orderedMessage => {
audioQueue.addToQueue(orderedMessage);
});
// Se é o último chunk, reseta o buffer
if (chunk.isLastChunk) {
setTimeout(() => {
chunkBuffer.resetBuffer();
}, 100);
}
}, [chunkBuffer, audioQueue]);
// Atualiza ref sempre que a função mudar
react.useEffect(() => {
processIncomingChunkRef.current = processIncomingChunk;
}, [processIncomingChunk]);
/**
* Inicializa conexão WebSocket
*/
react.useEffect(() => {
if (!socketRef.current) {
console.log('🔌 Conectando ao servidor WebSocket...');
socketRef.current = socket_ioClient.io(websocketUrl, {
transports: ['websocket'],
reconnection: true,
reconnectionAttempts: 5,
reconnectionDelay: 1000
});
// Eventos de conexão
socketRef.current.on('connect', () => {
console.log('✅ Conectado ao servidor');
setIsSocketConnected(true);
});
socketRef.current.on('disconnect', reason => {
console.log('❌ Desconectado:', reason);
setIsSocketConnected(false);
setIsProcessing(false);
});
socketRef.current.on('connect_error', error => {
console.error('❌ Erro de conexão:', error.message);
setIsSocketConnected(false);
});
// Handler de chunks de áudio
socketRef.current.on('audio-chunk', chunk => {
if (processIncomingChunkRef.current) {
processIncomingChunkRef.current(chunk);
}
});
// Handler de conclusão
socketRef.current.on('chat-complete', data => {
console.log(`✅ Processamento completo: ${data.chunksCount} chunks`);
setIsProcessing(false);
setLoading(false);
});
// Handler de erro
socketRef.current.on('chat-error', error => {
console.error('❌ Erro no processamento:', error);
setIsProcessing(false);
setLoading(false);
});
// Handler de transcrição
socketRef.current.on('transcription', data => {
console.log('📝 Transcrição recebida:', data.text);
setUserTranscription({
text: data.text,
timestamp: data.timestamp
});
// Não limpar automaticamente - só será substituída pela próxima transcrição
});
}
return () => {
if (socketRef.current) {
socketRef.current.off('connect');
socketRef.current.off('disconnect');
socketRef.current.off('connect_error');
socketRef.current.off('audio-chunk');
socketRef.current.off('chat-complete');
socketRef.current.off('chat-error');
socketRef.current.off('transcription');
if (socketRef.current.connected) {
socketRef.current.disconnect();
}
socketRef.current = null;
}
};
}, [websocketUrl]); // Remove processIncomingChunk from dependencies
/**
* Envia dados de áudio via WebSocket
*/
const sendAudioData = async audioArray => {
if (!socketRef.current || !isSocketConnected) {
console.error('❌ WebSocket não está conectado');
return;
}
if (isProcessing) {
console.warn('⚠️ Já existe um processamento em andamento');
return;
}
setIsProcessing(true);
setLoading(true);
startTimeRef.current = Date.now();
// Limpa fila e buffer antes de novo processamento
audioQueue.clearQueue();
chunkBuffer.resetBuffer();
console.log('📤 Enviando áudio via WebSocket...');
socketRef.current.emit('chat', {
audio: audioArray,
sampleRate: 16000
});
};
/**
* Callback quando mensagem termina de tocar
*/
const onMessagePlayed = playedAudioIdentifier => {
console.log('🎵 [Audio] Reprodução concluída');
// Atualiza estados
setIsAudioPlaying(false);
setAudioLoadingState('ended');
setIsAvatarSpeaking(false);
// Remove da fila
audioQueue.removeFromQueue(playedAudioIdentifier);
// Bloqueia VAD temporariamente
const blockUntil = Date.now() + 500;
setVadBlockedUntil(blockUntil);
};
/**
* Gerencia erros de reprodução
*/
const setAudioPlaybackError = error => {
setAudioPlaybackErrorState(error);
};
const clearAudioPlaybackError = () => {
setAudioPlaybackErrorState(null);
};
const audioInteractionCompleted = () => {
setAudioInteractionNeeded(false);
localStorage.setItem('audioEnabled', 'true');
};
// Inicializa player de áudio compartilhado
react.useEffect(() => {
if (!audioPlayerRef.current) {
audioPlayerRef.current = new Audio();
console.log("Player de áudio inicializado");
}
return () => {
if (audioPlayerRef.current) {
audioPlayerRef.current.pause();
audioPlayerRef.current.src = "";
audioPlayerRef.current = null;
}
};
}, []);
return /*#__PURE__*/jsxRuntime.jsx(ChatContext.Provider, {
value: {
// Funções principais
sendAudioData,
onMessagePlayed,
// Estado da mensagem atual
message: audioQueue.currentMessage,
// Estados de controle
loading,
isSocketConnected,
isProcessing,
// Estados de áudio
isAudioPlaying,
setIsAudioPlaying,
audioLoadingState,
setAudioLoadingState,
audioPlaybackError,
setAudioPlaybackError,
clearAudioPlaybackError,
audioInteractionNeeded,
audioInteractionCompleted,
audioPlayerRef,
// Estados do avatar
isAvatarSpeaking,
vadBlockedUntil,
// Estados UI
cameraZoomed,
setCameraZoomed,
// URLs
r2Url,
websocketUrl,
backendUrl,
// Informações da fila (para debug)
queueInfo: audioQueue.getQueueInfo(),
bufferInfo: chunkBuffer.getBufferInfo(),
// Transcrição do usuário
userTranscription
},
children: children
});
};
const useChat = () => {
const context = react.useContext(ChatContext);
if (!context) {
throw new Error("useChat must be used within a ChatProvider");
}
return context;
};
/**
* Gesture templates based on TalkingHead project
* Each gesture contains rotation values for avatar bones
*/
const gestureTemplates = {
handup: {
// Values from TalkingHead (in radians)
'LeftShoulder.rotation': {
x: 1.75,
y: 0.3,
z: -1.4
},
'LeftArm.rotation': {
x: 1.6,
y: -0.5,
z: 1.1
},
'LeftForeArm.rotation': {
x: -0.815,
y: -0.2,
z: 1.575
},
'LeftHand.rotation': {
x: -0.529,
y: -0.2,
z: 0.022
},
// Finger rotations for open hand
'LeftHandThumb1.rotation': {
x: 0.745,
y: -0.526,
z: 0.604
},
'LeftHandThumb2.rotation': {
x: -0.107,
y: -0.01,
z: -0.142
},
'LeftHandThumb3.rotation': {
x: 0,
y: 0.001,
z: 0
},
'LeftHandIndex1.rotation': {
x: -0.126,
y: -0.035,
z: -0.087
},
'LeftHandIndex2.rotation': {
x: 0.255,
y: 0.007,
z: -0.085
},
'LeftHandIndex3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandMiddle1.rotation': {
x: -0.019,
y: -0.128,
z: -0.082
},
'LeftHandMiddle2.rotation': {
x: 0.233,
y: 0.019,
z: -0.074
},
'LeftHandMiddle3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandRing1.rotation': {
x: 0.005,
y: -0.241,
z: -0.122
},
'LeftHandRing2.rotation': {
x: 0.261,
y: 0.021,
z: -0.076
},
'LeftHandRing3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandPinky1.rotation': {
x: 0.059,
y: -0.336,
z: -0.2
},
'LeftHandPinky2.rotation': {
x: 0.153,
y: 0.019,
z: 0.001
},
'LeftHandPinky3.rotation': {
x: 0,
y: 0,
z: 0
}
},
index: {
'LeftShoulder.rotation': {
x: 1.75,
y: 0.3,
z: -1.4
},
'LeftArm.rotation': {
x: 1.6,
y: -0.5,
z: 1.1
},
'LeftForeArm.rotation': {
x: -0.815,
y: -0.2,
z: 1.575
},
'LeftHand.rotation': {
x: -0.276,
y: -0.506,
z: -0.208
},
// Index finger extended
'LeftHandIndex1.rotation': {
x: 0,
y: -0.105,
z: 0.225
},
'LeftHandIndex2.rotation': {
x: 0.256,
y: -0.103,
z: -0.213
},
'LeftHandIndex3.rotation': {
x: 0,
y: 0,
z: 0
},
// Other fingers folded
'LeftHandThumb1.rotation': {
x: 0.579,
y: 0.228,
z: 0.363
},
'LeftHandThumb2.rotation': {
x: -0.027,
y: -0.04,
z: -0.662
},
'LeftHandThumb3.rotation': {
x: 0,
y: 0.001,
z: 0
},
'LeftHandMiddle1.rotation': {
x: 1.453,
y: 0.07,
z: 0.021
},
'LeftHandMiddle2.rotation': {
x: 1.599,
y: 0.062,
z: 0.07
},
'LeftHandMiddle3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandRing1.rotation': {
x: 1.528,
y: -0.073,
z: 0.052
},
'LeftHandRing2.rotation': {
x: 1.386,
y: 0.044,
z: 0.053
},
'LeftHandRing3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandPinky1.rotation': {
x: 1.65,
y: -0.204,
z: 0.031
},
'LeftHandPinky2.rotation': {
x: 1.302,
y: 0.071,
z: 0.085
},
'LeftHandPinky3.rotation': {
x: 0,
y: 0,
z: 0
}
},
ok: {
'LeftShoulder.rotation': {
x: 1.75,
y: 0.3,
z: -1.4
},
'LeftArm.rotation': {
x: 1.6,
y: -0.5,
z: 1.1
},
'LeftForeArm.rotation': {
x: -0.415,
y: -0.2,
z: 1.575
},
'LeftHand.rotation': {
x: -0.476,
y: -0.506,
z: -0.208
},
// Thumb and index forming circle
'LeftHandThumb1.rotation': {
x: 0.703,
y: 0.445,
z: 0.899
},
'LeftHandThumb2.rotation': {
x: -0.312,
y: -0.04,
z: -0.938
},
'LeftHandThumb3.rotation': {
x: -0.37,
y: 0.024,
z: -0.393
},
'LeftHandIndex1.rotation': {
x: 0.8,
y: -0.086,
z: -0.091
},
'LeftHandIndex2.rotation': {
x: 1.123,
y: -0.046,
z: -0.074
},
'LeftHandIndex3.rotation': {
x: 0.562,
y: -0.013,
z: -0.043
},
// Other fingers extended
'LeftHandMiddle1.rotation': {
x: -0.019,
y: -0.128,
z: -0.082
},
'LeftHandMiddle2.rotation': {
x: 0.233,
y: 0.019,
z: -0.074
},
'LeftHandMiddle3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandRing1.rotation': {
x: 0.005,
y: -0.241,
z: -0.122
},
'LeftHandRing2.rotation': {
x: 0.261,
y: 0.021,
z: -0.076
},
'LeftHandRing3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandPinky1.rotation': {
x: 0.059,
y: -0.336,
z: -0.2
},
'LeftHandPinky2.rotation': {
x: 0.153,
y: 0.019,
z: 0.001
},
'LeftHandPinky3.rotation': {
x: 0,
y: 0,
z: 0
}
},
thumbup: {
'LeftShoulder.rotation': {
x: 1.75,
y: 0.3,
z: -1.4
},
'LeftArm.rotation': {
x: 1.6,
y: -0.5,
z: 1.1
},
'LeftForeArm.rotation': {
x: -0.415,
y: 0.206,
z: 1.575
},
'LeftHand.rotation': {
x: -0.276,
y: -0.506,
z: -0.208
},
// Thumb up
'LeftHandThumb1.rotation': {
x: 0.208,
y: -0.189,
z: 0.685
},
'LeftHandThumb2.rotation': {
x: 0.129,
y: -0.285,
z: -0.163
},
'LeftHandThumb3.rotation': {
x: -0.047,
y: 0.068,
z: 0.401
},
// Other fingers folded
'LeftHandIndex1.rotation': {
x: 1.412,
y: -0.102,
z: -0.152
},
'LeftHandIndex2.rotation': {
x: 1.903,
y: -0.16,
z: -0.114
},
'LeftHandIndex3.rotation': {
x: 0.535,
y: -0.017,
z: -0.062
},
'LeftHandMiddle1.rotation': {
x: 1.424,
y: -0.103,
z: -0.12
},
'LeftHandMiddle2.rotation': {
x: 1.919,
y: -0.162,
z: -0.114
},
'LeftHandMiddle3.rotation': {
x: 0.44,
y: -0.012,
z: -0.051
},
'LeftHandRing1.rotation': {
x: 1.619,
y: -0.127,
z: -0.053
},
'LeftHandRing2.rotation': {
x: 1.898,
y: -0.16,
z: -0.115
},
'LeftHandRing3.rotation': {
x: 0.262,
y: -4e-3,
z: -0.031
},
'LeftHandPinky1.rotation': {
x: 1.661,
y: -0.131,
z: -0.016
},
'LeftHandPinky2.rotation': {
x: 1.715,
y: -0.067,
z: -0.13
},
'LeftHandPinky3.rotation': {
x: 0.627,
y: -0.023,
z: -0.071
}
},
thumbdown: {
// Thumbs down gesture - values from TalkingHead
'LeftShoulder.rotation': {
x: 1.75,
y: 0.3,
z: -1.4
},
// Using middle of array ranges
'LeftArm.rotation': {
x: 1.6,
y: -0.5,
z: 1.1
},
// Using middle of array ranges
'LeftForeArm.rotation': {
x: -2.015,
y: 0.406,
z: 1.575
},
// Key rotation for thumbdown
'LeftHand.rotation': {
x: -0.176,
y: -0.206,
z: -0.208
},
// Thumb pointing down
'LeftHandThumb1.rotation': {
x: 0.208,
y: -0.189,
z: 0.685
},
'LeftHandThumb2.rotation': {
x: 0.129,
y: -0.285,
z: -0.163
},
'LeftHandThumb3.rotation': {
x: -0.047,
y: 0.068,
z: 0.401
},
// Fingers folded
'LeftHandIndex1.rotation': {
x: 1.412,
y: -0.102,
z: -0.152
},
'LeftHandIndex2.rotation': {
x: 1.903,
y: -0.16,
z: -0.114
},
'LeftHandIndex3.rotation': {
x: 0.535,
y: -0.017,
z: -0.062
},
'LeftHandMiddle1.rotation': {
x: 1.424,
y: -0.103,
z: -0.12
},
'LeftHandMiddle2.rotation': {
x: 1.919,
y: -0.162,
z: -0.114
},
'LeftHandMiddle3.rotation': {
x: 0.44,
y: -0.012,
z: -0.051
},
'LeftHandRing1.rotation': {
x: 1.619,
y: -0.127,
z: -0.053
},
'LeftHandRing2.rotation': {
x: 1.898,
y: -0.16,
z: -0.115
},
'LeftHandRing3.rotation': {
x: 0.262,
y: -4e-3,
z: -0.031
},
'LeftHandPinky1.rotation': {
x: 1.661,
y: -0.131,
z: -0.016
},
'LeftHandPinky2.rotation': {
x: 1.715,
y: -0.067,
z: -0.13
},
'LeftHandPinky3.rotation': {
x: 0.627,
y: -0.023,
z: -0.071
}
},
salute: {
'LeftShoulder.rotation': {
x: 1.696,
y: -0.17,
z: -1.763
},
'LeftArm.rotation': {
x: 0.886,
y: -0.295,
z: 0.889
},
'LeftForeArm.rotation': {
x: 0,
y: 0,
z: 2.18
},
'LeftHand.rotation': {
x: 0.027,
y: -0.31,
z: 0.348
},
// Fingers in salute position
'LeftHandThumb1.rotation': {
x: 1.394,
y: -0.882,
z: 0.967
},
'LeftHandThumb2.rotation': {
x: -0.394,
y: 0.246,
z: 0.112
},
'LeftHandThumb3.rotation': {
x: -0.029,
y: -2e-3,
z: -7e-3
},
'LeftHandIndex1.rotation': {
x: 0.251,
y: -8e-3,
z: -0.084
},
'LeftHandIndex2.rotation': {
x: 0.009,
y: -7e-3,
z: -6e-3
},
'LeftHandIndex3.rotation': {
x: -0.046,
y: 0,
z: 0
},
'LeftHandMiddle1.rotation': {
x: 0.125,
y: -9e-3,
z: -0.061
},
'LeftHandMiddle2.rotation': {
x: 0.094,
y: -5e-3,
z: -1e-3
},
'LeftHandMiddle3.rotation': {
x: 0.092,
y: 0,
z: 0
},
'LeftHandRing1.rotation': {
x: 0.214,
y: -0.011,
z: 0.026
},
'LeftHandRing2.rotation': {
x: 0.103,
y: -4e-3,
z: -6e-3
},
'LeftHandRing3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandPinky1.rotation': {
x: 0.269,
y: 0.005,
z: 0.05
},
'LeftHandPinky2.rotation': {
x: 0.074,
y: -2e-3,
z: 0.003
},
'LeftHandPinky3.rotation': {
x: 0,
y: 0,
z: 0
}
},
side: {
// Side gesture - exact values from TalkingHead
'LeftShoulder.rotation': {
x: 1.755,
y: -0.035,
z: -1.63
},
'LeftArm.rotation': {
x: 1.263,
y: -0.955,
z: 1.024
},
'LeftForeArm.rotation': {
x: 0,
y: 0,
z: 0.8
},
'LeftHand.rotation': {
x: -0.36,
y: -1.353,
z: -0.184
},
// Finger positions
'LeftHandThumb1.rotation': {
x: 0.137,
y: -0.049,
z: 0.863
},
'LeftHandThumb2.rotation': {
x: -0.293,
y: 0.153,
z: -0.193
},
'LeftHandThumb3.rotation': {
x: -0.271,
y: -0.17,
z: 0.18
},
'LeftHandIndex1.rotation': {
x: -0.018,
y: 0.007,
z: 0.28
},
'LeftHandIndex2.rotation': {
x: 0.247,
y: -3e-3,
z: -0.025
},
'LeftHandIndex3.rotation': {
x: 0.13,
y: -1e-3,
z: -0.013
},
'LeftHandMiddle1.rotation': {
x: 0.333,
y: -0.015,
z: 0.182
},
'LeftHandMiddle2.rotation': {
x: 0.313,
y: -5e-3,
z: -0.032
},
'LeftHandMiddle3.rotation': {
x: 0.294,
y: -4e-3,
z: -0.03
},
'LeftHandRing1.rotation': {
x: 0.456,
y: -0.028,
z: -0.092
},
'LeftHandRing2.rotation': {
x: 0.53,
y: -0.014,
z: -0.052
},
'LeftHandRing3.rotation': {
x: 0.478,
y: -0.012,
z: -0.047
},
'LeftHandPinky1.rotation': {
x: 0.647,
y: -0.049,
z: -0.184
},
'LeftHandPinky2.rotation': {
x: 0.29,
y: -4e-3,
z: -0.029
},
'LeftHandPinky3.rotation': {
x: 0.501,
y: -0.013,
z: -0.049
}
},
shrug: {
// Shrug gesture - values from TalkingHead
'Neck.rotation': {
x: 0,
y: 0,
z: 0
},
// TalkingHead uses random values, we use neutral
'Head.rotation': {
x: 0,
y: 0,
z: 0
},
// TalkingHead uses random values, we use neutral
// Right side
'RightShoulder.rotation': {
x: 1.732,
y: -0.058,
z: 1.407
},
'RightArm.rotation': {
x: 1.305,
y: 0.46,
z: 0.118
},
'RightForeArm.rotation': {
x: 1.0,
y: -0.4,
z: -1.637
},
// Using middle of random range
'RightHand.rotation': {
x: -0.048,
y: 0.165,
z: -0.39
},
// Left side
'LeftShoulder.rotation': {
x: 1.713,
y: 0.141,
z: -1.433
},
'LeftArm.rotation': {
x: 1.136,
y: -0.422,
z: -0.416
},
'LeftForeArm.rotation': {
x: 1.42,
y: 0.123,
z: 1.506
},
'LeftHand.rotation': {
x: 0.073,
y: -0.138,
z: 0.064
},
// Right hand fingers
'RightHandThumb1.rotation': {
x: 1.467,
y: 0.599,
z: -1.315
},
'RightHandThumb2.rotation': {
x: -0.255,
y: -0.123,
z: 0.119
},
'RightHandThumb3.rotation': {
x: 0,
y: -2e-3,
z: 0
},
'RightHandIndex1.rotation': {
x: -0.293,
y: -0.066,
z: -0.112
},
'RightHandIndex2.rotation': {
x: 0.181,
y: 0.007,
z: 0.069
},
'RightHandIndex3.rotation': {
x: 0,
y: 0,
z: 0
},
'RightHandMiddle1.rotation': {
x: -0.063,
y: -0.041,
z: 0.032
},
'RightHandMiddle2.rotation': {
x: 0.149,
y: 0.005,
z: 0.05
},
'RightHandMiddle3.rotation': {
x: 0,
y: 0,
z: 0
},
'RightHandRing1.rotation': {
x: 0.152,
y: -0.03,
z: 0.132
},
'RightHandRing2.rotation': {
x: 0.194,
y: 0.007,
z: 0.058
},
'RightHandRing3.rotation': {
x: 0,
y: 0,
z: 0
},
'RightHandPinky1.rotation': {
x: 0.306,
y: -0.015,
z: 0.257
},
'RightHandPinky2.rotation': {
x: 0.15,
y: -3e-3,
z: -3e-3
},
'RightHandPinky3.rotation': {
x: 0,
y: 0,
z: 0
},
// Left hand fingers
'LeftHandThumb1.rotation': {
x: 1.467,
y: -0.599,
z: 1.314
},
'LeftHandThumb2.rotation': {
x: -0.255,
y: 0.123,
z: -0.119
},
'LeftHandThumb3.rotation': {
x: 0,
y: 0.001,
z: 0
},
'LeftHandIndex1.rotation': {
x: -0.293,
y: 0.066,
z: 0.112
},
'LeftHandIndex2.rotation': {
x: 0.181,
y: -7e-3,
z: -0.069
},
'LeftHandIndex3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandMiddle1.rotation': {
x: -0.062,
y: 0.041,
z: -0.032
},
'LeftHandMiddle2.rotation': {
x: 0.149,
y: -5e-3,
z: -0.05
},
'LeftHandMiddle3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandRing1.rotation': {
x: 0.152,
y: 0.03,
z: -0.132
},
'LeftHandRing2.rotation': {
x: 0.194,
y: -7e-3,
z: -0.058
},
'LeftHandRing3.rotation': {
x: 0,
y: 0,
z: 0
},
'LeftHandPinky1.rotation': {
x: 0.306,
y: 0.015,
z: -0.257
},
'LeftHandPinky2.rotation': {
x: 0.15,
y: 0.003,
z: 0.003
},
'LeftHandPinky3.rotation': {
x: 0,
y: 0,
z: 0
}
},
namaste: {
// Prayer position - values from TalkingHead
// Right side
'RightShoulder.rotation': {
x: 1.758,
y: 0.099,
z: 1.604
},
'RightArm.rotation': {
x: 0.862,
y: -0.292,
z: -0.932
},
'RightForeArm.rotation': {
x: 0.083,
y: 0.066,
z: -1.791
},
'RightHand.rotation': {
x: -0.52,
y: -1e-3,
z: -0.176
},
// Left side
'LeftShoulder.rotation': {
x: 1.711,
y: -2e-3,
z: -1.625
},
'LeftArm.rotation': {
x: 0.683,
y: 0.334,
z: 0.977
},
'LeftForeArm.rotation': {
x: 0.086,
y: -0.066,
z: 1.843
},
'LeftHand.rotation': {
x: -0.595,
y: -0.229,
z: 0.096
},
// Right hand fingers
'RightHandThumb1.rotation': {
x: 0.227,
y: 0.418,
z: -0.776
},
'RightHandThumb2.rotation': {
x: -0.011,
y: -3e-3,
z: 0.171
},
'RightHandThumb3.rotation': {
x: -0.041,
y: -1e-3,
z: -0.013
},
'RightHandIndex1.rotation': {
x: -0.236,
y: 0.003,
z: -0.028
},
'RightHandIndex2.rotation': {
x: 0.004,
y: 0,
z: 0.001
},
'RightHandIndex3.rotation': {
x: 0.002,
y: 0,
z: 0
},
'RightHandMiddle1.rotation': {
x: -0.236,
y: 0.003,
z: -0.028
},
'RightHandMiddle2.rotation': {
x: 0.004,
y: 0,
z: 0.001
},
'RightHandMiddle3.rotation': {
x: 0.002,
y: 0,
z: 0
},
'RightHandRing1.rotation': {
x: -0.236,
y: 0.003,
z: -0.028
},
'RightHandRing2.rotation': {
x: 0.004,
y: 0,
z: 0.001
},
'RightHandRing3.rotation': {
x: 0.002,
y: 0,
z: 0
},
'RightHandPinky1.rotation': {
x: -0.236,
y: 0.003,
z: -0.028
},
'RightHandPinky2.rotation': {
x: 0.004,
y: 0,
z: 0.001
},
'RightHandPinky3.rotation': {
x: 0.002,
y: 0,
z: 0
},
// Left hand fingers
'LeftHandThumb1.rotation': {
x: 0.404,
y: -0.05,
z: 0.537
},
'LeftHandThumb2.rotation': {
x: -0.02,
y: 0.004,
z: -0.154
},
'LeftHandThumb3.rotation': {
x: -0.049,
y: 0.002,
z: -0.019
},
'LeftHandIndex1.rotation': {
x: -0.113,
y: -1e-3,
z: 0.014
},
'LeftHandIndex2.rotation': {
x: 0.003,
y: 0,
z: 0
},
'LeftHandIndex3.rotation': {
x: 0.002,
y: 0,
z: 0
},
'LeftHandMiddle1.rotation': {
x: -0.113,
y: -1e-3,
z: 0.014
},
'LeftHandMiddle2.rotation': {
x: 0.003,
y: 0,
z: 0
},
'LeftHandMiddle3.rotation': {
x: 0.002,
y: 0,
z: 0
},
'LeftHandRing1.rotation': {
x: -0.113,
y: -1e-3,
z: 0.014
},
'LeftHandRing2.rotation': {
x: 0.004,
y: 0,
z: 0
},
'LeftHandRing3.rotation': {
x: 0.002,
y: 0,
z: 0
},
'LeftHandPinky1.rotation': {
x: -0.122,
y: -1e-3,
z: -0.057
},
'LeftHandPinky2.rotation': {
x: 0.012,
y: 0.001,
z: 0.07
},
'LeftHandPinky3.rotation': {
x: 0.002,
y: 0,
z: 0
}
}
};
/**
* Mirror a gesture (swap left/right and invert appropriate axes)
*/
function mirrorGesture(gesture) {
const mirrored = {};
Object.entries(gesture).forEach(([boneKey, rotation]) => {
let mirroredBoneKey = boneKey;
if (boneKey.includes('Left')) {
mirroredBoneKey = boneKey.replace('Left', 'Right');
} else if (boneKey.includes('Right')) {
mirroredBoneKey = boneKey.replace('Right', 'Left');
}
mirrored[mirroredBoneKey] = {
x: rotation.x,
y: -rotation.y,
z: -rotation.z
};
});
return mirrored;
}
/**
* Get all unique bone names from gesture templates
*/
function getAllGestureBones() {
const bones = new Set();
Object.values(gestureTemplates).forEach(gesture => {
Object.keys(gesture).forEach(boneKey => {
const boneName = boneKey.replace('.rotation', '');
bones.add(boneName);
// Also add mirrored version
if (boneName.includes('Left')) {
bones.add(boneName.replace('Left', 'Right'));
} else if (boneName.includes('Right')) {
bones.add(boneName.replace('Right', 'Left'));
}
});
});
return Array.from(bones);
}
/**
* Sigmoid factory similar to TalkingHead
* Creates smooth S-curve for natural movement
*/
const sigmoidFactory = (k = 5) => {
// Sigmoid curve creates smooth acceleration and deceleration
// k=5 gives a natural curve for gesture transitions
const base = t => 1 / (1 + Math.exp(-k * t)) - 0.5;
const corr = 0.5 / base(1);
return t => corr * base(2 * Math.max(Math.min(t, 1), 0) - 1) + 0.5;
};
/**
* Hook to manage gesture animations for the avatar
*/
function useGestures(avatarRef, onGestureStateChange, options = {}) {
// console.log('🎯 Using gesture system with TalkingHead values');
const gestureTimeoutRef = react.useRef(null);
const currentGestureRef = react.useRef(null);
const originalRotationsRef = react.useRef({});
/**
* Bone name mapping for different model formats
*/
const boneMapping = {
// Standard to mixamorig
'LeftShoulder': ['mixamorigLeftShoulder', 'mixamorig:LeftShoulder', 'left_shoulder', 'Left_Shoulder'],
'LeftArm': ['mixamorigLeftArm', 'mixamorig:LeftArm', 'left_arm', 'Left_Arm', 'LeftUpperArm'],
'LeftForeArm': ['mixamorigLeftForeArm', 'mixamorig:LeftForeArm', 'left_forearm', 'Left_ForeArm', 'LeftLowerArm'],
'LeftHand': ['mixamorigLeftHand', 'mixamorig:LeftHand', 'left_hand', 'Left_Hand'],
'RightShoulder': ['mixamorigRightShoulder', 'mixamorig:RightShoulder', 'right_shoulder', 'Right_Shoulder'],
'RightArm': ['mixamorigRightArm', 'mixamorig:RightArm', 'right_arm', 'Right_Arm', 'RightUpperArm'],
'RightForeArm': ['mixamorigRightForeArm', 'mixamorig:RightForeArm', 'right_forearm', 'Right_ForeArm', 'RightLowerArm'],
'RightHand': ['mixamorigRightHand', 'mixamorig:RightHand', 'right_hand', 'Right_Hand'],
'Head': ['mixamorigHead', 'mixamorig:Head', 'head'],
'Neck': ['mixamorigNeck', 'mixamorig:Neck', 'neck'],
// Add finger mappings
'LeftHandThumb1': ['mixamorigLeftHandThumb1', 'mixamorig:LeftHandThumb1', 'left_thumb_01'],
'LeftHandThumb2': ['mixamorigLeftHandThumb2', 'mixamorig:LeftHandThumb2', 'left_thumb_02'],
'LeftHandThumb3': ['mixamorigLeftHandThumb3', 'mixamorig:LeftHandThumb3', 'left_thumb_03'],
'LeftHandIndex1': ['mixamorigLeftHandIndex1', 'mixamorig:LeftHandIndex1', 'left_index_01'],
'LeftHandIndex2': ['mixamorigLeftHandIndex2', 'mixamorig:LeftHandIndex2', 'left_index_02'],
'LeftHandIndex3': ['mixamorigLeftHandIndex3', 'mixamorig:LeftHandIndex3', 'left_index_03'],
'LeftHandMiddle1': ['mixamorigLeftHandMiddle1', 'mixamorig:LeftHandMiddle1', 'left_middle_01'],
'LeftHandMiddle2': ['mixamorigLeftHandMiddle2', 'mixamorig:LeftHandMiddle2', 'left_middle_02'],
'LeftHandMiddle3': ['mixamorigLeftHandMiddle3', 'mixamorig:LeftHandMiddle3', 'left_middle_03'],
'LeftHandRing1': ['mixamorigLeftHandRing1', 'mixamorig:LeftHandRing1', 'left_ring_01'],
'LeftHandRing2': ['mixamorigLeftHandRing2', 'mixamorig:LeftHandRing2', 'left_ring_02'],
'LeftHandRing3': ['mixamorigLeftHandRing3', 'mixamorig:LeftHandRing3', 'left_ring_03'],
'LeftHandPinky1': ['mixamorigLeftHandPinky1', 'mixamorig:LeftHandPinky1', 'left_pinky_01'],
'LeftHandPinky2': ['mixamorigLeftHandPinky2', 'mixamorig:LeftHandPinky2', 'left_pinky_02'],
'LeftHandPinky3': ['mixamorigLeftHandPinky3', 'mixamorig:LeftHandPinky3', 'left_pinky_03'],
// Right hand fingers
'RightHandThumb1': ['mixamorigRightHandThumb1', 'mixamorig:RightHandThumb1', 'right_thumb_01'],
'RightHandThumb2': ['mixamorigRightHandThumb2', 'mixamorig:RightHandThumb2', 'right_thumb_02'],
'RightHandThumb3': ['mixamorigRightHandThumb3', 'mixamorig:RightHandThumb3', 'right_thumb_03'],
'RightHandIndex1': ['mixamorigRightHandIndex1', 'mixamorig:RightHandIndex1', 'right_index_01'],
'RightHandIndex2': ['mixamorigRightHandIndex2', 'mixamorig:RightHandIndex2', 'right_index_02'],
'RightHandIndex3': ['mixamorigRightHandIndex3', 'mixamorig:RightHandIndex3', 'right_index_03'],
'RightHandMiddle1': ['mixamorigRightHandMiddle1', 'mixamorig:RightHandMiddle1', 'right_middle_01'],
'RightHandMiddle2': ['mixamorigRightHandMiddle2', 'mixamorig:RightHandMiddle2', 'right_middle_02'],
'RightHandMiddle3': ['mixamorigRightHandMiddle3', 'mixamorig:RightHandMiddle3', 'right_middle_03'],
'RightHandRing1': ['mixamorigRightHandRing1', 'mixamorig:RightHandRing1', 'right_ring_01'],
'RightHandRing2': ['mixamorigRightHandRing2', 'mixamorig:RightHandRing2', 'right_ring_02'],
'RightHandRing3': ['mixamorigRightHandRing3', 'mixamorig:RightHandRing3', 'right_ring_03'],
'RightHandPinky1': ['mixamorigRightHandPinky1', 'mixamorig:RightHandPinky1', 'right_pinky_01'],
'RightHandPinky2': ['mixamorigRightHandPinky2', 'mixamorig:RightHandPinky2', 'right_pinky_02'],
'RightHandPinky3': ['mixamorigRightHandPinky3', 'mixamorig:RightHandPinky3', 'right_pinky_03']
};
/**
* Find bone by name in the avatar hierarchy with automatic mapping
*/
const findBone = react.useCallback(boneName => {
if (!avatarRef.current) return null;
// First try exact name
let bone = null;
avatarRef.current.traverse(child => {
if (child.isBone && child.name === boneName) {
bone = child;
}
});
// If not found, try mapped names
if (!bone && boneMapping[boneName]) {
const possibleNames = boneMapping[boneName];
for (const mappedName of possibleNames) {
avatarRef.current.traverse(child => {
if (child.isBone && child.name === mappedName) {
bone = child;
}
});
if (bone) {
console.log(`🔄 Mapped bone ${boneName} -> ${mappedName}`);
break;
}
}
}
return bone;
}, [avatarRef]);
/**
* Store original bone rotations (rest pose)
*/
const storeOriginalRotations = react.useCallback(() => {
const bones = getAllGestureBones();
let storedCount = 0;
bones.forEach(boneName => {
const bone = findBone(boneName);
if (bone) {
// Store the rest pose rotation
originalRotationsRef.current[boneName] = bone.quaternion.clone();
storedCount++;
console.log(`💾 Stored rest pose for ${boneName}:`, {
x: bone.quaternion.x.toFixed(3),
y: bone.quaternion.y.toFixed(3),
z: bone.quaternion.z.toFixed(3),
w: bone.quaternion.w.toFixed(3)
});
}
});
console.log(`💾 Total stored rest poses: ${storedCount}`);
}, [findBone]);
/**
* Restore bones to original position
*/
const restoreBones = react.useCallback((duration = 500) => {
const startTime = Date.now();
const easing = sigmoidFactory(5);
// Store current positions
const currentRotations = {};
const bones = getAllGestureBones();
bones.forEach(boneName => {
const bone = findBone(boneName);
if (bone) {
currentRotations[boneName] = bone.quaternion.clone();
}
});
const animate = () => {
const elapsed = Date.now() - startTime;
const progress = Math.min(elapsed / duration, 1);
const alpha = easing(progress);
bones.forEach(boneName => {
const bone = findBone(boneName);
if (bone && originalRotationsRef.current[boneName] && currentRotations[boneName]) {
// Create temporary quaternion for interpolation
const tempQuat = new THREE__namespace.Quaternion();
tempQuat.copy(currentRotations[boneName]);
tempQuat.slerp(originalRotationsRef.current[boneName], alpha);
// Apply the interpolated quaternion
bone.quaternion.copy(tempQuat);
}
});
if (progress < 1) {
requestAnimationFrame(animate);
} else {
console.log('✅ Restored to original position');
}
};
animate();
}, [findBone]);
/**
* Apply gesture rotations to bones
*/
const applyGesture = react.useCallback((gesture, duration = 500, gestureName = '') => {
console.log(`⏱️ Starting gesture animation with duration: ${duration}ms`);
const startTime = Date.now();
const targetRotations = {};
const affectedBones = {};
// Create sigmoid easing function
const easing = sigmoidFactory(5);
// Store current rotations as start point
const startRotations = {};
// Debug: Log gesture input
console.log('🎯 Applying gesture with bones:', Object.keys(gesture));
// Convert euler angles to quaternions
Object.entries(gesture).forEach(([boneKey, rotation]) => {
// Extract bone name (remove .rotation suffix)
const boneName = boneKey.replace('.rotation', '');
const bone = findBone(boneName);
if (bone) {
// Store bone reference
affectedBones[boneName] = bone;
// Store starting rotation
startRotations[boneName] = bone.quaternion.clone();
// Debug: Log original rotation values
console.log(`🦴 ${boneName} (${boneKey}) input rotation (radians):`, rotation);
console.log(`🦴 ${boneName} input rotation (degrees):`, {
x: THREE__namespace.MathUtils.radToDeg(rotation.x).toFixed(1),
y: THREE__namespace.MathUtils.radToDeg(rotation.y).toFixed(1),
z: THREE__namespace.MathUtils.radToDeg(rotation.z).toFixed(1)
});
// Use rotation values directly (no processing)
const x = rotation.x;
const y = rotation.y;
const z = rotation.z;
// Create gesture rotation (TalkingHead uses XYZ order)
const euler = new THREE__namespace.Euler(x, y, z, 'XYZ');
const gestureQuat = new THREE__namespace.Quaternion().setFromEuler(euler).normalize();
// Debug: Log quaternion
console.log(`🎲 ${boneName} gesture quaternion:`, {
x: gestureQuat.x.toFixed(3),
y: gestureQuat.y.toFixed(3),
z: gestureQuat.z.toFixed(3),
w: gestureQuat.w.toFixed(3)
});
// Apply gesture rotation
targetRotations[boneName] = new THREE__namespace.Quaternion();
// TalkingHead applies gestures as absolute rotations, not relative
// The gesture values already include the rest pose
targetRotations[boneName].copy(gestureQuat);
console.log(`🎯 ${boneName} gesture applied as absolute rotation`);
} else {
console.warn(`❌ Bone not found: ${boneName} (from ${boneKey})`);
}
});
// Special adjustment for namaste gesture (from TalkingHead)
if (gestureName === 'namaste') {
console.log('🙏 Namaste gesture - checking adjustment need');
// Log what bones we have before adjustment
console.log('🙏 Bones found for namaste:', Object.keys(targetRotations));
{
console.log('🙏 Namaste adjustment skipped (testing without it)');
}
}
console.log(`📊 Total bones to animate: ${Object.keys(targetRotations).length}`);
const animate = () => {
const elapsed = Date.now() - startTime;
const progress = Math.min(elapsed / duration, 1);
// Apply easing
const alpha = easing(progress);
Object.entries(targetRotations).forEach(([boneName, targetQuat]) => {
const bone = affectedBones[boneName];
if (bone && startRotations[boneName]) {
// Log first frame only
if (elapsed < 50 && boneName === 'LeftArm') {
console.log(`🎬 Animation frame - ${boneName}:`, {
progress: progress.toFixed(2),
alpha: alpha.toFixed(2),
currentQuat: {
x: bone.quaternion.x.toFixed(3),
y: bone.quaternion.y.toFixed(3),
z: bone.quaternion.z.toFixed(3),
w: bone.quaternion.w.toFixed(3)
}
});
}
// Create a temporary quaternion for interpolation
const tempQuat = new THREE__namespace.Quaternion();
tempQuat.copy(startRotations[boneName]);
tempQuat.slerp(targetQuat, alpha);
// Apply the interpolated quaternion
bone.quaternion.copy(tempQuat);
}
});
if (progress < 1) {
requestAnimationFrame(animate);
} else {
console.log('✅ Gesture animation completed');
}
};
animate();
}, [findBone]);
/**
* Play a gesture animation
*/
const playGesture = react.useCallback((gestureName, duration = 2, mirror = false, transitionMs = 500) => {
console.log(`🤚 Playing gesture: ${gestureName} for ${duration}s`);
console.log(`🤚 Avatar ref status:`, {
hasRef: !!avatarRef.current,
refType: avatarRef.current?.type,
refChildren: avatarRef.current?.children?.length
});
// Clear any existing gesture timeout
if (gestureTimeoutRef.current) {
clearTimeout(gestureTimeoutRef.current);
}
// Get gesture template
let gesture = gestureTemplates[gestureName];
if (!gesture) {
console.warn(`Gesture "${gestureName}" not found`);
return;
}
// Mirror if requested
if (mirror) {
gesture = mirrorGesture(gesture);
}
// Store current gesture
currentGestureRef.current = gestureName;
// Notify that gesture is starting
if (onGestureStateChange) {
onGestureStateChange(true);
}
// Apply the gesture
console.log(`🎯 Applying gesture with transitionMs: ${transitionMs}ms`);
applyGesture(gesture, transitionMs, gestureName);
// Schedule restoration
const totalDuration = duration * 1000;
console.log(`⏰ Scheduling gesture end in ${totalDuration}ms (${duration} seconds)`);
gestureTimeou