avatoon
Version:
A React Three Fiber component for realistic avatar lip-syncing and animations.
2 lines (1 loc) • 11.5 kB
JavaScript
(function(l,s){typeof exports=="object"&&typeof module<"u"?s(exports,require("react/jsx-runtime"),require("react"),require("@react-three/fiber"),require("@react-three/drei"),require("three"),require("three/examples/jsm/utils/SkeletonUtils")):typeof define=="function"&&define.amd?define(["exports","react/jsx-runtime","react","@react-three/fiber","@react-three/drei","three","three/examples/jsm/utils/SkeletonUtils"],s):(l=typeof globalThis<"u"?globalThis:l||self,s(l.Avatoon={},l.jsxRuntime,l.React,l.fiber,l.drei,l.THREE,l.SkeletonUtils))})(this,function(l,s,e,b,D,W,z){"use strict";function j(r){const t=Object.create(null,{[Symbol.toStringTag]:{value:"Module"}});if(r){for(const n in r)if(n!=="default"){const c=Object.getOwnPropertyDescriptor(r,n);Object.defineProperty(t,n,c.get?c:{enumerable:!0,get:()=>r[n]})}}return t.default=r,Object.freeze(t)}const O=j(W),H={A:"viseme_aa",B:"viseme_PP",C:"viseme_CH",D:"viseme_DD",E:"viseme_E",F:"viseme_FF",G:"viseme_kk",H:"viseme_sil",I:"viseme_I",J:"viseme_RR",K:"viseme_U",X:"viseme_sil",sil:"viseme_sil",PP:"viseme_PP",FF:"viseme_FF",TH:"viseme_TH",DD:"viseme_DD",kk:"viseme_kk",CH:"viseme_CH",SS:"viseme_SS",nn:"viseme_nn",RR:"viseme_RR",aa:"viseme_aa",O:"viseme_O",U:"viseme_U"};function Y(r,t){return r.visemes.map(n=>({time:n.time,viseme:t[n.viseme||""]||null})).filter(n=>n.viseme!==null)}function $(r,t){for(let n=r.length-1;n>=0;n--)if(r[n].time<=t)return r[n];return null}function N(r,t=.15){return r<0||r>t?0:1-Math.abs(r/t*2-1)}function X({url:r,goal:t,onRenderComplete:n,shouldPlay:c,visemeJson:g}){const[_,F]=e.useState(!1),I=e.useRef(!1),{gl:k}=b.useThree(),[T,d]=e.useState(!1),[a,u]=e.useState([]),i=e.useRef(null),f=e.useRef(null),{scene:m}=D.useGLTF(r),y=e.useMemo(()=>z.clone(m),[m]),P=e.useRef(null),R=e.useRef(null),B=e.useRef(null),q=e.useRef(null),C=e.useRef(null),w=e.useRef(null),A=e.useRef(null),ue=e.useRef(null),ce=e.useRef(null),L=e.useRef([]),Z=e.useRef([]),le=e.useRef({timer:0,nextAt:3,value:0}),K=e.useRef(null),fe=o=>o.isBone===!0;return e.useEffect(()=>((()=>{if(g&&(u(Y(g,H)),g&&g.audio_base64&&typeof window<"u"&&typeof Audio<"u")){const p=new Audio(`data:audio/wav;base64,${g.audio_base64}`);i.current=p}})(),()=>{i.current&&(i.current.pause(),i.current.src="",i.current=null)}),[g]),e.useEffect(()=>{c&&i.current?i.current.play().then(()=>{d(!0)}).catch(o=>{console.warn("User interaction required to start audio:",o)}):!c&&i.current&&(i.current.pause(),d(!1))},[c]),e.useEffect(()=>{y&&y.traverse(o=>{const p=o.name.toLowerCase();if(o.name==="Head"&&(A.current=o),o.isMesh){const v=o,h=v.morphTargetDictionary;if(h){const E={};for(const M of Object.keys(H)){const x=H[M];h[x]!==void 0&&(E[x]=h[x])}Object.keys(E).length>0&&L.current.push({mesh:v,visemes:E});const S=["eyeBlinkLeft","eyeBlinkRight","eyesClosed","blink"].map(M=>h[M]).filter(M=>M!==void 0);S.length>0&&Z.current.push({mesh:v,indices:S})}}if(fe(o)){switch(o.name){case"LeftArm":P.current=o;break;case"RightArm":B.current=o;break;case"LeftForeArm":C.current=o;break;case"RightForeArm":w.current=o;break;case"LeftHand":R.current=o;break;case"RightHand":q.current=o;break;case"Head":A.current=o;break;case"LeftEye":ue.current=o;break;case"RightEye":ce.current=o;break}p.includes("leftarm")&&o.quaternion.setFromEuler(new O.Euler(Math.PI/2,0,0)),p.includes("rightarm")&&o.quaternion.setFromEuler(new O.Euler(Math.PI/2,0,0))}})},[y]),b.useFrame((o,p)=>{const v=Z.current;if(v.length===0)return;const h=le.current;h.timer+=p;const E=.15,S=h.timer-h.nextAt;let M=0;if(S>=0&&(S<=E?M=N(S,E):(h.timer=0,h.nextAt=2+Math.random()*3)),M!==h.value){h.value=M;for(const{mesh:x,indices:me}of v)if(x.morphTargetInfluences)for(const he of me)x.morphTargetInfluences[he]=M}}),b.useFrame(()=>{if(!I.current&&y&&k&&!_&&(I.current=!0,F(!0),requestAnimationFrame(()=>{n==null||n()})),!T||!i.current||L.current.length===0)return;const o=i.current.currentTime,p=$(a,o);if(!(!p||p.viseme===K.current)){K.current=p.viseme;for(const{mesh:v,visemes:h}of L.current)if(v.morphTargetInfluences)for(const E of Object.keys(h)){const S=h[E];v.morphTargetInfluences[S]=0}for(const{mesh:v,visemes:h}of L.current){if(!v.morphTargetInfluences||!p.viseme)continue;const E=h[p.viseme];E!==void 0&&(v.morphTargetInfluences[E]=.6,setTimeout(()=>{v.morphTargetInfluences&&(v.morphTargetInfluences[E]=0)},50))}}}),b.useFrame(({clock:o})=>{const p=o.getElapsedTime();if(T&&A.current){const h=.03*Math.sin(p*2),E=.015*Math.sin(p*2.2),S=.025*Math.sin(p*1.5);A.current.rotation.set(-.2+h,S,E)}if(t==="Muscle"){const v=Math.sin(p*4)*.03;P.current&&P.current.rotation.set(0,0,Math.PI/2+v),B.current&&B.current.rotation.set(0,0,-Math.PI/2-v),C.current&&C.current.rotation.set(0,0,Math.PI/2),w.current&&w.current.rotation.set(0,0,-Math.PI/2),R.current&&R.current.rotation.set(0,0,Math.PI/2),q.current&&q.current.rotation.set(0,0,-Math.PI/2),A.current&&A.current.rotation.set(0,0,0)}else t==="Sleep"?A.current&&(A.current.rotation.x=Math.sin(p*1.5)*.05):(C.current&&C.current.rotation.set(0,0,0),w.current&&w.current.rotation.set(0,0,0))}),s.jsx("group",{ref:f,position:[0,-.4,0],children:s.jsx("primitive",{object:y})})}function G({readyToPlay:r,baseFov:t=24}){const{camera:n}=b.useThree(),[c,g]=e.useState(r?t:t-2);return e.useEffect(()=>{g(r?t:t-2)},[r,t]),b.useFrame(()=>{n instanceof O.PerspectiveCamera&&(n.fov+=(c-n.fov)*.05,n.updateProjectionMatrix())}),null}class U extends e.Component{constructor(){super(...arguments),this.state={hasError:!1}}static getDerivedStateFromError(){return{hasError:!0}}componentDidCatch(t){var n,c;(c=(n=this.props).onError)==null||c.call(n,t)}render(){return this.state.hasError?this.props.fallback??null:this.props.children}}function V({environmentPreset:r="sunset",environmentFiles:t,environmentBackground:n=!1}){return t?s.jsx(D.Environment,{files:t,background:n}):s.jsx(D.Environment,{preset:r,background:n})}const Q={position:"absolute",top:5,left:5,padding:"10px 20px",background:"white",border:"1px solid #d0d0d0",borderRadius:8,fontWeight:600,fontSize:14,color:"#222",cursor:"pointer",boxShadow:"0 2px 10px rgba(0, 0, 0, 0.05)",transition:"all 0.2s ease",zIndex:999},J=e.forwardRef(function({glbUrl:t,goal:n,onRenderComplete:c,onError:g,visemeJson:_,showPlayVoiceButton:F=!1,fov:I=24,cameraPosition:k=[0,1.45,2.3],cameraTarget:T=[0,1.35,0],environmentPreset:d,environmentFiles:a,environmentBackground:u},i){const[f,m]=e.useState(!1),[y,P]=e.useState(!1);return e.useImperativeHandle(i,()=>({play:()=>m(!0),stop:()=>m(!1),toggle:()=>m(R=>!R)}),[]),s.jsxs(s.Fragment,{children:[F&&y&&s.jsx("button",{style:Q,onClick:()=>m(!f),children:f?"⏹️ Stop talking":"▶️ Talk with me now"}),s.jsxs(b.Canvas,{"data-testid":"avatoon-canvas",camera:{position:k,fov:I},children:[s.jsx("ambientLight",{intensity:.6}),s.jsx(U,{onError:g,children:s.jsxs(e.Suspense,{fallback:null,children:[s.jsx(X,{url:t,goal:n,onRenderComplete:()=>{P(!0),c==null||c()},shouldPlay:f,visemeJson:_}),s.jsx(V,{environmentPreset:d,environmentFiles:a,environmentBackground:u})]})}),s.jsx(G,{readyToPlay:f,baseFov:I}),s.jsx(D.OrbitControls,{target:T,enablePan:!0,enableZoom:!0,enableRotate:!0})]})]})});function ee({url:r,isTalking:t}){const{scene:n}=D.useGLTF(r),c=e.useMemo(()=>z.clone(n),[n]),g=e.useRef([]),_=e.useRef([]),F=e.useRef({timer:0,nextAt:3,value:0}),I=e.useRef(null);e.useEffect(()=>{c&&c.traverse(T=>{T.name==="Head"&&(I.current=T);const d=T.name.toLowerCase();(d.includes("leftarm")||d.includes("rightarm"))&&T.quaternion.setFromEuler(new O.Euler(Math.PI/2,0,0));const a=T;if(a.isMesh&&a.morphTargetDictionary){const u=a.morphTargetDictionary,i={};Object.values(H).forEach(m=>{u[m]!==void 0&&(i[m]=u[m])}),Object.keys(i).length>0&&g.current.push({mesh:a,visemes:i});const f=["eyeBlinkLeft","eyeBlinkRight","eyesClosed","blink"].map(m=>u[m]).filter(m=>m!==void 0);f.length>0&&_.current.push({mesh:a,indices:f})}})},[c]),b.useFrame((T,d)=>{const a=_.current;if(a.length===0)return;const u=F.current;u.timer+=d;const i=.15,f=u.timer-u.nextAt;let m=0;if(f>=0&&(f<=i?m=N(f,i):(u.timer=0,u.nextAt=2+Math.random()*3)),m!==u.value){u.value=m;for(const{mesh:y,indices:P}of a)if(y.morphTargetInfluences)for(const R of P)y.morphTargetInfluences[R]=m}});const k=e.useRef({nextChange:0,currentViseme:"viseme_aa",targetInfluence:0});return b.useFrame(({clock:T})=>{if(!t){g.current.forEach(({mesh:a,visemes:u})=>{a.morphTargetInfluences&&Object.values(u).forEach(i=>{a.morphTargetInfluences[i]>.01?a.morphTargetInfluences[i]=O.MathUtils.lerp(a.morphTargetInfluences[i],0,.2):a.morphTargetInfluences[i]=0})});return}const d=T.getElapsedTime();if(d>k.current.nextChange){k.current.nextChange=d+.05+Math.random()*.15;const u=Array.from(new Set(Object.values(H))).filter(f=>f!=="viseme_sil"),i=u[Math.floor(Math.random()*u.length)];k.current.currentViseme=i,k.current.targetInfluence=.3+Math.random()*.7}if(g.current.forEach(({mesh:a,visemes:u})=>{a.morphTargetInfluences&&Object.entries(u).forEach(([i,f])=>{i===k.current.currentViseme?a.morphTargetInfluences[f]=O.MathUtils.lerp(a.morphTargetInfluences[f],k.current.targetInfluence,.2):a.morphTargetInfluences[f]=O.MathUtils.lerp(a.morphTargetInfluences[f],0,.2)})}),I.current){const a=.05*Math.sin(d*3),u=.02*Math.cos(d*2);I.current.rotation.x=-.2+a,I.current.rotation.y=u}}),s.jsx("primitive",{object:c,position:[0,-.4,0]})}function te({glbUrl:r="/avatar.glb",fov:t=24,cameraPosition:n=[0,1.45,2.3],cameraTarget:c=[0,1.35,0],onError:g,environmentPreset:_,environmentFiles:F,environmentBackground:I}){const[k,T]=e.useState(!1);return s.jsxs("div",{style:{width:"100%",height:"100%",position:"relative",minHeight:"400px",backgroundColor:"#f0f0f0"},children:[s.jsxs(b.Canvas,{camera:{position:n,fov:t},style:{borderRadius:"8px"},children:[s.jsx("ambientLight",{intensity:.6}),s.jsx(U,{onError:g,children:s.jsxs(e.Suspense,{fallback:null,children:[s.jsx(ee,{url:r,isTalking:k}),s.jsx(V,{environmentPreset:_,environmentFiles:F,environmentBackground:I})]})}),s.jsx(D.OrbitControls,{target:c,enablePan:!1,enableZoom:!0,enableRotate:!0})]}),s.jsx("button",{onClick:()=>T(!k),style:{position:"absolute",bottom:"20px",left:"50%",transform:"translateX(-50%)",padding:"12px 24px",fontSize:"16px",fontWeight:"bold",color:"#fff",backgroundColor:k?"#ef4444":"#3b82f6",border:"none",borderRadius:"25px",cursor:"pointer",boxShadow:"0 4px 6px rgba(0,0,0,0.1)",transition:"background-color 0.2s",zIndex:10},onMouseOver:d=>d.currentTarget.style.transform="translateX(-50%) scale(1.05)",onMouseOut:d=>d.currentTarget.style.transform="translateX(-50%) scale(1)",children:k?"Stop Talking":"Start Talking"})]})}const ne={0:"sil",1:"aa",2:"aa",3:"O",4:"E",5:"RR",6:"I",7:"U",8:"O",9:"aa",10:"O",11:"aa",12:"sil",13:"RR",14:"nn",15:"SS",16:"CH",17:"TH",18:"FF",19:"DD",20:"kk",21:"PP"};function re(r){return{visemes:r.map(t=>({time:t.audioOffset/1e7,viseme:ne[t.visemeId]??"sil"}))}}const se={p:"PP",t:"DD",S:"CH",T:"TH",f:"FF",k:"kk",i:"I",r:"RR",s:"SS",u:"U","@":"E",a:"aa",e:"E",E:"E",o:"O",O:"O",sil:"sil"};function ie(r){return{visemes:r.filter(t=>t.type===void 0||t.type==="viseme").map(t=>({time:t.time/1e3,viseme:se[t.value]??"sil"}))}}const oe={A:"PP",B:"I",C:"E",D:"aa",E:"O",F:"U",G:"FF",H:"nn",X:"sil"};function ae(r){return{visemes:(Array.isArray(r)?r:r.mouthCues).map(n=>({time:n.start,viseme:oe[n.value]??"sil"}))}}l.Avatoon=J,l.AvatoonErrorBoundary=U,l.AvatoonModel=X,l.CameraFovAnimator=G,l.LipSyncAvatoon=te,l.SceneEnvironment=V,l.fromAzureVisemes=re,l.fromPollySpeechMarks=ie,l.fromRhubarb=ae,Object.defineProperty(l,Symbol.toStringTag,{value:"Module"})});