avatoon
Version:
A React Three Fiber component for realistic avatar lip-syncing and animations.
207 lines (206 loc) • 6.85 kB
JavaScript
import { jsx as u, jsxs as w, Fragment as X } from "react/jsx-runtime";
import { useState as d, useRef as r, useEffect as E, Suspense as Z } from "react";
import { useThree as O, useFrame as D, Canvas as $ } from "@react-three/fiber";
import { useGLTF as z, Environment as Q, OrbitControls as W } from "@react-three/drei";
import * as x from "three";
const H = {
A: "viseme_aa",
// e.g., "a" as in "apple"
B: "viseme_PP",
// lips together
C: "viseme_CH",
// "ch", "sh", "j"
D: "viseme_DD",
// "d", "t", "th"
E: "viseme_E",
// "eh", "ae"
F: "viseme_FF",
// "f", "v"
G: "viseme_kk",
// "g", "k"
H: "viseme_sil",
// silence or breath
I: "viseme_I",
// "ee" sound
J: "viseme_RR",
// "r", "l", "y"
K: "viseme_oo",
// "oo", "u"
X: "viseme_sil"
// default fallback or silent frame
};
function J({
url: l,
goal: i,
onRenderComplete: a,
shouldPlay: f,
visemeJson: m
}) {
const [v, P] = d(!1), F = r(!1), { gl: _ } = O(), [L, q] = d(!1), [S, B] = d([]), h = r(null), V = r(null), { scene: g } = z(l), b = r(null), y = r(null), A = r(null), R = r(null), p = r(null), k = r(null), o = r(null), G = r(null), K = r(null), I = r([]), Y = r(null), C = (e) => e.isBone === !0;
return E(() => {
(() => {
if (!m) return;
const n = m.visemes.map((t) => ({
time: t.time,
viseme: H[t.viseme || ""] || null
})).filter((t) => t.viseme !== null);
if (B(n), m && m.audio_base64) {
const t = new Audio(
`data:audio/wav;base64,${m.audio_base64}`
);
h.current = t;
}
})();
}, []), E(() => {
f && h.current ? h.current.play().then(() => {
q(!0);
}).catch((e) => {
console.warn("User interaction required to start audio:", e);
}) : !f && h.current && (h.current.pause(), q(!1));
}, [f]), E(() => {
g && g.traverse((e) => {
const n = e.name.toLowerCase();
if (e.name === "Head" && (o.current = e), e.isMesh) {
const t = e, c = t.morphTargetDictionary;
if (c) {
const s = {};
for (const M of Object.keys(H)) {
const T = H[M];
c[T] !== void 0 && (s[T] = c[T]);
}
Object.keys(s).length > 0 && I.current.push({ mesh: t, visemes: s });
}
}
if (C(e)) {
switch (e.name) {
case "LeftArm":
b.current = e;
break;
case "RightArm":
A.current = e;
break;
case "LeftForeArm":
p.current = e;
break;
case "RightForeArm":
k.current = e;
break;
case "LeftHand":
y.current = e;
break;
case "RightHand":
R.current = e;
break;
case "Head":
o.current = e;
break;
case "LeftEye":
G.current = e;
break;
case "RightEye":
K.current = e;
break;
}
n.includes("leftarm") && e.quaternion.setFromEuler(new x.Euler(Math.PI / 2, 0, 0)), n.includes("rightarm") && e.quaternion.setFromEuler(new x.Euler(Math.PI / 2, 0, 0));
}
});
}, [g]), D(() => {
if (!F.current && g && _ && !v && (F.current = !0, P(!0), requestAnimationFrame(() => {
a == null || a();
})), !L || !h.current || I.current.length === 0)
return;
const e = h.current.currentTime, n = [...S].reverse().find((t) => t.time <= e);
if (!(!n || n.viseme === Y.current)) {
for (const { mesh: t, visemes: c } of I.current)
if (t.morphTargetInfluences)
for (const s of Object.keys(c)) {
const M = c[s];
t.morphTargetInfluences[M] = 0;
}
for (const { mesh: t, visemes: c } of I.current) {
if (!t.morphTargetInfluences || !n.viseme) continue;
const s = c[n.viseme];
s !== void 0 && (t.morphTargetInfluences[s] = 0.6, setTimeout(() => {
t.morphTargetInfluences && (t.morphTargetInfluences[s] = 0);
}, 50));
}
}
}), D(({ clock: e }) => {
const n = e.getElapsedTime();
if (L) {
const t = e.getElapsedTime();
if (o.current) {
const T = 0.03 * Math.sin(t * 2), N = 0.015 * Math.sin(t * 2.2), U = 0.025 * Math.sin(t * 1.5);
o.current.rotation.set(
-0.2 + T,
// X (nodding with slight upward bias)
0 + U,
// Y (left-right)
0 + N
// Z (tilt)
);
}
}
if (i === "Muscle") {
const t = Math.sin(n * 4) * 0.03;
b.current && b.current.rotation.set(0, 0, Math.PI / 2 + t), A.current && A.current.rotation.set(0, 0, -Math.PI / 2 - t), p.current && p.current.rotation.set(0, 0, Math.PI / 2), k.current && k.current.rotation.set(0, 0, -Math.PI / 2), y.current && y.current.rotation.set(0, 0, Math.PI / 2), R.current && R.current.rotation.set(0, 0, -Math.PI / 2), o.current && o.current.rotation.set(0, 0, 0);
} else i === "Sleep" ? o.current && (o.current.rotation.x = Math.sin(n * 1.5) * 0.05) : (p.current && p.current.rotation.set(0, 0, 0), k.current && k.current.rotation.set(0, 0, 0));
}), /* @__PURE__ */ u("group", { ref: V, position: [0, -0.4, 0], children: /* @__PURE__ */ u("primitive", { object: g }) });
}
function j({ readyToPlay: l }) {
const { camera: i } = O(), [a, f] = d(l ? 17 : 15);
return E(() => {
f(l ? 17 : 15);
}, [l]), D(() => {
i instanceof x.PerspectiveCamera && (i.fov += (a - i.fov) * 0.05, i.updateProjectionMatrix());
}), null;
}
function se({
glbUrl: l,
goal: i,
onRenderComplete: a,
visemeJson: f,
showPlayVoiceButton: m = !1
}) {
const [v, P] = d(!1), [F, _] = d(!1);
return /* @__PURE__ */ w(X, { children: [
m && F && /* @__PURE__ */ u(
"button",
{
className: "talk-button",
onClick: () => P(!v),
children: v ? "⏹️ Stop talking" : "▶️ Talk with me now"
}
),
/* @__PURE__ */ w(
$,
{
"data-testid": "avatoon-canvas",
camera: { position: [0, 1.5, 2.2], fov: 17 },
children: [
/* @__PURE__ */ u("ambientLight", { intensity: 0.6 }),
/* @__PURE__ */ w(Z, { fallback: null, children: [
/* @__PURE__ */ u(
J,
{
url: l,
goal: i,
onRenderComplete: () => {
_(!0), a == null || a();
},
shouldPlay: v,
visemeJson: f
}
),
/* @__PURE__ */ u(Q, { preset: "sunset" })
] }),
/* @__PURE__ */ u(j, { readyToPlay: v }),
/* @__PURE__ */ u(W, { target: [0, 1.2, 0], enablePan: !0, enableZoom: !0, enableRotate: !0 })
]
}
)
] });
}
export {
se as Avatoon
};