avatoon
Version:
A React Three Fiber component for realistic avatar lip-syncing and animations.
588 lines (587 loc) • 16.9 kB
JavaScript
import { jsx as p, jsxs as F, Fragment as oe } from "react/jsx-runtime";
import { useState as O, useRef as a, useMemo as N, useEffect as w, Component as ae, forwardRef as ce, useImperativeHandle as ue, Suspense as G } from "react";
import { useThree as Z, useFrame as D, Canvas as K } from "@react-three/fiber";
import { useGLTF as W, Environment as X, OrbitControls as Y } from "@react-three/drei";
import * as x from "three";
import { clone as $ } from "three/examples/jsm/utils/SkeletonUtils";
const L = {
// Legacy single-letter codes
A: "viseme_aa",
// "a" as in "apple"
B: "viseme_PP",
// lips together: p, b, m
C: "viseme_CH",
// ch, sh, j
D: "viseme_DD",
// d, t, th
E: "viseme_E",
// eh, ae
F: "viseme_FF",
// f, v
G: "viseme_kk",
// g, k
H: "viseme_sil",
// silence or breath
I: "viseme_I",
// "ee" sound
J: "viseme_RR",
// r, l, y
K: "viseme_U",
// "oo", "u" (was viseme_oo, which standard rigs don't have)
X: "viseme_sil",
// default fallback / silent frame
// Oculus / Ready Player Me viseme names (accepted as codes directly)
sil: "viseme_sil",
PP: "viseme_PP",
FF: "viseme_FF",
TH: "viseme_TH",
DD: "viseme_DD",
kk: "viseme_kk",
CH: "viseme_CH",
SS: "viseme_SS",
nn: "viseme_nn",
RR: "viseme_RR",
aa: "viseme_aa",
O: "viseme_O",
U: "viseme_U"
};
function le(r, e) {
return r.visemes.map((t) => ({
time: t.time,
viseme: e[t.viseme || ""] || null
})).filter((t) => t.viseme !== null);
}
function fe(r, e) {
for (let t = r.length - 1; t >= 0; t--)
if (r[t].time <= e) return r[t];
return null;
}
function Q(r, e = 0.15) {
return r < 0 || r > e ? 0 : 1 - Math.abs(r / e * 2 - 1);
}
function me({
url: r,
goal: e,
onRenderComplete: t,
shouldPlay: l,
visemeJson: g
}) {
const [_, A] = O(!1), T = a(!1), { gl: v } = Z(), [k, m] = O(!1), [i, o] = O([]), n = a(null), c = a(null), { scene: u } = W(r), b = N(() => $(u), [u]), R = a(null), S = a(null), U = a(null), B = a(null), H = a(null), C = a(null), y = a(null), ee = a(null), te = a(null), V = a([]), q = a([]), re = a({ timer: 0, nextAt: 3, value: 0 }), z = a(null), ne = (s) => s.isBone === !0;
return w(() => ((() => {
if (g && (o(le(g, L)), g && g.audio_base64 && typeof window < "u" && typeof Audio < "u")) {
const h = new Audio(
`data:audio/wav;base64,${g.audio_base64}`
);
n.current = h;
}
})(), () => {
n.current && (n.current.pause(), n.current.src = "", n.current = null);
}), [g]), w(() => {
l && n.current ? n.current.play().then(() => {
m(!0);
}).catch((s) => {
console.warn("User interaction required to start audio:", s);
}) : !l && n.current && (n.current.pause(), m(!1));
}, [l]), w(() => {
b && b.traverse((s) => {
const h = s.name.toLowerCase();
if (s.name === "Head" && (y.current = s), s.isMesh) {
const d = s, f = d.morphTargetDictionary;
if (f) {
const I = {};
for (const M of Object.keys(L)) {
const P = L[M];
f[P] !== void 0 && (I[P] = f[P]);
}
Object.keys(I).length > 0 && V.current.push({ mesh: d, visemes: I });
const E = [
"eyeBlinkLeft",
"eyeBlinkRight",
"eyesClosed",
"blink"
].map((M) => f[M]).filter((M) => M !== void 0);
E.length > 0 && q.current.push({ mesh: d, indices: E });
}
}
if (ne(s)) {
switch (s.name) {
case "LeftArm":
R.current = s;
break;
case "RightArm":
U.current = s;
break;
case "LeftForeArm":
H.current = s;
break;
case "RightForeArm":
C.current = s;
break;
case "LeftHand":
S.current = s;
break;
case "RightHand":
B.current = s;
break;
case "Head":
y.current = s;
break;
case "LeftEye":
ee.current = s;
break;
case "RightEye":
te.current = s;
break;
}
h.includes("leftarm") && s.quaternion.setFromEuler(new x.Euler(Math.PI / 2, 0, 0)), h.includes("rightarm") && s.quaternion.setFromEuler(new x.Euler(Math.PI / 2, 0, 0));
}
});
}, [b]), D((s, h) => {
const d = q.current;
if (d.length === 0) return;
const f = re.current;
f.timer += h;
const I = 0.15, E = f.timer - f.nextAt;
let M = 0;
if (E >= 0 && (E <= I ? M = Q(E, I) : (f.timer = 0, f.nextAt = 2 + Math.random() * 3)), M !== f.value) {
f.value = M;
for (const { mesh: P, indices: se } of d)
if (P.morphTargetInfluences)
for (const ie of se) P.morphTargetInfluences[ie] = M;
}
}), D(() => {
if (!T.current && b && v && !_ && (T.current = !0, A(!0), requestAnimationFrame(() => {
t == null || t();
})), !k || !n.current || V.current.length === 0)
return;
const s = n.current.currentTime, h = fe(i, s);
if (!(!h || h.viseme === z.current)) {
z.current = h.viseme;
for (const { mesh: d, visemes: f } of V.current)
if (d.morphTargetInfluences)
for (const I of Object.keys(f)) {
const E = f[I];
d.morphTargetInfluences[E] = 0;
}
for (const { mesh: d, visemes: f } of V.current) {
if (!d.morphTargetInfluences || !h.viseme) continue;
const I = f[h.viseme];
I !== void 0 && (d.morphTargetInfluences[I] = 0.6, setTimeout(() => {
d.morphTargetInfluences && (d.morphTargetInfluences[I] = 0);
}, 50));
}
}
}), D(({ clock: s }) => {
const h = s.getElapsedTime();
if (k && y.current) {
const f = 0.03 * Math.sin(h * 2), I = 0.015 * Math.sin(h * 2.2), E = 0.025 * Math.sin(h * 1.5);
y.current.rotation.set(-0.2 + f, E, I);
}
if (e === "Muscle") {
const d = Math.sin(h * 4) * 0.03;
R.current && R.current.rotation.set(0, 0, Math.PI / 2 + d), U.current && U.current.rotation.set(0, 0, -Math.PI / 2 - d), H.current && H.current.rotation.set(0, 0, Math.PI / 2), C.current && C.current.rotation.set(0, 0, -Math.PI / 2), S.current && S.current.rotation.set(0, 0, Math.PI / 2), B.current && B.current.rotation.set(0, 0, -Math.PI / 2), y.current && y.current.rotation.set(0, 0, 0);
} else e === "Sleep" ? y.current && (y.current.rotation.x = Math.sin(h * 1.5) * 0.05) : (H.current && H.current.rotation.set(0, 0, 0), C.current && C.current.rotation.set(0, 0, 0));
}), /* @__PURE__ */ p("group", { ref: c, position: [0, -0.4, 0], children: /* @__PURE__ */ p("primitive", { object: b }) });
}
function he({
readyToPlay: r,
baseFov: e = 24
}) {
const { camera: t } = Z(), [l, g] = O(
r ? e : e - 2
);
return w(() => {
g(r ? e : e - 2);
}, [r, e]), D(() => {
t instanceof x.PerspectiveCamera && (t.fov += (l - t.fov) * 0.05, t.updateProjectionMatrix());
}), null;
}
class J extends ae {
constructor() {
super(...arguments), this.state = { hasError: !1 };
}
static getDerivedStateFromError() {
return { hasError: !0 };
}
componentDidCatch(e) {
var t, l;
(l = (t = this.props).onError) == null || l.call(t, e);
}
render() {
return this.state.hasError ? this.props.fallback ?? null : this.props.children;
}
}
function j({
environmentPreset: r = "sunset",
environmentFiles: e,
environmentBackground: t = !1
}) {
return e ? /* @__PURE__ */ p(
X,
{
files: e,
background: t
}
) : /* @__PURE__ */ p(
X,
{
preset: r,
background: t
}
);
}
const de = {
position: "absolute",
top: 5,
left: 5,
padding: "10px 20px",
background: "white",
border: "1px solid #d0d0d0",
borderRadius: 8,
fontWeight: 600,
fontSize: 14,
color: "#222",
cursor: "pointer",
boxShadow: "0 2px 10px rgba(0, 0, 0, 0.05)",
transition: "all 0.2s ease",
zIndex: 999
}, _e = ce(function({
glbUrl: e,
goal: t,
onRenderComplete: l,
onError: g,
visemeJson: _,
showPlayVoiceButton: A = !1,
fov: T = 24,
cameraPosition: v = [0, 1.45, 2.3],
cameraTarget: k = [0, 1.35, 0],
environmentPreset: m,
environmentFiles: i,
environmentBackground: o
}, n) {
const [c, u] = O(!1), [b, R] = O(!1);
return ue(
n,
() => ({
play: () => u(!0),
stop: () => u(!1),
toggle: () => u((S) => !S)
}),
[]
), /* @__PURE__ */ F(oe, { children: [
A && b && /* @__PURE__ */ p(
"button",
{
style: de,
onClick: () => u(!c),
children: c ? "⏹️ Stop talking" : "▶️ Talk with me now"
}
),
/* @__PURE__ */ F(
K,
{
"data-testid": "avatoon-canvas",
camera: { position: v, fov: T },
children: [
/* @__PURE__ */ p("ambientLight", { intensity: 0.6 }),
/* @__PURE__ */ p(J, { onError: g, children: /* @__PURE__ */ F(G, { fallback: null, children: [
/* @__PURE__ */ p(
me,
{
url: e,
goal: t,
onRenderComplete: () => {
R(!0), l == null || l();
},
shouldPlay: c,
visemeJson: _
}
),
/* @__PURE__ */ p(
j,
{
environmentPreset: m,
environmentFiles: i,
environmentBackground: o
}
)
] }) }),
/* @__PURE__ */ p(he, { readyToPlay: c, baseFov: T }),
/* @__PURE__ */ p(
Y,
{
target: k,
enablePan: !0,
enableZoom: !0,
enableRotate: !0
}
)
]
}
)
] });
});
function pe({ url: r, isTalking: e }) {
const { scene: t } = W(r), l = N(() => $(t), [t]), g = a([]), _ = a([]), A = a({ timer: 0, nextAt: 3, value: 0 }), T = a(null);
w(() => {
l && l.traverse((k) => {
k.name === "Head" && (T.current = k);
const m = k.name.toLowerCase();
(m.includes("leftarm") || m.includes("rightarm")) && k.quaternion.setFromEuler(new x.Euler(Math.PI / 2, 0, 0));
const i = k;
if (i.isMesh && i.morphTargetDictionary) {
const o = i.morphTargetDictionary, n = {};
Object.values(L).forEach((u) => {
o[u] !== void 0 && (n[u] = o[u]);
}), Object.keys(n).length > 0 && g.current.push({ mesh: i, visemes: n });
const c = [
"eyeBlinkLeft",
"eyeBlinkRight",
"eyesClosed",
"blink"
].map((u) => o[u]).filter((u) => u !== void 0);
c.length > 0 && _.current.push({ mesh: i, indices: c });
}
});
}, [l]), D((k, m) => {
const i = _.current;
if (i.length === 0) return;
const o = A.current;
o.timer += m;
const n = 0.15, c = o.timer - o.nextAt;
let u = 0;
if (c >= 0 && (c <= n ? u = Q(c, n) : (o.timer = 0, o.nextAt = 2 + Math.random() * 3)), u !== o.value) {
o.value = u;
for (const { mesh: b, indices: R } of i)
if (b.morphTargetInfluences)
for (const S of R) b.morphTargetInfluences[S] = u;
}
});
const v = a({
nextChange: 0,
currentViseme: "viseme_aa",
targetInfluence: 0
});
return D(({ clock: k }) => {
if (!e) {
g.current.forEach(({ mesh: i, visemes: o }) => {
i.morphTargetInfluences && Object.values(o).forEach((n) => {
i.morphTargetInfluences[n] > 0.01 ? i.morphTargetInfluences[n] = x.MathUtils.lerp(
i.morphTargetInfluences[n],
0,
0.2
) : i.morphTargetInfluences[n] = 0;
});
});
return;
}
const m = k.getElapsedTime();
if (m > v.current.nextChange) {
v.current.nextChange = m + 0.05 + Math.random() * 0.15;
const o = Array.from(
new Set(Object.values(L))
).filter((c) => c !== "viseme_sil"), n = o[Math.floor(Math.random() * o.length)];
v.current.currentViseme = n, v.current.targetInfluence = 0.3 + Math.random() * 0.7;
}
if (g.current.forEach(({ mesh: i, visemes: o }) => {
i.morphTargetInfluences && Object.entries(o).forEach(([n, c]) => {
n === v.current.currentViseme ? i.morphTargetInfluences[c] = x.MathUtils.lerp(
i.morphTargetInfluences[c],
v.current.targetInfluence,
0.2
// speed of transition
) : i.morphTargetInfluences[c] = x.MathUtils.lerp(
i.morphTargetInfluences[c],
0,
0.2
);
});
}), T.current) {
const i = 0.05 * Math.sin(m * 3), o = 0.02 * Math.cos(m * 2);
T.current.rotation.x = -0.2 + i, T.current.rotation.y = o;
}
}), /* @__PURE__ */ p("primitive", { object: l, position: [0, -0.4, 0] });
}
function ye({
glbUrl: r = "/avatar.glb",
fov: e = 24,
cameraPosition: t = [0, 1.45, 2.3],
cameraTarget: l = [0, 1.35, 0],
onError: g,
environmentPreset: _,
environmentFiles: A,
environmentBackground: T
}) {
const [v, k] = O(!1);
return /* @__PURE__ */ F(
"div",
{
style: {
width: "100%",
height: "100%",
position: "relative",
minHeight: "400px",
backgroundColor: "#f0f0f0"
},
children: [
/* @__PURE__ */ F(
K,
{
camera: { position: t, fov: e },
style: { borderRadius: "8px" },
children: [
/* @__PURE__ */ p("ambientLight", { intensity: 0.6 }),
/* @__PURE__ */ p(J, { onError: g, children: /* @__PURE__ */ F(G, { fallback: null, children: [
/* @__PURE__ */ p(pe, { url: r, isTalking: v }),
/* @__PURE__ */ p(
j,
{
environmentPreset: _,
environmentFiles: A,
environmentBackground: T
}
)
] }) }),
/* @__PURE__ */ p(
Y,
{
target: l,
enablePan: !1,
enableZoom: !0,
enableRotate: !0
}
)
]
}
),
/* @__PURE__ */ p(
"button",
{
onClick: () => k(!v),
style: {
position: "absolute",
bottom: "20px",
left: "50%",
transform: "translateX(-50%)",
padding: "12px 24px",
fontSize: "16px",
fontWeight: "bold",
color: "#fff",
backgroundColor: v ? "#ef4444" : "#3b82f6",
border: "none",
borderRadius: "25px",
cursor: "pointer",
boxShadow: "0 4px 6px rgba(0,0,0,0.1)",
transition: "background-color 0.2s",
zIndex: 10
},
onMouseOver: (m) => m.currentTarget.style.transform = "translateX(-50%) scale(1.05)",
onMouseOut: (m) => m.currentTarget.style.transform = "translateX(-50%) scale(1)",
children: v ? "Stop Talking" : "Start Talking"
}
)
]
}
);
}
const ge = {
0: "sil",
1: "aa",
2: "aa",
3: "O",
4: "E",
5: "RR",
6: "I",
7: "U",
8: "O",
9: "aa",
10: "O",
11: "aa",
12: "sil",
13: "RR",
14: "nn",
15: "SS",
16: "CH",
17: "TH",
18: "FF",
19: "DD",
20: "kk",
21: "PP"
};
function Ae(r) {
return {
visemes: r.map((e) => ({
time: e.audioOffset / 1e7,
// 100-ns ticks → seconds
viseme: ge[e.visemeId] ?? "sil"
}))
};
}
const ve = {
p: "PP",
t: "DD",
S: "CH",
T: "TH",
f: "FF",
k: "kk",
i: "I",
r: "RR",
s: "SS",
u: "U",
"@": "E",
a: "aa",
e: "E",
E: "E",
o: "O",
O: "O",
sil: "sil"
};
function Re(r) {
return {
visemes: r.filter((e) => e.type === void 0 || e.type === "viseme").map((e) => ({
time: e.time / 1e3,
// ms → seconds
viseme: ve[e.value] ?? "sil"
}))
};
}
const ke = {
A: "PP",
// closed: p, b, m
B: "I",
// slightly open, teeth: consonants / "ee"
C: "E",
// open: eh, ae
D: "aa",
// wide open: ah
E: "O",
// rounded: oh
F: "U",
// puckered: oo, w
G: "FF",
// upper teeth on lower lip: f, v
H: "nn",
// l
X: "sil"
// idle / rest
};
function Se(r) {
return {
visemes: (Array.isArray(r) ? r : r.mouthCues).map((t) => ({
time: t.start,
// already in seconds
viseme: ke[t.value] ?? "sil"
}))
};
}
export {
_e as Avatoon,
J as AvatoonErrorBoundary,
me as AvatoonModel,
he as CameraFovAnimator,
ye as LipSyncAvatoon,
j as SceneEnvironment,
Ae as fromAzureVisemes,
Re as fromPollySpeechMarks,
Se as fromRhubarb
};