wawa-lipsync
Version:
A simple and easy-to-use library for real-time lipsyncing with JS and web audio API.
186 lines (185 loc) • 6.78 kB
JavaScript
var x = Object.defineProperty;
var F = (i, t, s) => t in i ? x(i, t, { enumerable: !0, configurable: !0, writable: !0, value: s }) : i[t] = s;
var c = (i, t, s) => F(i, typeof t != "symbol" ? t + "" : t, s);
function _(i) {
return i.length ? i.reduce((t, s) => t + s, 0) / i.length : 0;
}
var e = /* @__PURE__ */ ((i) => (i.sil = "viseme_sil", i.PP = "viseme_PP", i.FF = "viseme_FF", i.TH = "viseme_TH", i.DD = "viseme_DD", i.kk = "viseme_kk", i.CH = "viseme_CH", i.SS = "viseme_SS", i.nn = "viseme_nn", i.RR = "viseme_RR", i.aa = "viseme_aa", i.E = "viseme_E", i.I = "viseme_I", i.O = "viseme_O", i.U = "viseme_U", i))(e || {});
const A = {
[e.sil]: "silence",
[e.PP]: "plosive",
[e.FF]: "fricative",
[e.TH]: "fricative",
[e.DD]: "plosive",
[e.kk]: "plosive",
[e.CH]: "fricative",
[e.SS]: "fricative",
[e.nn]: "plosive",
[e.RR]: "fricative",
[e.aa]: "vowel",
[e.E]: "vowel",
[e.I]: "vowel",
[e.O]: "vowel",
[e.U]: "vowel"
/* vowel */
};
class R {
constructor(t = {
fftSize: 2048,
historySize: 10
}) {
c(this, "features", null);
c(this, "viseme", e.sil);
c(this, "audioContext");
c(this, "analyser");
c(this, "dataArray");
c(this, "history");
c(this, "historySize");
c(this, "sampleRate");
c(this, "binWidth");
c(this, "bands");
c(this, "audioSource");
c(this, "state", "silence");
const { fftSize: s = 2048, historySize: o = 10 } = t;
this.audioContext = new (window.AudioContext || window.webkitAudioContext)(), this.analyser = this.audioContext.createAnalyser(), this.analyser.fftSize = s, this.dataArray = new Uint8Array(this.analyser.frequencyBinCount), this.history = [], this.historySize = o, this.sampleRate = this.audioContext.sampleRate, this.binWidth = this.sampleRate / s, this.bands = [
{ start: 50, end: 200 },
// Band 1: Low energy
{ start: 200, end: 400 },
// Band 2: F1 lower
{ start: 400, end: 800 },
// Band 3: F1 mid
{ start: 800, end: 1500 },
// Band 4: F2 front
{ start: 1500, end: 2500 },
// Band 5: F2/F3
{ start: 2500, end: 4e3 },
// Band 6: Fricatives
{ start: 4e3, end: 8e3 }
// Band 7: High fricatives
];
}
connectAudio(t) {
if (this.audioContext.resume(), this.history = [], this.features = null, this.state = "silence", this.audioSource === t)
return;
if (this.audioSource = t, !t.src) {
console.warn("An audio source must be set before connecting");
return;
}
this.audioContext.createMediaElementSource(t).connect(this.analyser), this.analyser.connect(this.audioContext.destination);
}
// Connect live microphone
async connectMicrophone() {
try {
const t = await navigator.mediaDevices.getUserMedia({ audio: !0 }), s = this.audioContext.createMediaStreamSource(t);
return s.connect(this.analyser), this.analyser.connect(this.audioContext.destination), s;
} catch (t) {
throw console.error("Error accessing microphone:", t), t;
}
}
extractFeatures() {
this.analyser.getByteFrequencyData(this.dataArray);
const t = this.bands.map(({ start: r, end: u }) => {
const d = Math.round(r / this.binWidth), b = Math.min(
Math.round(u / this.binWidth),
this.dataArray.length - 1
);
return _(Array.from(this.dataArray.slice(d, b))) / 255;
});
let s = 0, o = 0;
for (let r = 0; r < this.dataArray.length; r++) {
const u = r * this.binWidth, d = this.dataArray[r] / 255;
s += d, o += u * d;
}
const a = s > 0 ? o / s : 0, n = _(t), l = t.map((r, u) => {
if (this.history.length < 2) return 0;
const d = this.history[this.history.length - 2].bands[u];
return r - d;
}), v = {
bands: t,
deltaBands: l,
volume: n,
centroid: a
};
return s > 0 && (this.history.push(v), this.history.length > this.historySize && this.history.shift()), v;
}
getAveragedFeatures() {
const t = this.history.length, s = {
volume: 0,
centroid: 0,
bands: Array(this.bands.length).fill(0)
};
for (const a of this.history)
s.volume += a.volume, s.centroid += a.centroid, a.bands.forEach((n, l) => s.bands[l] += n);
const o = s.bands.map((a) => a / t);
return {
volume: s.volume / t,
centroid: s.centroid / t,
bands: o,
deltaBands: o
};
}
detectState() {
const t = this.history[this.history.length - 1];
if (!t) {
this.state = "silence", this.viseme = e.sil;
return;
}
const s = this.getAveragedFeatures(), o = t.volume - s.volume, a = t.centroid - s.centroid, n = this.computeVisemeScores(
t,
s,
o,
a
), l = this.adjustScoresForConsistency(n);
let v = -1 / 0, r = e.sil;
for (const d in l)
l[d] > v && (v = l[d], r = d);
let u = A[r];
this.state = u, this.viseme = r;
}
// Compute scores for each viseme
computeVisemeScores(t, s, o, a) {
const n = {
[e.sil]: 0,
[e.PP]: 0,
[e.FF]: 0,
[e.TH]: 0,
[e.DD]: 0,
[e.kk]: 0,
[e.CH]: 0,
[e.SS]: 0,
[e.nn]: 0,
[e.RR]: 0,
[e.aa]: 0,
[e.E]: 0,
[e.I]: 0,
[e.O]: 0,
[e.U]: 0
}, [l, v, r, u, d, b, C] = t.bands;
if (s.volume < 0.2 && t.volume < 0.2 && (n[e.sil] = 1), Object.entries(A).forEach(([y, m]) => {
m === "plosive" && (o < 0.01 && (n[y] -= 0.5), s.volume < 0.2 && (n[y] += 0.2), a > 1e3 && (n[y] += 0.2));
}), t.centroid > 1e3 && t.centroid < 8e3 && (t.centroid > 7e3 ? n[e.DD] += 0.6 : t.centroid > 5e3 ? n[e.kk] += 0.6 : t.centroid > 4e3 ? (n[e.PP] += 1, C > 0.25 && t.centroid < 6e3 && (n[e.DD] += 1.4)) : n[e.nn] += 0.6), a > 1e3 && t.centroid > 6e3 && s.centroid > 5e3 && t.bands[6] > 0.4 && s.bands[6] > 0.3 && (n[e.FF] = 0.7), s.volume > 0.1 && s.centroid < 6e3 && t.centroid < 6e3) {
const [y, m, h, f, p] = s.bands, g = Math.abs(y - m), w = Math.max(
Math.abs(m - h),
Math.abs(m - f),
Math.abs(h - f)
);
(h > 0.1 || f > 0.1) && (f > h && (n[e.aa] = 0.8, h > m && (n[e.aa] += 0.2)), h > m && h > f && (n[e.I] = 0.7), g < 0.25 && (n[e.U] = 0.7), w < 0.25 && (n[e.O] = 0.9), m > h && h > f && (n[e.E] = 1), h < 0.2 && f > 0.3 && (n[e.I] = 0.7), h > 0.25 && p > 0.25 && (n[e.O] = 0.7), h < 0.15 && p < 0.15 && (n[e.U] = 0.7));
}
return n;
}
// Adjust scores based on current state and viseme for consistency
adjustScoresForConsistency(t) {
const s = { ...t };
if (this.viseme && this.state)
for (const o in s)
o === this.viseme && (s[o] *= 1.3);
return s;
}
processAudio() {
this.features = this.extractFeatures(), this.detectState();
}
}
export {
R as Lipsync,
e as VISEMES
};