lightweight-expression-detector/dist/core/classifyBlendshapes.js

A lightweight hybrid expression detection module using MediaPipe face blendshapes

/**
 * Classify emotion from blendshapes and baseline.
 * @param blendshapes Category[] from FaceLandmarker
 * @param baseline Record of baseline scores
 * @param history Array of recent emotion results for smoothing
 */
export function classifyBlendshapes(blendshapes, baseline, history = []) {
    const scoreMap = Object.fromEntries(blendshapes.map(({ categoryName, score }) => [categoryName, score]));
    const delta = {};
    Object.entries(scoreMap).forEach(([key, score]) => {
        delta[key] = score - (baseline[key] ?? 0);
    });
    const emotions = {
        "Happy 😄": (delta["mouthSmileLeft"] ?? 0) +
            (delta["mouthSmileRight"] ?? 0) +
            0.5 * (delta["cheekPuff"] ?? 0),
        "Sad 😢": (delta["mouthFrownLeft"] ?? 0) +
            (delta["mouthFrownRight"] ?? 0) +
            (delta["browInnerUp"] ?? 0),
        "Angry 😠": (delta["browDownLeft"] ?? 0) +
            (delta["browDownRight"] ?? 0) +
            (delta["eyeSquintLeft"] ?? 0) +
            (delta["jawClench"] ?? 0),
        "Surprised 😮": (delta["eyeWideLeft"] ?? 0) +
            (delta["eyeWideRight"] ?? 0) +
            (delta["jawDrop"] ?? 0),
        "Frustrated 😤": (delta["browInnerUp"] ?? 0) +
            (delta["eyeBlinkLeft"] ?? 0) +
            (delta["mouthFrownLeft"] ?? 0),
    };
    let emotion = "Neutral 😐";
    let max = 0;
    Object.entries(emotions).forEach(([label, score]) => {
        if (score > max && score > 0.3) {
            emotion = label;
            max = score;
        }
    });
    // Update and smooth emotion history
    history.push(emotion);
    if (history.length > 15)
        history.shift();
    const freqMap = history.reduce((acc, e) => {
        acc[e] = (acc[e] || 0) + 1;
        return acc;
    }, {});
    const stableEmotion = Object.entries(freqMap).sort((a, b) => b[1] - a[1])[0][0];
    // Extract readable hints
    const labelHints = [];
    blendshapes.forEach(({ categoryName, score }) => {
        if (score > 0.3) {
            if (categoryName.includes("Smile"))
                labelHints.push("Smile");
            if (categoryName.includes("brow"))
                labelHints.push("Eyebrow Move");
            if (categoryName.includes("eyeSquint"))
                labelHints.push("Squint");
            if (categoryName.includes("mouthOpen") ||
                categoryName.includes("jawDrop"))
                labelHints.push("Mouth Open");
        }
    });
    return {
        emotion: stableEmotion,
        labelHints: [...new Set(labelHints)],
    };
}