lightweight-expression-detector/dist/react/useEmotionDetector.js

A lightweight hybrid expression detection module using MediaPipe face blendshapes

import { useEffect, useRef, useState } from "react";
import { FilesetResolver, FaceLandmarker, } from "@mediapipe/tasks-vision";
import { classifyBlendshapes, } from "../core/classifyBlendshapes";
export function useEmotionDetector() {
    const [emotion, setEmotion] = useState("Neutral 😐");
    const [isDetecting, setIsDetecting] = useState(false);
    const [expressions, setExpressions] = useState([]);
    const videoRef = useRef(null);
    const landmarkerRef = useRef(null);
    const baselineRef = useRef({});
    const frameCount = useRef(0);
    const collectingBaseline = useRef(true);
    const accumulator = useRef({});
    const emotionHistory = useRef([]);
    const baselineFrames = 10;
    useEffect(() => {
        const init = async () => {
            const fileset = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm");
            landmarkerRef.current = await FaceLandmarker.createFromOptions(fileset, {
                baseOptions: {
                    modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
                },
                runningMode: "VIDEO",
                numFaces: 1,
                outputFaceBlendshapes: true,
            });
            const stream = await navigator.mediaDevices.getUserMedia({ video: true });
            if (videoRef.current) {
                videoRef.current.srcObject = stream;
                await videoRef.current.play();
            }
            setIsDetecting(true);
            detectLoop();
        };
        init();
        return () => stop();
    }, []);
    const stop = () => {
        if (videoRef.current?.srcObject) {
            const tracks = videoRef.current.srcObject.getTracks();
            tracks.forEach((t) => t.stop());
        }
        setIsDetecting(false);
    };
    const detectLoop = () => {
        const run = () => {
            const landmarker = landmarkerRef.current;
            const video = videoRef.current;
            if (!landmarker || !video || video.readyState < 2) {
                requestAnimationFrame(run);
                return;
            }
            let result;
            try {
                result = landmarker.detectForVideo(video, performance.now());
            }
            catch (e) {
                requestAnimationFrame(run);
                return;
            }
            const blendshapes = result?.faceBlendshapes?.[0]?.categories ?? [];
            if (collectingBaseline.current) {
                blendshapes.forEach(({ categoryName, score }) => {
                    accumulator.current[categoryName] =
                        (accumulator.current[categoryName] || 0) + score;
                });
                frameCount.current++;
                if (frameCount.current >= baselineFrames) {
                    for (const [key, sum] of Object.entries(accumulator.current)) {
                        baselineRef.current[key] = sum / baselineFrames;
                    }
                    collectingBaseline.current = false;
                    console.log("✅ Baseline ready:", baselineRef.current);
                }
                requestAnimationFrame(run);
                return;
            }
            const predicted = classifyBlendshapes(blendshapes, baselineRef.current, emotionHistory.current);
            setEmotion(predicted.emotion);
            setExpressions(predicted.labelHints);
            requestAnimationFrame(run);
        };
        requestAnimationFrame(run);
    };
    return {
        videoRef,
        emotion,
        expressions,
        isDetecting,
    };
}