sandai-react/dist/ai-character/src/managers/PuppeteerManager.d.ts

React components and utilities for the Sandai 3D AI Characters.

import * as poseDetector from "@tensorflow-models/pose-detection";
import { WebCamManager } from "./WebCamManager";
import { FaceExpression, VRMHumanBoneName, VRMManager, VRMPose } from "@davidcks/r3f-vrm";
import { MimicFaceProfileManager } from "./submanagers/MimicFaceProfileManager";
import { DistilbertGoEmotion } from "./submanagers/MimicFaceProfileDefault";
import { PoseConversionProfileManager } from "./submanagers/PoseConversionProfileManager";
export declare class PuppeteerManager {
    /** Uses the Webcam to create an OffscreenCanvas that MediaPipe can work with */
    private webcamManager;
    /** MediaPipe Face Landmarker */
    private faceLandmarker;
    private handLandmarker;
    /** BlazePose Pose Detector */
    private poseDetector;
    /** Override for debugging purposes */
    poseOverride?: poseDetector.Pose;
    /** Manages and converts the output of the MediaPipe Face Landmarker to FaceExpressions.
     *
     * It has a pretrained profile that contains the 52 Face Landmark values that correspond to
     * one of the 27 Distil Go Emotions from google, which are then be converted to Face Expressions
     * that the VRM can use.
     */
    mimicFaceProfileManager: MimicFaceProfileManager;
    /** Manages and converts the output of the BlazePose Pose Predictor and MediaPipe Hand Landmarker to VRMPose.
     *
     * It has submanagers for each hand, arm and leg as well as the body and the head.
     * It takes a VRMPose where only the position of the bones is known and returns and blends
     * the poses that come closest by score based on a predefined set of poses and pose profiles.
     *
     * It delegates the predictions to the afore mentioned submanagers to decouple body portions
     * from each other. Ideally this is done on a per bone basis, but there are too many bones
     * and too little development time right now, so this will have to do for now.
     */
    poseConversionProfileManager: PoseConversionProfileManager;
    /** The inferred Face Expression. This updates whenever the Webcam updates
     *  and should be used to be returned by a frameOverride callback
     */
    currentFaceExpression: Omit<FaceExpression, "duration" | "metadata">;
    /** The inferred Face Blendshapes returned by MediaPipe.
     *  They can be used to change the MimicFaceProfileManagers training
     *  Data to optimize the FaceExpression inferrence.
     */
    currentFaceBlendshapes: Map<string, number>;
    /** The inferred Pose. This updates whenever the Webcam updates
     *  and should be used to be returned by a frameOverride callback
     *  to track the users body
     */
    currentPose: VRMPose;
    /** The inferred Pose that only containes the VRMPose bone positions
     *  that were converted from the BlazePose and MediaPipe Hand Landmarks
     */
    currentVRMPosePositions: Partial<Record<VRMHumanBoneName, {
        position: [number, number, number];
    }>>;
    /** The pose positions. This updates whenever the Webcam updates
     *  and should be used to be returned by a frameOverride callback
     */
    currentPosePositions: poseDetector.Pose;
    /** The VRM Manager for the 3D Model */
    private _vrmManager;
    /** The Pose Conversion Manager that turns BlazePose landmarks into
     *  VRMPose objects
     */
    private _poseConverter;
    /** IK calculator */
    private _poseIK;
    private _poseIK2;
    /** Whether to use the profile based ik or normal ik engine */
    private _poseConversionBackend?;
    constructor(webcamManager: WebCamManager, vrmManager: VRMManager);
    initialize(opt?: {
        backend?: "ik" | "profile" | "ik2";
        withAutoCalibratedPoseDetection?: boolean;
    }): Promise<void>;
    private initializeLandmarker;
    private initializeLandmarkerWithCPU;
    private initializeBlazePose;
    private processFrame;
    private processFaceFrame;
    private processPoseFrame;
    private processFaceBlendshapes;
    getSetupPromiseFactory(): readonly [{
        snapshot: Map<"neutral" | "admiration" | "amusement" | "anger" | "annoyance" | "approval" | "caring" | "confusion" | "curiosity" | "desire" | "disappointment" | "disapproval" | "disgust" | "embarrassment" | "excitement" | "fear" | "gratitude" | "grief" | "joy" | "love" | "nervousness" | "optimism" | "pride" | "realization" | "relief" | "remorse" | "sadness" | "surprise" | `la__${string}` | `la__${number}` | `bo__${string}` | `bo__${number}`, () => void>;
        lifecycle: {
            end: () => void;
        };
    }, (emotion: DistilbertGoEmotion) => Promise<void>, (pose: ReturnType<typeof this.poseConversionProfileManager.getProfileKeys>[number]) => Promise<void>];
    dispose(): void;
}
//# sourceMappingURL=PuppeteerManager.d.ts.map