@glyph-cat/ml-helpers/lib/types/index.d.ts

Helper functions to use in conjunction with machine learning outputs such as mediapipe.

import { Awaitable } from '@glyph-cat/swiss-army-knife';
import { FaceLandmarker } from '@mediapipe/tasks-vision';
import { FilesetResolver } from '@mediapipe/tasks-vision';
import { GestureRecognizer } from '@mediapipe/tasks-vision';
import { GestureRecognizerResult } from '@mediapipe/tasks-vision';
import { HandLandmarker } from '@mediapipe/tasks-vision';
import { HandLandmarkerResult } from '@mediapipe/tasks-vision';
import { LazyValue } from '@glyph-cat/swiss-army-knife';
import { NormalizedLandmark } from '@mediapipe/tasks-vision';
import { NormalizedLandmark as NormalizedLandmark_2 } from '@mediapipe/hands';
import { PartialRecord } from '@glyph-cat/swiss-army-knife';
import { PoseLandmarker } from '@mediapipe/tasks-vision';
import { PoseLandmarkerResult } from '@mediapipe/tasks-vision';
import { SimpleFiniteStateManager } from 'cotton-box';
import { SimpleStateManager } from 'cotton-box';
import { StringRecord } from '@glyph-cat/swiss-army-knife';
import { Value2D } from '@glyph-cat/swiss-army-knife';

/**
 * @public
 */
export declare class BaseLandmarkAnalyzer<Landmarker extends VisionLandmarker, Result> extends BaseVisionAnalyzer<Landmarker, Result> {
    constructor(videoElement: HTMLVideoElement, initialResult: Result, taskRunnerGetter: LazyValue<Awaitable<Landmarker>>, displayName: string);
}

/**
 * @public
 */
export declare class BaseVisionAnalyzer<TaskRunner extends StringRecord<any>, Result> {
    readonly videoElement: HTMLVideoElement;
    readonly detectionMethodName: 'detectForVideo' | 'recognizeForVideo';
    private static _vision;
    static getVision(): Promise<WasmFileset>;
    protected taskRunner: TaskRunner;
    protected lastRequestedAnimationFrame: number;
    readonly result: SimpleStateManager<Result>;
    readonly state: SimpleFiniteStateManager<VisionAnalyzerState>;
    constructor(videoElement: HTMLVideoElement, initialResult: Result, taskRunnerGetter: LazyValue<Awaitable<TaskRunner>>, detectionMethodName: 'detectForVideo' | 'recognizeForVideo', displayName: string);
    start(): Promise<void>;
    stop(): Promise<void>;
    dispose(): Promise<void>;
    private performAnalysis;
    protected getProcessedResult(rawResult: ReturnType<TaskRunner[typeof BaseVisionAnalyzer.detectionMethodName]>): Result;
}

/**
 * @public
 */
export declare enum BodyPoseLandmark {
    NOSE = 0,
    LEFT_EYE_INNER = 1,
    LEFT_EYE = 2,
    LEFT_EYE_OUTER = 3,
    RIGHT_EYE_INNER = 4,
    RIGHT_EYE = 5,
    RIGHT_EYE_OUTER = 6,
    LEFT_EAR = 7,
    RIGHT_EAR = 8,
    MOUTH_LEFT = 9,
    MOUTH_RIGHT = 10,
    LEFT_SHOULDER = 11,
    RIGHT_SHOULDER = 12,
    LEFT_ELBOW = 13,
    RIGHT_ELBOW = 14,
    LEFT_WRIST = 15,
    RIGHT_WRIST = 16,
    LEFT_PINKY = 17,
    RIGHT_PINKY = 18,
    LEFT_INDEX = 19,
    RIGHT_INDEX = 20,
    LEFT_THUMB = 21,
    RIGHT_THUMB = 22,
    LEFT_HIP = 23,
    RIGHT_HIP = 24,
    LEFT_KNEE = 25,
    RIGHT_KNEE = 26,
    LEFT_ANKLE = 27,
    RIGHT_ANKLE = 28,
    LEFT_HEEL = 29,
    RIGHT_HEEL = 30,
    LEFT_FOOT_INDEX = 31,
    RIGHT_FOOT_INDEX = 32
}

/**
 * @public
 */
export declare class ComplexHandGesture {
    static getFingerCurlAngles(hand: Array<NormalizedLandmark_2>, finger: Finger): Array<number>;
    static determineFingerCurl(hand: Array<NormalizedLandmark_2>, finger: Finger): FingerCurl;
    constructor(fingerCurlStates: PartialRecord<Finger, FingerCurlExpression>);
    isMatchedBy(hand: Array<NormalizedLandmark_2>): boolean;
}

/**
 * @public
 */
export declare interface DrawConnectorOptions {
    /**
     * @defaultValue `'#ffffff'`
     */
    color?: string;
    /**
     * @defaultValue `1`
     */
    lineWidth?: number;
}

/**
 * @public
 */
export declare function drawConnectors(ctx: CanvasRenderingContext2D, landmarks: Array<Value2D>, connections: Array<[start: number, end: number]>, style?: DrawConnectorOptions): void;

/**
 * @public
 */
export declare interface DrawLandmarkOptions {
    /**
     * @defaultValue `'#ffffff'`
     */
    color?: string;
    /**
     * @defaultValue `3`
     */
    radius?: number;
}

/**
 * @public
 */
export declare function drawLandmarks(ctx: CanvasRenderingContext2D, landmarks: Array<Value2D>, style?: DrawLandmarkOptions): void;

/**
 * @public
 */
export declare enum Finger {
    THUMB = "T",
    INDEX = "I",
    MIDDLE = "M",
    RING = "R",
    PINKY = "P"
}

/**
 * @public
 */
export declare enum FingerCurl {
    STRAIGHT = 1,
    HALF = 2,
    FULL = 3
}

/**
 * @public
 */
export declare type FingerCurlExpression = {
    is: FingerCurl;
    isOneOf?: never;
    isNot?: never;
    isNotOneOf?: never;
} | {
    is?: never;
    isOneOf: Array<FingerCurl>;
    isNot?: never;
    isNotOneOf?: never;
} | {
    is?: never;
    isOneOf?: never;
    isNot: FingerCurl;
    isNotOneOf?: never;
} | {
    is?: never;
    isOneOf?: never;
    isNot?: never;
    isNotOneOf: Array<FingerCurl>;
};

/**
 * - A replacement for `import { HAND_CONNECTIONS } from '@mediapipe/hands'`
 * - "TypeError: connections is not iterable" is thrown when trying to use it.
 * - No idea why the f••• it is `undefined`.
 * @public
 */
export declare const HAND_CONNECTIONS: Array<[start: number, end: number]>;

/**
 * @public
 */
export declare enum HandGesture {
    NONE = "None",
    CLOSED_FIST = "Closed_Fist",
    OPEN_PALM = "Open_Palm",
    POINTING_UP = "Pointing_Up",
    THUMB_DOWN = "Thumb_Down",
    THUMB_UP = "Thumb_Up",
    VICTORY = "Victory",
    I_LOVE_YOU = "ILoveYou"
}

/**
 * @public
 */
export declare enum HandPoseLandmark {
    WRIST = 0,
    THUMB_CMC = 1,
    THUMB_MCP = 2,
    THUMB_IP = 3,
    THUMB_TIP = 4,
    INDEX_FINGER_MCP = 5,
    INDEX_FINGER_PIP = 6,
    INDEX_FINGER_DIP = 7,
    INDEX_FINGER_TIP = 8,
    MIDDLE_FINGER_MCP = 9,
    MIDDLE_FINGER_PIP = 10,
    MIDDLE_FINGER_DIP = 11,
    MIDDLE_FINGER_TIP = 12,
    RING_FINGER_MCP = 13,
    RING_FINGER_PIP = 14,
    RING_FINGER_DIP = 15,
    RING_FINGER_TIP = 16,
    PINKY_FINGER_MCP = 17,
    PINKY_FINGER_PIP = 18,
    PINKY_FINGER_DIP = 19,
    PINKY_FINGER_TIP = 20
}

/**
 * @public
 */
export declare class OnePersonBodyPoseAnalyzer extends BaseLandmarkAnalyzer<PoseLandmarker, OnePersonBodyPoseAnalyzerResult> {
    constructor(videoElement: HTMLVideoElement);
    protected getProcessedResult(rawResult: PoseLandmarkerResult): OnePersonBodyPoseAnalyzerResult;
}

/**
 * @public
 */
export declare type OnePersonBodyPoseAnalyzerResult = Array<NormalizedLandmark>;

/**
 * @public
 */
export declare class OnePersonHandGestureAnalyzer extends BaseVisionAnalyzer<GestureRecognizer, OnePersonHandGestureAnalyzerResult> {
    private readonly bodyPoseAnalyzer;
    constructor(bodyPoseAnalyzer: OnePersonBodyPoseAnalyzer);
    protected getProcessedResult(rawResult: GestureRecognizerResult): OnePersonHandGestureAnalyzerResult;
}

/**
 * @public
 */
export declare type OnePersonHandGestureAnalyzerHandResult = [
gesture: HandGesture,
landmarks: Array<NormalizedLandmark>
];

/**
 * @public
 */
export declare interface OnePersonHandGestureAnalyzerResult {
    L?: OnePersonHandGestureAnalyzerHandResult;
    R?: OnePersonHandGestureAnalyzerHandResult;
}

/**
 * @public
 */
export declare class OnePersonHandPoseAnalyzer extends BaseLandmarkAnalyzer<HandLandmarker, OnePersonHandPoseAnalyzerResult> {
    private readonly bodyPoseAnalyzer;
    constructor(bodyPoseAnalyzer: OnePersonBodyPoseAnalyzer);
    protected getProcessedResult(rawResult: HandLandmarkerResult): OnePersonHandPoseAnalyzerResult;
}

/**
 * @public
 */
export declare type OnePersonHandPoseAnalyzerHandResult = Array<NormalizedLandmark>;

/**
 * @public
 */
export declare interface OnePersonHandPoseAnalyzerResult {
    L?: OnePersonHandPoseAnalyzerHandResult;
    R?: OnePersonHandPoseAnalyzerHandResult;
}

/**
 * @public
 */
export declare enum VisionAnalyzerState {
    CREATED = 0,
    INITIALIZING = 1,
    STANDBY = 2,
    ACTIVE = 3,
    DISPOSED = 4
}

export declare type VisionLandmarker = FaceLandmarker | HandLandmarker | PoseLandmarker;

/**
 * "Module '"@mediapipe/tasks-vision"' declares 'WasmFileset' locally, but it is not exported."
 * BRUH... zzz
 */
export declare type WasmFileset = Awaited<ReturnType<typeof FilesetResolver.forVisionTasks>>;

export { }