@nuralogix.ai/tf-face-tracker-worker-ts/lib/taskVision/index.d.ts

MediaPipe Tasks-vision Face Landmarker - Module Worker

/**
 * Copyright 2022 The MediaPipe Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/** Options to configure MediaPipe model loading and processing. */
declare interface BaseOptions_2 {
    /**
     * The model path to the model asset file. Only one of `modelAssetPath` or
     * `modelAssetBuffer` can be set.
     */
    modelAssetPath?: string | undefined;
    /**
     * A buffer or stream reader containing the model asset. Only one of
     * `modelAssetPath` or `modelAssetBuffer` can be set.
     */
    modelAssetBuffer?: Uint8Array | ReadableStreamDefaultReader | undefined;
    /** Overrides the default backend to use for the provided model. */
    delegate?: "CPU" | "GPU" | undefined;
}

/**
 * Copyright 2022 The MediaPipe Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/** A classification category. */
declare interface Category {
    /** The probability score of this label category. */
    score: number;
    /** The index of the category in the corresponding label file. */
    index: number;
    /**
     * The label of this category object. Defaults to an empty string if there is
     * no category.
     */
    categoryName: string;
    /**
     * The display name of the label, which may be translated for different
     * locales. For example, a label, "apple", may be translated into Spanish for
     * display purpose, so that the `display_name` is "manzana". Defaults to an
     * empty string if there is no display name.
     */
    displayName: string;
}

/** Classification results for a given classifier head. */
declare interface Classifications {
    /**
     * The array of predicted categories, usually sorted by descending scores,
     * e.g., from high to low probability.
     */
    categories: Category[];
    /**
     * The index of the classifier head these categories refer to. This is
     * useful for multi-head models.
     */
    headIndex: number;
    /**
     * The name of the classifier head, which is the corresponding tensor
     * metadata name. Defaults to an empty string if there is no such metadata.
     */
    headName: string;
}

/** A connection between two landmarks. */
declare interface Connection {
    start: number;
    end: number;
}

/**
 * Performs face landmarks detection on images.
 *
 * This API expects a pre-trained face landmarker model asset bundle.
 */
declare class FaceLandmarker extends VisionTaskRunner {
    /**
     * Initializes the Wasm runtime and creates a new `FaceLandmarker` from the
     * provided options.
     * @export
     * @param wasmFileset A configuration object that provides the location of the
     *     Wasm binary and its loader.
     * @param faceLandmarkerOptions The options for the FaceLandmarker.
     *     Note that either a path to the model asset or a model buffer needs to
     *     be provided (via `baseOptions`).
     */
    static createFromOptions(wasmFileset: WasmFileset, faceLandmarkerOptions: FaceLandmarkerOptions): Promise<FaceLandmarker>;
    /**
     * Initializes the Wasm runtime and creates a new `FaceLandmarker` based on
     * the provided model asset buffer.
     * @export
     * @param wasmFileset A configuration object that provides the location of the
     *     Wasm binary and its loader.
     * @param modelAssetBuffer An array or a stream containing a binary
     *    representation of the model.
     */
    static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array | ReadableStreamDefaultReader): Promise<FaceLandmarker>;
    /**
     * Initializes the Wasm runtime and creates a new `FaceLandmarker` based on
     * the path to the model asset.
     * @export
     * @param wasmFileset A configuration object that provides the location of the
     *     Wasm binary and its loader.
     * @param modelAssetPath The path to the model asset.
     */
    static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceLandmarker>;
    /**
     * Landmark connections to draw the connection between a face's lips.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_LIPS: Connection[];
    /**
     * Landmark connections to draw the connection between a face's left eye.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_LEFT_EYE: Connection[];
    /**
     * Landmark connections to draw the connection between a face's left eyebrow.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_LEFT_EYEBROW: Connection[];
    /**
     * Landmark connections to draw the connection between a face's left iris.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_LEFT_IRIS: Connection[];
    /**
     * Landmark connections to draw the connection between a face's right eye.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_RIGHT_EYE: Connection[];
    /**
     * Landmark connections to draw the connection between a face's right
     * eyebrow.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_RIGHT_EYEBROW: Connection[];
    /**
     * Landmark connections to draw the connection between a face's right iris.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_RIGHT_IRIS: Connection[];
    /**
     * Landmark connections to draw the face's oval.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_FACE_OVAL: Connection[];
    /**
     * Landmark connections to draw the face's contour.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_CONTOURS: Connection[];
    /**
     * Landmark connections to draw the face's tesselation.
     * @export
     * @nocollapse
     */
    static FACE_LANDMARKS_TESSELATION: Connection[];
    private constructor();
    /**
     * Sets new options for this `FaceLandmarker`.
     *
     * Calling `setOptions()` with a subset of options only affects those options.
     * You can reset an option back to its default value by explicitly setting it
     * to `undefined`.
     *
     * @export
     * @param options The options for the face landmarker.
     */
    setOptions(options: FaceLandmarkerOptions): Promise<void>;
    /**
     * Performs face landmarks detection on the provided single image and waits
     * synchronously for the response. Only use this method when the
     * FaceLandmarker is created with running mode `image`.
     *
     * @export
     * @param image An image to process.
     * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
     *    to process the input image before running inference.
     * @return The detected face landmarks.
     */
    detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): FaceLandmarkerResult;
    /**
     * Performs face landmarks detection on the provided video frame and waits
     * synchronously for the response. Only use this method when the
     * FaceLandmarker is created with running mode `video`.
     *
     * @export
     * @param videoFrame A video frame to process.
     * @param timestamp The timestamp of the current frame, in ms.
     * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
     *    to process the input image before running inference.
     * @return The detected face landmarks.
     */
    detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): FaceLandmarkerResult;
}

/** Options to configure the MediaPipe FaceLandmarker Task */
declare interface FaceLandmarkerOptions extends VisionTaskOptions {
    /**
     * The maximum number of faces can be detected by the FaceLandmarker.
     * Defaults to 1.
     */
    numFaces?: number | undefined;
    /**
     * The minimum confidence score for the face detection to be considered
     * successful. Defaults to 0.5.
     */
    minFaceDetectionConfidence?: number | undefined;
    /**
     * The minimum confidence score of face presence score in the face landmark
     * detection. Defaults to 0.5.
     */
    minFacePresenceConfidence?: number | undefined;
    /**
     * The minimum confidence score for the face tracking to be considered
     * successful. Defaults to 0.5.
     */
    minTrackingConfidence?: number | undefined;
    /**
     * Whether FaceLandmarker outputs face blendshapes classification. Face
     * blendshapes are used for rendering the 3D face model.
     */
    outputFaceBlendshapes?: boolean | undefined;
    /**
     * Whether FaceLandmarker outputs facial transformation_matrix. Facial
     * transformation matrix is used to transform the face landmarks in canonical
     * face to the detected face, so that users can apply face effects on the
     * detected landmarks.
     */
    outputFacialTransformationMatrixes?: boolean | undefined;
}

/**
 * Represents the face landmarks deection results generated by `FaceLandmarker`.
 */
declare interface FaceLandmarkerResult {
    /** Detected face landmarks in normalized image coordinates. */
    faceLandmarks: NormalizedLandmark[][];
    /** Optional face blendshapes results. */
    faceBlendshapes: Classifications[];
    /** Optional facial transformation matrix. */
    facialTransformationMatrixes: Matrix[];
}

/**
 * Options for image processing.
 *
 * If both region-or-interest and rotation are specified, the crop around the
 * region-of-interest is extracted first, then the specified rotation is applied
 * to the crop.
 */
declare interface ImageProcessingOptions {
    /**
     * The optional region-of-interest to crop from the image. If not specified,
     * the full image is used.
     *
     * Coordinates must be in [0,1] with 'left' < 'right' and 'top' < bottom.
     */
    regionOfInterest?: RectF;
    /**
     * The rotation to apply to the image (or cropped region-of-interest), in
     * degrees clockwise.
     *
     * The rotation must be a multiple (positive or negative) of 90°.
     */
    rotationDegrees?: number;
}

/**
 * Valid types of image sources which we can run our GraphRunner over.
 *
 * @deprecated Use TexImageSource instead.
 */
declare type ImageSource = TexImageSource;

/**
 * Copyright 2023 The MediaPipe Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/** A two-dimensional matrix. */
declare interface Matrix {
    /** The number of rows. */
    rows: number;
    /** The number of columns. */
    columns: number;
    /** The values as a flattened one-dimensional array. */
    data: number[];
}

/**
 * Copyright 2022 The MediaPipe Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * Normalized Landmark represents a point in 3D space with x, y, z coordinates.
 * x and y are normalized to [0.0, 1.0] by the image width and height
 * respectively. z represents the landmark depth, and the smaller the value the
 * closer the landmark is to the camera. The magnitude of z uses roughly the
 * same scale as x.
 */
declare interface NormalizedLandmark {
    /** The x coordinates of the normalized landmark. */
    x: number;
    /** The y coordinates of the normalized landmark. */
    y: number;
    /** The z coordinates of the normalized landmark. */
    z: number;
    /** The likelihood of the landmark being visible within the image. */
    visibility: number;
}

/**
 * Defines a rectangle, used e.g. as part of detection results or as input
 * region-of-interest.
 *
 * The coordinates are normalized with respect to the image dimensions, i.e.
 * generally in [0,1] but they may exceed these bounds if describing a region
 * overlapping the image. The origin is on the top-left corner of the image.
 */
declare interface RectF {
    left: number;
    top: number;
    right: number;
    bottom: number;
}

/**
 * The two running modes of a vision task.
 * 1) The image mode for processing single image inputs.
 * 2) The video mode for processing decoded frames of a video.
 */
declare type RunningMode = "IMAGE" | "VIDEO";

/** Base class for all MediaPipe Tasks. */
declare abstract class TaskRunner {
    protected constructor();
    /** Configures the task with custom options. */
    abstract setOptions(options: TaskRunnerOptions): Promise<void>;
    /**
     * Closes and cleans up the resources held by this task.
     * @export
     */
    close(): void;
}

/** Options to configure MediaPipe Tasks in general. */
declare interface TaskRunnerOptions {
    /** Options to configure the loading of the model assets. */
    baseOptions?: BaseOptions_2;
}

/** The options for configuring a MediaPipe vision task. */
declare interface VisionTaskOptions extends TaskRunnerOptions {
    /**
     * The canvas element to bind textures to. This has to be set for GPU
     * processing. The task will initialize a WebGL context and throw an error if
     * this fails (e.g. if you have already initialized a different type of
     * context).
     */
    canvas?: HTMLCanvasElement | OffscreenCanvas;
    /**
     * The running mode of the task. Default to the image mode.
     * Vision tasks have two running modes:
     * 1) The image mode for processing single image inputs.
     * 2) The video mode for processing decoded frames of a video.
     */
    runningMode?: RunningMode;
}

/** Base class for all MediaPipe Vision Tasks. */
declare abstract class VisionTaskRunner extends TaskRunner {
    protected constructor();
    /**
     * Closes and cleans up the resources held by this task.
     * @export
     */
    close(): void;
}

/**
 * Copyright 2022 The MediaPipe Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/** An object containing the locations of the Wasm assets */
declare interface WasmFileset {
    /** The path to the Wasm loader script. */
    wasmLoaderPath: string;
    /** The path to the Wasm binary. */
    wasmBinaryPath: string;
    /** The optional path to the asset loader script. */
    assetLoaderPath?: string;
    /** The optional path to the assets binary. */
    assetBinaryPath?: string;
}

declare enum ObjectFitType {
    CONTAIN = "contain",
    COVER = "cover",
    NONE = "none"
}

interface MediaPipeOptions {
    wasmLoaderFile: ArrayBuffer | undefined;
    wasmFile: ArrayBuffer | undefined;
    modelFile: ArrayBuffer | undefined;
    delegate: string;
}

interface FaceTrackerSettings {
    faceTrackerWidth?: number;
    faceTrackerHeight?: number;
    displayMediaStream?: boolean;
    objectFit?: ObjectFitType;
    isMaskVisible?: boolean;
}

interface Options {
    faceTrackerType: 'MEDIAPIPE' | 'TFJS';
    mediapipe: MediaPipeOptions;
    // tfjs: TFJSOptions;
    broadcastChannelName?: string;
    mediaElement: HTMLDivElement;
    numOfWorkers: number;
    mirrorVideo?: boolean;
    settings?: FaceTrackerSettings;
}

interface DfxRect {
    x: number;
    y: number;
    width: number;
    height: number;
}

interface Centroids {
    [id: string]: {x: number, y: number };
}

interface FaceTrackerResponse {
    face: DfxFace;
    annotations: {
        silhouette: MeshAnnotation[],
        centroids: Centroids;
    }
    frameTimestamp: DOMHighResTimeStamp;
    frameTrackingTime: DOMHighResTimeStamp;
    frameNumber: number;
    trackedFramesCount: number;
}

interface FaceTrackerInvalidInput {
    error: { code: string }
}

type FTResponse = FaceTrackerResponse | FaceTrackerInvalidInput;

interface TypedArrayInput {
    frameTimestamp: DOMHighResTimeStamp;
    width: number;
    height: number,
    typedArray: Uint8ClampedArray;
}

interface PreparePayload {
    frameNumber: number,
    frameTimestamp: number,
    frameTrackingTime: number,
}

interface PrepareMediaPipePayload extends PreparePayload {
    width: number,
    height: number,
    predictions: FaceLandmarkerResult | undefined
}

interface VideoFrameInput {
    videoFrame: VideoFrame;
    frameTimestamp: number;
}

interface MediaPipeTaskVisionInterface {
    faceTracker: FaceLandmarker | null;
    senderId: string;
    broadcastChannel: BroadcastChannel | null;  
    trackedFramesCount: number;
    isWarmedUp: boolean;
    warmUpFrameNumber: number;
    init(options: MediaPipeOptions): Promise<void>;
    setVersion(mediaPipeVersion: string): void;
    getVersion(): string;
    trackFace(source: VideoFrameInput | TypedArrayInput, frameNumber: number): Promise<FTResponse>;
    trackVideoFrame(videoFrame: VideoFrame): Promise<FaceLandmarkerResult | undefined>;
    trackTypedArray(
        { frameTimestamp, width, height, typedArray}: TypedArrayInput
    ): Promise<FaceLandmarkerResult | undefined>;
    preparePayload(
        { frameNumber, frameTimestamp, frameTrackingTime, width, height, predictions}: PrepareMediaPipePayload
    ): FaceTrackerResponse;
    setOptions(senderId: string, broadcastChannelName?: string): void;
    sendMessage(payload: any, action: string): void;
    destroy(): void;
    isWorker(): boolean;
    onReadyCallback(): void;
    onDestroyCallback(): void;
}

interface PosePoints {
    [x: string]: {
        point: number[],
        valid: boolean;
        estimated: boolean;
        quality: number;
    }  
}

interface DfxFace {
    detected: boolean;
    id: string;
    poseValid: boolean;
    posePoints: PosePoints,
    faceRect: DfxRect;
}

interface MeshAnnotation {
    x: number;
    y: number;
    z: number | undefined;
}

/**
 * @license
 * Copyright 2019 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

declare const proxyMarker: unique symbol;
declare const createEndpoint: unique symbol;
declare const releaseProxy: unique symbol;
/**
 * Interface of values that were marked to be proxied with `comlink.proxy()`.
 * Can also be implemented by classes.
 */
interface ProxyMarked {
    [proxyMarker]: true;
}
/**
 * Takes a type and wraps it in a Promise, if it not already is one.
 * This is to avoid `Promise<Promise<T>>`.
 *
 * This is the inverse of `Unpromisify<T>`.
 */
type Promisify<T> = T extends Promise<unknown> ? T : Promise<T>;
/**
 * Takes a type that may be Promise and unwraps the Promise type.
 * If `P` is not a Promise, it returns `P`.
 *
 * This is the inverse of `Promisify<T>`.
 */
type Unpromisify<P> = P extends Promise<infer T> ? T : P;
/**
 * Takes the raw type of a remote property and returns the type that is visible to the local thread on the proxy.
 *
 * Note: This needs to be its own type alias, otherwise it will not distribute over unions.
 * See https://www.typescriptlang.org/docs/handbook/advanced-types.html#distributive-conditional-types
 */
type RemoteProperty<T> = T extends Function | ProxyMarked ? Remote<T> : Promisify<T>;
/**
 * Takes the raw type of a property as a remote thread would see it through a proxy (e.g. when passed in as a function
 * argument) and returns the type that the local thread has to supply.
 *
 * This is the inverse of `RemoteProperty<T>`.
 *
 * Note: This needs to be its own type alias, otherwise it will not distribute over unions. See
 * https://www.typescriptlang.org/docs/handbook/advanced-types.html#distributive-conditional-types
 */
type LocalProperty<T> = T extends Function | ProxyMarked ? Local<T> : Unpromisify<T>;
/**
 * Proxies `T` if it is a `ProxyMarked`, clones it otherwise (as handled by structured cloning and transfer handlers).
 */
type ProxyOrClone<T> = T extends ProxyMarked ? Remote<T> : T;
/**
 * Inverse of `ProxyOrClone<T>`.
 */
type UnproxyOrClone<T> = T extends RemoteObject<ProxyMarked> ? Local<T> : T;
/**
 * Takes the raw type of a remote object in the other thread and returns the type as it is visible to the local thread
 * when proxied with `Comlink.proxy()`.
 *
 * This does not handle call signatures, which is handled by the more general `Remote<T>` type.
 *
 * @template T The raw type of a remote object as seen in the other thread.
 */
type RemoteObject<T> = {
    [P in keyof T]: RemoteProperty<T[P]>;
};
/**
 * Takes the type of an object as a remote thread would see it through a proxy (e.g. when passed in as a function
 * argument) and returns the type that the local thread has to supply.
 *
 * This does not handle call signatures, which is handled by the more general `Local<T>` type.
 *
 * This is the inverse of `RemoteObject<T>`.
 *
 * @template T The type of a proxied object.
 */
type LocalObject<T> = {
    [P in keyof T]: LocalProperty<T[P]>;
};
/**
 * Additional special comlink methods available on each proxy returned by `Comlink.wrap()`.
 */
interface ProxyMethods {
    [createEndpoint]: () => Promise<MessagePort>;
    [releaseProxy]: () => void;
}
/**
 * Takes the raw type of a remote object, function or class in the other thread and returns the type as it is visible to
 * the local thread from the proxy return value of `Comlink.wrap()` or `Comlink.proxy()`.
 */
type Remote<T> = RemoteObject<T> & (T extends (...args: infer TArguments) => infer TReturn ? (...args: {
    [I in keyof TArguments]: UnproxyOrClone<TArguments[I]>;
}) => Promisify<ProxyOrClone<Unpromisify<TReturn>>> : unknown) & (T extends {
    new (...args: infer TArguments): infer TInstance;
} ? {
    new (...args: {
        [I in keyof TArguments]: UnproxyOrClone<TArguments[I]>;
    }): Promisify<Remote<TInstance>>;
} : unknown) & ProxyMethods;
/**
 * Expresses that a type can be either a sync or async.
 */
type MaybePromise<T> = Promise<T> | T;
/**
 * Takes the raw type of a remote object, function or class as a remote thread would see it through a proxy (e.g. when
 * passed in as a function argument) and returns the type the local thread has to supply.
 *
 * This is the inverse of `Remote<T>`. It takes a `Remote<T>` and returns its original input `T`.
 */
type Local<T> = Omit<LocalObject<T>, keyof ProxyMethods> & (T extends (...args: infer TArguments) => infer TReturn ? (...args: {
    [I in keyof TArguments]: ProxyOrClone<TArguments[I]>;
}) => MaybePromise<UnproxyOrClone<Unpromisify<TReturn>>> : unknown) & (T extends {
    new (...args: infer TArguments): infer TInstance;
} ? {
    new (...args: {
        [I in keyof TArguments]: ProxyOrClone<TArguments[I]>;
    }): MaybePromise<Local<Unpromisify<TInstance>>>;
} : unknown);

declare const faceLandmarker: (options: Options, name: string) => Promise<MediaPipeTaskVisionInterface>;
declare const faceLandmarkerWorker: (options: Options, name?: string) => Promise<Remote<MediaPipeTaskVisionInterface>>;

export { type Remote, faceLandmarker, faceLandmarkerWorker };