openvino-genai-node/types/perfMetrics.d.ts

OpenVINO™ GenAI pipelines for using from Node.js environment

/** Structure holding mean and standard deviation values. */
export type MeanStdPair = {
    mean: number;
    std: number;
};
/** Structure holding summary of statistical values */
export type SummaryStats = MeanStdPair & {
    min: number;
    max: number;
};
/** Structure with raw performance metrics for each generation before any statistics are calculated. */
export type RawMetrics = {
    /** Durations for each generate call in milliseconds. */
    generateDurations: number[];
    /** Durations for the tokenization process in milliseconds. */
    tokenizationDurations: number[];
    /** Durations for the detokenization process in milliseconds. */
    detokenizationDurations: number[];
    /** Times to the first token for each call in milliseconds. */
    timesToFirstToken: number[];
    /** Timestamps of generation every token or batch of tokens in milliseconds. */
    newTokenTimes: number[];
    /** Inference time for each token in milliseconds. */
    tokenInferDurations: number[];
    /** Batch sizes for each generate call. */
    batchSizes: number[];
    /** Total durations for each generate call in milliseconds. */
    durations: number[];
    /** Total inference duration for each generate call in microseconds. */
    inferenceDurations: number[];
    /** Time to compile the grammar in milliseconds. */
    grammarCompileTimes: number[];
};
/** Structure with raw performance metrics for VLM generation. */
export type VLMRawMetrics = {
    /** Durations for embedding preparation in milliseconds. */
    prepareEmbeddingsDurations: number[];
};
/** Structure with raw performance metrics for Whisper generation. */
export type WhisperRawMetrics = {
    /** Durations for features extraction in milliseconds. */
    featuresExtractionDurations: number[];
    /** Durations for word-level timestamps processing in milliseconds. */
    wordLevelTimestampsProcessingDurations: number[];
};
/**
 * Holds performance metrics for each generate call.
 *
 * PerfMetrics holds the following metrics with mean and standard deviations:
    - Time To the First Token (TTFT), ms
    - Time per Output Token (TPOT), ms/token
    - Inference time per Output Token (IPOT), ms/token
    - Generate total duration, ms
    - Inference duration, ms
    - Tokenization duration, ms
    - Detokenization duration, ms
    - Throughput, tokens/s
    - Load time, ms
    - Number of generated tokens
    - Number of tokens in the input prompt
    - Time to initialize grammar compiler for each backend, ms
    - Time to compile grammar, ms
 * Preferable way to access metrics is via getter methods. Getter methods calculate mean and std values from rawMetrics and return pairs.
 * If mean and std were already calculated, getter methods return cached values.
 */
export interface PerfMetrics {
    /** Returns the load time in milliseconds. */
    getLoadTime(): number;
    /** Returns the number of generated tokens. */
    getNumGeneratedTokens(): number;
    /** Returns the number of tokens in the input prompt. */
    getNumInputTokens(): number;
    /** Returns the mean and standard deviation of Time To the First Token (TTFT) in milliseconds. */
    getTTFT(): MeanStdPair;
    /** Returns the mean and standard deviation of Time Per Output Token (TPOT) in milliseconds. */
    getTPOT(): MeanStdPair;
    /** Returns the mean and standard deviation of Inference time Per Output Token in milliseconds. */
    getIPOT(): MeanStdPair;
    /** Returns the mean and standard deviation of throughput in tokens per second. */
    getThroughput(): MeanStdPair;
    /** Returns the mean and standard deviation of the time spent on model inference during generate call in milliseconds. */
    getInferenceDuration(): MeanStdPair;
    /** Returns the mean and standard deviation of generate durations in milliseconds. */
    getGenerateDuration(): MeanStdPair;
    /** Returns the mean and standard deviation of tokenization durations in milliseconds. */
    getTokenizationDuration(): MeanStdPair;
    /** Returns the mean and standard deviation of detokenization durations in milliseconds. */
    getDetokenizationDuration(): MeanStdPair;
    /** Returns a map with the time to initialize the grammar compiler for each backend in milliseconds. */
    getGrammarCompilerInitTimes(): {
        [key: string]: number;
    };
    /** Returns the mean, standard deviation, min, and max of grammar compile times in milliseconds. */
    getGrammarCompileTime(): SummaryStats;
    /** A structure of RawPerfMetrics type that holds raw metrics. */
    rawMetrics: RawMetrics;
    /** Adds the metrics from another PerfMetrics object to this one.
     * @returns The current PerfMetrics instance.
     */
    add(other: PerfMetrics): this;
}
/**
 * Holds performance metrics for each VLM generate call.
 *
 * VLMPerfMetrics extends PerfMetrics with VLM-specific metrics:
 *  - Prepare embeddings duration, ms
 */
export interface VLMPerfMetrics extends PerfMetrics {
    /** Returns the mean and standard deviation of embeddings preparation duration in milliseconds. */
    getPrepareEmbeddingsDuration(): MeanStdPair;
    /** VLM specific raw metrics */
    vlmRawMetrics: VLMRawMetrics;
    /** Adds the metrics from another VLMPerfMetrics object to this one.
     * @returns The current VLMPerfMetrics instance.
     */
    add(other: VLMPerfMetrics): this;
}
/**
 * Holds performance metrics for each Whisper generate call.
 *
 * WhisperPerfMetrics extends PerfMetrics with Whisper-specific metrics:
 *  - Features extraction duration, ms
 *  - Word-level timestamps processing duration, ms
 */
export interface WhisperPerfMetrics extends PerfMetrics {
    /** Returns the mean and standard deviation of features extraction duration in milliseconds. */
    getFeaturesExtractionDuration(): MeanStdPair;
    /** Returns the mean and standard deviation of word-level timestamps processing duration in milliseconds. */
    getWordLevelTimestampsProcessingDuration(): MeanStdPair;
    /** Whisper-specific raw metrics */
    whisperRawMetrics: WhisperRawMetrics;
    /** Adds the metrics from another WhisperPerfMetrics object to this one.
     * @returns The current WhisperPerfMetrics instance.
     */
    add(other: WhisperPerfMetrics): this;
}
/**
 * Holds performance metrics for each Text2Speech generate call.
 *
 * Text2SpeechPerfMetrics extends PerfMetrics with speech-generation-specific metrics:
 *  - Number of generated audio samples
 */
export interface Text2SpeechPerfMetrics extends PerfMetrics {
    /** Returns the total number of generated audio samples. */
    getNumGeneratedSamples(): number;
    /** Adds the metrics from another Text2SpeechPerfMetrics object to this one.
     * @returns The current Text2SpeechPerfMetrics instance.
     */
    add(other: Text2SpeechPerfMetrics): this;
}
/** Raw performance metrics for image generation pipelines. */
export type RawImageGenerationPerfMetrics = {
    /** UNet inference duration for each denoising step, milliseconds. */
    unetInferenceDurations: number[];
    /** Transformer inference duration for each denoising step, milliseconds. */
    transformerInferenceDurations: number[];
    /** Total iteration duration for each denoising step, milliseconds. */
    iterationDurations: number[];
};
/**
 * Holds performance metrics for each image generate call.
 */
export interface ImageGenerationPerfMetrics {
    /** Returns model load time in milliseconds. */
    getLoadTime(): number;
    /** Returns total generate call duration in milliseconds. */
    getGenerateDuration(): number;
    /** Returns mean/std duration of one denoising iteration in milliseconds. */
    getIterationDuration(): MeanStdPair;
    /** Returns mean/std duration of UNet inference in milliseconds. */
    getUnetInferDuration(): MeanStdPair;
    /** Returns mean/std duration of transformer inference in milliseconds. */
    getTransformerInferDuration(): MeanStdPair;
    /** Returns VAE encoder inference duration in milliseconds. */
    getVaeEncoderInferDuration(): number;
    /** Returns VAE decoder inference duration in milliseconds. */
    getVaeDecoderInferDuration(): number;
    /** Returns text encoder durations keyed by encoder name, milliseconds. */
    getTextEncoderInferDuration(): {
        [key: string]: number;
    };
    /** Returns total model inference duration in milliseconds. */
    getInferenceDuration(): number;
    /** Raw image-generation-specific metrics. */
    rawMetrics: RawImageGenerationPerfMetrics;
}
/**
 * @deprecated Use `ImageGenerationPerfMetrics` instead.
 */
export type Text2ImagePerfMetrics = ImageGenerationPerfMetrics;