openvino-genai-node/types/pipelines/vlmPipeline.d.ts

OpenVINO™ GenAI pipelines for using from Node.js environment

import { VLMPipeline as VLMPipelineWrapper, type ChatHistory } from "../addon.js";
import { GenerationConfig, VLMPipelineProperties, StreamingStatus } from "../utils.js";
import { VLMDecodedResults } from "../decodedResults.js";
import { Tokenizer } from "../tokenizer.js";
import type { Tensor } from "openvino-node";
/**
 * Options for VLM generation methods.
 */
export type VLMGenerateOptions = {
    /** Array of image tensors to include in the prompt. */
    images?: Tensor[];
    /** Array of video frame tensors to include in the prompt. */
    videos?: Tensor[];
    /** Generation configuration parameters such as max_length, temperature, etc. */
    generationConfig?: GenerationConfig;
};
/**
 * This class is used for generation with Visual Language Models (VLMs)
 */
export declare class VLMPipeline {
    protected readonly modelPath: string;
    protected readonly device: string;
    protected pipeline: VLMPipelineWrapper | null;
    protected readonly properties: VLMPipelineProperties;
    /**
     * Construct a VLM pipeline from a folder containing tokenizer and model IRs.
     * @param modelPath - A folder to read tokenizer and model IRs.
     * @param device - Inference device. A tokenizer is always compiled for CPU.
     * @param properties - Device and pipeline properties.
     */
    constructor(modelPath: string, device: string, properties: VLMPipelineProperties);
    /**
     * Initialize the underlying native pipeline.
     * @returns Resolves when initialization is complete.
     */
    init(): Promise<void>;
    /**
     * Start a chat session with an optional system message.
     * @param systemMessage - Optional system message to initialize chat context.
     * @returns Resolves when chat session is started.
     * @deprecated startChat() / finishChat() API is deprecated and will be removed in the next major release.
     * Please, use generate() with ChatHistory argument.
     */
    startChat(systemMessage?: string): Promise<void>;
    /**
     * Finish the current chat session and clear chat-related state.
     * @returns Resolves when chat session is finished.
     * @deprecated startChat() / finishChat() API is deprecated and will be removed in the next major release.
     * Please, use generate() with ChatHistory argument.
     */
    finishChat(): Promise<void>;
    /**
     * Stream generation results as an async iterator of strings.
     * The iterator yields subword chunks during generation.
     * When generation finishes, the full decoded text is returned as the final
     * iterator value (`done: true`). This value is not available through
     * `for await...of`; call `next()` directly to read it.
     *
     * For custom streaming control, use {@link generate} with a streamer callback instead.
     *
     * @param inputs - Input prompt string or chat history. May contain image/video tags recognized by the model.
     * @param options - Optional parameters.
     * @param options.images - Array of image tensors to include in the prompt.
     * @param options.videos - Array of video frame tensors to include in the prompt.
     * @param options.generationConfig - Generation parameters.
     * @returns Async iterator producing subword chunks.
     */
    stream(inputs: string | ChatHistory, options?: VLMGenerateOptions): AsyncIterableIterator<string>;
    /**
     * Generate sequences for VLMs with optional streaming.
     *
     * For simple streaming use cases, consider using {@link stream}, which provides
     * a convenient async iterator interface.
     *
     * @param inputs - Input prompt string or chat history. May contain model-specific image/video tags.
     * @param options - Optional parameters.
     * @param options.images - Array of image tensors to include in the prompt.
     * @param options.videos - Array of video frame tensors to include in the prompt.
     * @param options.generationConfig - Generation configuration parameters (e.g., max_new_tokens, temperature).
     * @param options.streamer - Optional callback invoked for each generated subword chunk.
     * - Return a `StreamingStatus` flag to indicate whether generation should be stopped or cancelled
     * @returns Promise resolving to {@link VLMDecodedResults} containing texts, scores, and performance metrics.
     */
    generate(inputs: string | ChatHistory, options?: VLMGenerateOptions & {
        streamer?: (chunk: string) => StreamingStatus;
    }): Promise<VLMDecodedResults>;
    /**
     * Get the pipeline tokenizer instance.
     * @returns Tokenizer used by the pipeline.
     */
    getTokenizer(): Tokenizer;
    /**
     * Set the chat template used when formatting chat history and prompts.
     * @param chatTemplate - Chat template string.
     */
    setChatTemplate(chatTemplate: string): void;
    /**
     * Set generation configuration parameters.
     * @param config - Generation configuration parameters.
     */
    setGenerationConfig(config: GenerationConfig): void;
    /**
     * Get the current generation config (model defaults).
     * @returns The current GenerationConfig object.
     */
    getGenerationConfig(): GenerationConfig;
}