openvino-genai-node/types/pipelines/llmPipeline.d.ts

OpenVINO™ GenAI pipelines for using from Node.js environment

import { ChatHistory, LLMPipeline as LLMPipelineWrapper } from "../addon.js";
import { GenerationConfig, StreamingStatus, LLMPipelineProperties } from "../utils.js";
import { DecodedResults } from "../decodedResults.js";
import { Tokenizer } from "../tokenizer.js";
/**
 * This class is used for generation with Large Language Models (LLMs)
 */
export declare class LLMPipeline {
    modelPath: string;
    device: string;
    pipeline: LLMPipelineWrapper | null;
    properties: LLMPipelineProperties;
    /**
     * Construct an LLM pipeline from a folder containing tokenizer and model IRs.
     * @param modelPath - A folder to read tokenizer and model IRs.
     * @param device - Inference device. A tokenizer is always compiled for CPU.
     * @param properties - Device and pipeline properties.
     */
    constructor(modelPath: string, device: string, properties: LLMPipelineProperties);
    /**
     * Initialize the underlying native pipeline.
     * @returns Resolves when initialization is complete.
     */
    init(): Promise<void>;
    /**
     * Start a chat session with an optional system message.
     * @param systemMessage - Optional system message to initialize chat context.
     * @returns Resolves when chat session is started.
     * @deprecated startChat() / finishChat() API is deprecated and will be removed in the next major release.
     * Please, use generate() with ChatHistory argument.
     */
    startChat(systemMessage?: string): Promise<void>;
    /**
     * Finish the current chat session and clear chat-related state.
     * @returns Resolves when chat session is finished.
     * @deprecated startChat() / finishChat() API is deprecated and will be removed in the next major release.
     * Please, use generate() with ChatHistory argument.
     */
    finishChat(): Promise<void>;
    /**
     * Get the current generation config (model defaults).
     * @returns The current GenerationConfig object.
     */
    getGenerationConfig(): GenerationConfig;
    /**
     * Set generation configuration parameters.
     * @param config - Generation configuration parameters.
     */
    setGenerationConfig(config: GenerationConfig): void;
    /**
     * Stream generation results as an async iterator of strings.
     * The iterator yields subword chunks during generation.
     * When generation finishes, the full decoded text is returned as the final
     * iterator value (`done: true`). This value is not available through
     * `for await...of`; call `next()` directly to read it.
     *
     * For batch processing or custom streaming control, see {@link generate}.
     *
     * @param inputs - Input prompt string or chat history.
     * @param generationConfig - Generation configuration parameters.
     * @returns Async iterator producing subword chunks.
     *
     * @example
     * // Stream subword chunks to console
     * for await (const chunk of pipe.stream(prompt, { max_new_tokens: 100 })) {
     *   process.stdout.write(chunk);
     * }
     *
     * @throws {Error} If inputs is an array - use {@link generate} for batch processing
     */
    stream(inputs: string | ChatHistory, generationConfig?: GenerationConfig): {
        next(): Promise<{
            value: string;
            done: boolean;
        }>;
        return(): Promise<{
            done: boolean;
            value: string;
        }>;
        [Symbol.asyncIterator](): /*elided*/ any;
    };
    /**
     * Generate text sequences with optional streaming.
     *
     * This method supports:
     * - Single prompt generation
     * - Batch generation (array of prompts)
     * - Chat history-based generation
     * - Optional custom streaming via callback
     *
     * For simple streaming use cases, consider using {@link stream}, which provides
     * a convenient async iterator interface.
     *
     * @param inputs - Input prompt string, array of prompts, or chat history.
     * @param generationConfig - Generation configuration parameters.
     * @param streamer - Optional callback invoked for each generated text chunk.
     * - Return a `StreamingStatus` flag to indicate whether generation should be stopped or cancelled
     * @returns Resolves with decoded results once generation finishes.
     *
     * @example
     * // Simple generation without streaming
     * const result = await pipe.generate("Hello", { max_new_tokens: 50 });
     * console.log(result.texts[0]);
     *
     * @example
     * // With custom streamer
     * const result = await pipe.generate(prompt, config, (chunk) => {
     *   process.stdout.write(chunk);
     *   return StreamingStatus.RUNNING;
     * });
     */
    generate(inputs: string | string[] | ChatHistory, generationConfig?: GenerationConfig, streamer?: (chunk: string) => StreamingStatus): Promise<DecodedResults>;
    /**
     * Get the pipeline tokenizer instance.
     * @returns Tokenizer used by the pipeline.
     */
    getTokenizer(): Tokenizer;
}