openvino-genai-node
Version:
OpenVINO™ GenAI pipelines for using from Node.js environment
211 lines • 9.37 kB
JavaScript
// Copyright (C) 2018-2026 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
import util from "node:util";
import { VLMPipeline as VLMPipelineWrapper } from "../addon.js";
import { StreamingStatus, } from "../utils.js";
import { VLMDecodedResults } from "../decodedResults.js";
/**
* This class is used for generation with Visual Language Models (VLMs)
*/
export class VLMPipeline {
/**
* Construct a VLM pipeline from a folder containing tokenizer and model IRs.
* @param modelPath - A folder to read tokenizer and model IRs.
* @param device - Inference device. A tokenizer is always compiled for CPU.
* @param properties - Device and pipeline properties.
*/
constructor(modelPath, device, properties) {
this.pipeline = null;
this.modelPath = modelPath;
this.device = device;
this.properties = properties;
}
/**
* Initialize the underlying native pipeline.
* @returns Resolves when initialization is complete.
*/
async init() {
const pipeline = new VLMPipelineWrapper();
const initPromise = util.promisify(pipeline.init.bind(pipeline));
await initPromise(this.modelPath, this.device, this.properties);
this.pipeline = pipeline;
}
/**
* Start a chat session with an optional system message.
* @param systemMessage - Optional system message to initialize chat context.
* @returns Resolves when chat session is started.
* @deprecated startChat() / finishChat() API is deprecated and will be removed in the next major release.
* Please, use generate() with ChatHistory argument.
*/
async startChat(systemMessage = "") {
console.warn("DEPRECATION WARNING: startChat() / finishChat() API is deprecated and will be removed in the next major release.", "Please, use generate() with ChatHistory argument.");
if (!this.pipeline)
throw new Error("Pipeline is not initialized");
const startChatPromise = util.promisify(this.pipeline.startChat.bind(this.pipeline));
const result = await startChatPromise(systemMessage);
return result;
}
/**
* Finish the current chat session and clear chat-related state.
* @returns Resolves when chat session is finished.
* @deprecated startChat() / finishChat() API is deprecated and will be removed in the next major release.
* Please, use generate() with ChatHistory argument.
*/
async finishChat() {
console.warn("DEPRECATION WARNING: startChat() / finishChat() API is deprecated and will be removed in the next major release.", "Please, use generate() with ChatHistory argument.");
if (!this.pipeline)
throw new Error("Pipeline is not initialized");
const finishChatPromise = util.promisify(this.pipeline.finishChat.bind(this.pipeline));
const result = await finishChatPromise();
return result;
}
/**
* Stream generation results as an async iterator of strings.
* The iterator yields subword chunks during generation.
* When generation finishes, the full decoded text is returned as the final
* iterator value (`done: true`). This value is not available through
* `for await...of`; call `next()` directly to read it.
*
* For custom streaming control, use {@link generate} with a streamer callback instead.
*
* @param inputs - Input prompt string or chat history. May contain image/video tags recognized by the model.
* @param options - Optional parameters.
* @param options.images - Array of image tensors to include in the prompt.
* @param options.videos - Array of video frame tensors to include in the prompt.
* @param options.generationConfig - Generation parameters.
* @returns Async iterator producing subword chunks.
*/
stream(inputs, options = {}) {
if (!this.pipeline)
throw new Error("Pipeline is not initialized");
const { images, videos, generationConfig } = options;
let streamingStatus = StreamingStatus.RUNNING;
const queue = [];
let resolvePromise;
let rejectPromise;
const callback = (error, result) => {
if (error) {
if (rejectPromise) {
rejectPromise(error);
// Reset promises
resolvePromise = null;
rejectPromise = null;
}
else {
throw error;
}
}
else {
const decodedResult = new VLMDecodedResults(result.texts, result.scores, result.perfMetrics, result.parsed, result.finishReasons);
const fullText = decodedResult.toString();
if (resolvePromise) {
// Fulfill pending request
resolvePromise({ done: true, value: fullText });
// Reset promises
resolvePromise = null;
rejectPromise = null;
}
else {
// Add data to queue if no pending promise
queue.push({ done: true, subword: fullText });
}
}
};
const streamer = (chunk) => {
if (resolvePromise) {
// Fulfill pending request
resolvePromise({ done: false, value: chunk });
// Reset promises
resolvePromise = null;
rejectPromise = null;
}
else {
// Add data to queue if no pending promise
queue.push({ done: false, subword: chunk });
}
return streamingStatus;
};
this.pipeline.generate(inputs, images, videos, streamer, generationConfig, callback);
return {
async next() {
// If there is data in the queue, return it
// Otherwise, return a promise that will resolve when data is available
const data = queue.shift();
if (data) {
return { value: data.subword, done: data.done };
}
return new Promise((resolve, reject) => {
resolvePromise = resolve;
rejectPromise = reject;
});
},
async return() {
streamingStatus = StreamingStatus.CANCEL;
return { done: true, value: "" };
},
[Symbol.asyncIterator]() {
return this;
},
};
}
/**
* Generate sequences for VLMs with optional streaming.
*
* For simple streaming use cases, consider using {@link stream}, which provides
* a convenient async iterator interface.
*
* @param inputs - Input prompt string or chat history. May contain model-specific image/video tags.
* @param options - Optional parameters.
* @param options.images - Array of image tensors to include in the prompt.
* @param options.videos - Array of video frame tensors to include in the prompt.
* @param options.generationConfig - Generation configuration parameters (e.g., max_new_tokens, temperature).
* @param options.streamer - Optional callback invoked for each generated subword chunk.
* - Return a `StreamingStatus` flag to indicate whether generation should be stopped or cancelled
* @returns Promise resolving to {@link VLMDecodedResults} containing texts, scores, and performance metrics.
*/
async generate(inputs, options = {}) {
const { images, videos, generationConfig, streamer } = options;
if (!this.pipeline)
throw new Error("Pipeline is not initialized");
const innerGenerate = util.promisify(this.pipeline.generate.bind(this.pipeline));
const result = await innerGenerate(inputs, images, videos, streamer, generationConfig);
return new VLMDecodedResults(result.texts, result.scores, result.perfMetrics, result.parsed, result.finishReasons);
}
/**
* Get the pipeline tokenizer instance.
* @returns Tokenizer used by the pipeline.
*/
getTokenizer() {
if (!this.pipeline)
throw new Error("Pipeline is not initialized");
return this.pipeline.getTokenizer();
}
/**
* Set the chat template used when formatting chat history and prompts.
* @param chatTemplate - Chat template string.
*/
setChatTemplate(chatTemplate) {
if (!this.pipeline)
throw new Error("Pipeline is not initialized");
this.pipeline.setChatTemplate(chatTemplate);
}
/**
* Set generation configuration parameters.
* @param config - Generation configuration parameters.
*/
setGenerationConfig(config) {
if (!this.pipeline)
throw new Error("Pipeline is not initialized");
this.pipeline.setGenerationConfig(config);
}
/**
* Get the current generation config (model defaults).
* @returns The current GenerationConfig object.
*/
getGenerationConfig() {
if (!this.pipeline)
throw new Error("Pipeline is not initialized");
return this.pipeline.getGenerationConfig();
}
}
//# sourceMappingURL=vlmPipeline.js.map