openvino-genai-node
Version:
OpenVINO™ GenAI pipelines for using from Node.js environment
231 lines • 9.85 kB
JavaScript
// Copyright (C) 2025-2026 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
import util from "node:util";
import { LLMPipeline as LLMPipelineWrapper } from "../addon.js";
import { StreamingStatus, } from "../utils.js";
import { DecodedResults } from "../decodedResults.js";
/**
* This class is used for generation with Large Language Models (LLMs)
*/
export class LLMPipeline {
/**
* Construct an LLM pipeline from a folder containing tokenizer and model IRs.
* @param modelPath - A folder to read tokenizer and model IRs.
* @param device - Inference device. A tokenizer is always compiled for CPU.
* @param properties - Device and pipeline properties.
*/
constructor(modelPath, device, properties) {
this.pipeline = null;
this.modelPath = modelPath;
this.device = device;
this.properties = properties;
}
/**
* Initialize the underlying native pipeline.
* @returns Resolves when initialization is complete.
*/
async init() {
if (this.pipeline)
throw new Error("LLMPipeline is already initialized");
const pipeline = new LLMPipelineWrapper();
const initPromise = util.promisify(pipeline.init.bind(pipeline));
const result = await initPromise(this.modelPath, this.device, this.properties);
this.pipeline = pipeline;
return result;
}
/**
* Start a chat session with an optional system message.
* @param systemMessage - Optional system message to initialize chat context.
* @returns Resolves when chat session is started.
* @deprecated startChat() / finishChat() API is deprecated and will be removed in the next major release.
* Please, use generate() with ChatHistory argument.
*/
async startChat(systemMessage = "") {
console.warn("DEPRECATION WARNING: startChat() / finishChat() API is deprecated and will be removed in the next major release.", "Please, use generate() with ChatHistory argument.");
if (!this.pipeline)
throw new Error("LLMPipeline is not initialized");
const startChatPromise = util.promisify(this.pipeline.startChat.bind(this.pipeline));
const result = await startChatPromise(systemMessage);
return result;
}
/**
* Finish the current chat session and clear chat-related state.
* @returns Resolves when chat session is finished.
* @deprecated startChat() / finishChat() API is deprecated and will be removed in the next major release.
* Please, use generate() with ChatHistory argument.
*/
async finishChat() {
console.warn("DEPRECATION WARNING: startChat() / finishChat() API is deprecated and will be removed in the next major release.", "Please, use generate() with ChatHistory argument.");
if (!this.pipeline)
throw new Error("LLMPipeline is not initialized");
const finishChatPromise = util.promisify(this.pipeline.finishChat.bind(this.pipeline));
const result = await finishChatPromise();
return result;
}
/**
* Get the current generation config (model defaults).
* @returns The current GenerationConfig object.
*/
getGenerationConfig() {
if (!this.pipeline)
throw new Error("LLMPipeline is not initialized");
return this.pipeline.getGenerationConfig();
}
/**
* Set generation configuration parameters.
* @param config - Generation configuration parameters.
*/
setGenerationConfig(config) {
if (!this.pipeline)
throw new Error("LLMPipeline is not initialized");
this.pipeline.setGenerationConfig(config);
}
/**
* Stream generation results as an async iterator of strings.
* The iterator yields subword chunks during generation.
* When generation finishes, the full decoded text is returned as the final
* iterator value (`done: true`). This value is not available through
* `for await...of`; call `next()` directly to read it.
*
* For batch processing or custom streaming control, see {@link generate}.
*
* @param inputs - Input prompt string or chat history.
* @param generationConfig - Generation configuration parameters.
* @returns Async iterator producing subword chunks.
*
* @example
* // Stream subword chunks to console
* for await (const chunk of pipe.stream(prompt, { max_new_tokens: 100 })) {
* process.stdout.write(chunk);
* }
*
* @throws {Error} If inputs is an array - use {@link generate} for batch processing
*/
stream(inputs, generationConfig = {}) {
if (!this.pipeline)
throw new Error("LLMPipeline is not initialized");
if (Array.isArray(inputs))
throw new Error("Streaming is not supported for array of inputs. Please use LLMPipeline.generate() method.");
if (typeof generationConfig !== "object")
throw new Error("Options must be an object");
let streamingStatus = StreamingStatus.RUNNING;
const queue = [];
let resolvePromise;
let rejectPromise;
const callback = (error, result) => {
if (error) {
if (rejectPromise) {
rejectPromise(error);
// Reset promises
resolvePromise = null;
rejectPromise = null;
}
else {
throw error;
}
}
else {
const decodedResult = new DecodedResults(result.texts, result.scores, result.perfMetrics, result.parsed, result.finishReasons);
const fullText = decodedResult.toString();
if (resolvePromise) {
// Fulfill pending request
resolvePromise({ done: true, value: fullText });
// Reset promises
resolvePromise = null;
rejectPromise = null;
}
else {
// Add data to queue if no pending promise
queue.push({ done: true, subword: fullText });
}
}
};
const streamer = (chunk) => {
if (resolvePromise) {
// Fulfill pending request
resolvePromise({ done: false, value: chunk });
// Reset promises
resolvePromise = null;
rejectPromise = null;
}
else {
// Add data to queue if no pending promise
queue.push({ done: false, subword: chunk });
}
return streamingStatus;
};
this.pipeline.generate(inputs, generationConfig, streamer, callback);
return {
async next() {
// If there is data in the queue, return it
// Otherwise, return a promise that will resolve when data is available
const data = queue.shift();
if (data !== undefined) {
return { value: data.subword, done: data.done };
}
return new Promise((resolve, reject) => {
resolvePromise = resolve;
rejectPromise = reject;
});
},
async return() {
streamingStatus = StreamingStatus.CANCEL;
return { done: true, value: "" };
},
[Symbol.asyncIterator]() {
return this;
},
};
}
/**
* Generate text sequences with optional streaming.
*
* This method supports:
* - Single prompt generation
* - Batch generation (array of prompts)
* - Chat history-based generation
* - Optional custom streaming via callback
*
* For simple streaming use cases, consider using {@link stream}, which provides
* a convenient async iterator interface.
*
* @param inputs - Input prompt string, array of prompts, or chat history.
* @param generationConfig - Generation configuration parameters.
* @param streamer - Optional callback invoked for each generated text chunk.
* - Return a `StreamingStatus` flag to indicate whether generation should be stopped or cancelled
* @returns Resolves with decoded results once generation finishes.
*
* @example
* // Simple generation without streaming
* const result = await pipe.generate("Hello", { max_new_tokens: 50 });
* console.log(result.texts[0]);
*
* @example
* // With custom streamer
* const result = await pipe.generate(prompt, config, (chunk) => {
* process.stdout.write(chunk);
* return StreamingStatus.RUNNING;
* });
*/
async generate(inputs, generationConfig = {}, streamer) {
if (!this.pipeline)
throw new Error("LLMPipeline is not initialized");
if (typeof generationConfig !== "object")
throw new Error("Options must be an object");
if (streamer !== undefined && typeof streamer !== "function")
throw new Error("Streamer must be a function");
const innerGenerate = util.promisify(this.pipeline.generate.bind(this.pipeline));
const result = await innerGenerate(inputs, generationConfig, streamer);
return new DecodedResults(result.texts, result.scores, result.perfMetrics, result.parsed, result.finishReasons);
}
/**
* Get the pipeline tokenizer instance.
* @returns Tokenizer used by the pipeline.
*/
getTokenizer() {
if (!this.pipeline)
throw new Error("LLMPipeline is not initialized");
return this.pipeline.getTokenizer();
}
}
//# sourceMappingURL=llmPipeline.js.map