node-llama-cpp
Version:
Run AI models locally on your machine with node.js bindings for llama.cpp. Enforce a JSON schema on the model output on the generation level
139 lines (138 loc) • 6.23 kB
TypeScript
import { ChatHistoryItem, ChatModelFunctions, ChatWrapperGenerateContextStateOptions, ChatWrapperGeneratedContextState, ChatWrapperSettings, Tokenizer } from "../../types.js";
import { LlamaText } from "../../utils/LlamaText.js";
import { ChatWrapper } from "../../ChatWrapper.js";
import { ChatHistoryFunctionCallMessageTemplate } from "./utils/chatHistoryFunctionCallMessageTemplate.js";
import { TemplateChatWrapperSegmentsOptions } from "./utils/templateSegmentOptionsToChatWrapperSettings.js";
export type JinjaTemplateChatWrapperOptions = {
template: string;
/**
* Defaults to `"assistant"`.
*/
modelRoleName?: string;
/**
* Defaults to `"user"`.
*/
userRoleName?: string;
/**
* Defaults to `"system"`.
*/
systemRoleName?: string;
/**
* Some Jinja templates may not support system messages, and in such cases,
* it'll be detected and system messages can be converted to user messages.
*
* You can specify the format of the converted user message.
* - **"auto"**: Convert system messages to user messages only if the template does not support system messages.
* - **`true`**: Always convert system messages to user messages.
* - **`false`**: Never convert system messages to user messages.
* May throw an error if some system messages don't appear in the template.
* - **`{use: "ifNeeded", format: "..."}`**: Convert system messages to user messages only if the template does not support system
* messages with the specified format.
* - **`{use: "always", format: "..."}`**: Always convert system messages to user messages with the specified format.
*
* Defaults to `"auto"`.
*/
convertUnsupportedSystemMessagesToUserMessages?: "auto" | boolean | JinjaTemplateChatWrapperOptionsConvertMessageFormat;
/**
* Template format for how functions can be called by the model and how their results are fed to the model after function calls.
*
* - **`"auto"`**: Extract the function call message template from the Jinja template.
* Fallback to the default template if not found.
* - **`"noJinja"`**: Use the default template.
* - **Custom template**: Use the specified {@link ChatHistoryFunctionCallMessageTemplate template}.
* See {@link ChatHistoryFunctionCallMessageTemplate `ChatHistoryFunctionCallMessageTemplate`} for more details.
*
* Defaults to `"auto"`.
*/
functionCallMessageTemplate?: "auto" | "noJinja" | ChatHistoryFunctionCallMessageTemplate;
/**
* Whether to join adjacent messages of the same type.
* Some Jinja templates may throw an error if this is not set to `true`.
*
* Defaults to `true`.
*/
joinAdjacentMessagesOfTheSameType?: boolean;
/**
* Whether to trim leading whitespace in responses.
*
* Defaults to `true`.
*/
trimLeadingWhitespaceInResponses?: boolean;
/**
* Additional parameters to use for rendering the Jinja template.
*/
additionalRenderParameters?: Record<string, any>;
/**
* Format of the segments generated by the model (like thought segments)
*/
segments?: TemplateChatWrapperSegmentsOptions;
/**
* Pass a model's tokenizer to attempt to detect common tokens used for chat formatting from it.
*
* Currently only used for detecting support for `<think>` tags for thought segments.
*/
tokenizer?: Tokenizer;
};
export type JinjaTemplateChatWrapperOptionsConvertMessageFormat = {
use?: "always" | "ifNeeded";
format: `${string}{{message}}${string}`;
};
/**
* A chat wrapper based on a Jinja template.
* Useful for using the original model's Jinja template as-is without any additional conversion work to chat with a model.
*
* If you want to create a new chat wrapper from scratch, using this chat wrapper is not recommended, and instead you better inherit
* from the `ChatWrapper` class and implement a custom chat wrapper of your own in TypeScript.
*
* For a simpler way to create a chat wrapper, see the `TemplateChatWrapper` class.
* @example
* <span v-pre>
*
* ```ts
* import {JinjaTemplateChatWrapper} from "node-llama-cpp";
*
* const chatWrapper = new JinjaTemplateChatWrapper({
* template: "<Jinja template here>",
* // functionCallMessageTemplate: { // optional
* // call: "[[call: {{functionName}}({{functionParams}})]]",
* // result: " [[result: {{functionCallResult}}]]"
* // },
* // segments: {
* // thoughtTemplate: "<think>{{content}}</think>",
* // reopenThoughtAfterFunctionCalls: true
* // }
* });
* ```
*
* </span>
*/
export declare class JinjaTemplateChatWrapper extends ChatWrapper {
readonly wrapperName = "JinjaTemplate";
readonly settings: ChatWrapperSettings;
readonly template: string;
readonly modelRoleName: string;
readonly userRoleName: string;
readonly systemRoleName: string;
readonly convertUnsupportedSystemMessagesToUserMessages?: JinjaTemplateChatWrapperOptionsConvertMessageFormat;
readonly joinAdjacentMessagesOfTheSameType: boolean;
readonly trimLeadingWhitespaceInResponses: boolean;
readonly additionalRenderParameters?: Record<string, any>;
/**
* @param options
*/
constructor(options: JinjaTemplateChatWrapperOptions);
/**
* Whether the function call syntax settings were extracted from the given Jinja template.
*
* The function call syntax settings can be accessed using the `.settings.functions` property.
*/
get usingJinjaFunctionCallTemplate(): boolean;
generateContextState({ chatHistory, availableFunctions, documentFunctionParams }: ChatWrapperGenerateContextStateOptions): ChatWrapperGeneratedContextState & {
transformedSystemMessagesToUserMessages: boolean;
};
addAvailableFunctionsSystemMessageToHistory(history: readonly ChatHistoryItem[], availableFunctions?: ChatModelFunctions, options?: {
documentParams?: boolean;
}): readonly ChatHistoryItem[];
generateFunctionCall(name: string, params: any): LlamaText;
generateFunctionCallResult(functionName: string, functionParams: any, result: any): LlamaText;
}