UNPKG

node-llama-cpp

Version:

Run AI models locally on your machine with node.js bindings for llama.cpp. Enforce a JSON schema on the model output on the generation level

139 lines (138 loc) 6.23 kB
import { ChatHistoryItem, ChatModelFunctions, ChatWrapperGenerateContextStateOptions, ChatWrapperGeneratedContextState, ChatWrapperSettings, Tokenizer } from "../../types.js"; import { LlamaText } from "../../utils/LlamaText.js"; import { ChatWrapper } from "../../ChatWrapper.js"; import { ChatHistoryFunctionCallMessageTemplate } from "./utils/chatHistoryFunctionCallMessageTemplate.js"; import { TemplateChatWrapperSegmentsOptions } from "./utils/templateSegmentOptionsToChatWrapperSettings.js"; export type JinjaTemplateChatWrapperOptions = { template: string; /** * Defaults to `"assistant"`. */ modelRoleName?: string; /** * Defaults to `"user"`. */ userRoleName?: string; /** * Defaults to `"system"`. */ systemRoleName?: string; /** * Some Jinja templates may not support system messages, and in such cases, * it'll be detected and system messages can be converted to user messages. * * You can specify the format of the converted user message. * - **"auto"**: Convert system messages to user messages only if the template does not support system messages. * - **`true`**: Always convert system messages to user messages. * - **`false`**: Never convert system messages to user messages. * May throw an error if some system messages don't appear in the template. * - **`{use: "ifNeeded", format: "..."}`**: Convert system messages to user messages only if the template does not support system * messages with the specified format. * - **`{use: "always", format: "..."}`**: Always convert system messages to user messages with the specified format. * * Defaults to `"auto"`. */ convertUnsupportedSystemMessagesToUserMessages?: "auto" | boolean | JinjaTemplateChatWrapperOptionsConvertMessageFormat; /** * Template format for how functions can be called by the model and how their results are fed to the model after function calls. * * - **`"auto"`**: Extract the function call message template from the Jinja template. * Fallback to the default template if not found. * - **`"noJinja"`**: Use the default template. * - **Custom template**: Use the specified {@link ChatHistoryFunctionCallMessageTemplate template}. * See {@link ChatHistoryFunctionCallMessageTemplate `ChatHistoryFunctionCallMessageTemplate`} for more details. * * Defaults to `"auto"`. */ functionCallMessageTemplate?: "auto" | "noJinja" | ChatHistoryFunctionCallMessageTemplate; /** * Whether to join adjacent messages of the same type. * Some Jinja templates may throw an error if this is not set to `true`. * * Defaults to `true`. */ joinAdjacentMessagesOfTheSameType?: boolean; /** * Whether to trim leading whitespace in responses. * * Defaults to `true`. */ trimLeadingWhitespaceInResponses?: boolean; /** * Additional parameters to use for rendering the Jinja template. */ additionalRenderParameters?: Record<string, any>; /** * Format of the segments generated by the model (like thought segments) */ segments?: TemplateChatWrapperSegmentsOptions; /** * Pass a model's tokenizer to attempt to detect common tokens used for chat formatting from it. * * Currently only used for detecting support for `<think>` tags for thought segments. */ tokenizer?: Tokenizer; }; export type JinjaTemplateChatWrapperOptionsConvertMessageFormat = { use?: "always" | "ifNeeded"; format: `${string}{{message}}${string}`; }; /** * A chat wrapper based on a Jinja template. * Useful for using the original model's Jinja template as-is without any additional conversion work to chat with a model. * * If you want to create a new chat wrapper from scratch, using this chat wrapper is not recommended, and instead you better inherit * from the `ChatWrapper` class and implement a custom chat wrapper of your own in TypeScript. * * For a simpler way to create a chat wrapper, see the `TemplateChatWrapper` class. * @example * <span v-pre> * * ```ts * import {JinjaTemplateChatWrapper} from "node-llama-cpp"; * * const chatWrapper = new JinjaTemplateChatWrapper({ * template: "<Jinja template here>", * // functionCallMessageTemplate: { // optional * // call: "[[call: {{functionName}}({{functionParams}})]]", * // result: " [[result: {{functionCallResult}}]]" * // }, * // segments: { * // thoughtTemplate: "<think>{{content}}</think>", * // reopenThoughtAfterFunctionCalls: true * // } * }); * ``` * * </span> */ export declare class JinjaTemplateChatWrapper extends ChatWrapper { readonly wrapperName = "JinjaTemplate"; readonly settings: ChatWrapperSettings; readonly template: string; readonly modelRoleName: string; readonly userRoleName: string; readonly systemRoleName: string; readonly convertUnsupportedSystemMessagesToUserMessages?: JinjaTemplateChatWrapperOptionsConvertMessageFormat; readonly joinAdjacentMessagesOfTheSameType: boolean; readonly trimLeadingWhitespaceInResponses: boolean; readonly additionalRenderParameters?: Record<string, any>; /** * @param options */ constructor(options: JinjaTemplateChatWrapperOptions); /** * Whether the function call syntax settings were extracted from the given Jinja template. * * The function call syntax settings can be accessed using the `.settings.functions` property. */ get usingJinjaFunctionCallTemplate(): boolean; generateContextState({ chatHistory, availableFunctions, documentFunctionParams }: ChatWrapperGenerateContextStateOptions): ChatWrapperGeneratedContextState & { transformedSystemMessagesToUserMessages: boolean; }; addAvailableFunctionsSystemMessageToHistory(history: readonly ChatHistoryItem[], availableFunctions?: ChatModelFunctions, options?: { documentParams?: boolean; }): readonly ChatHistoryItem[]; generateFunctionCall(name: string, params: any): LlamaText; generateFunctionCallResult(functionName: string, functionParams: any, result: any): LlamaText; }