ai-utils.js/model-provider/llamacpp/LlamaCppTokenizer.js

Build AI applications, chatbots, and agents with JavaScript and TypeScript.

import z from "zod";
import { callWithRetryAndThrottle } from "../../util/api/callWithRetryAndThrottle.js";
import { createJsonResponseHandler, postJsonToApi, } from "../../util/api/postToApi.js";
import { failedLlamaCppCallResponseHandler } from "./LlamaCppError.js";
/**
 * Tokenizer for LlamaCpp.

 * @example
 * const tokenizer = new LlamaCppTokenizer();
 *
 * const text = "At first, Nox didn't know what to do with the pup.";
 *
 * const tokenCount = await countTokens(tokenizer, text);
 * const tokens = await tokenizer.tokenize(text);
 * const tokensAndTokenTexts = await tokenizer.tokenizeWithTexts(text);
 * const reconstructedText = await tokenizer.detokenize(tokens);
 */
export class LlamaCppTokenizer {
    constructor(settings = {}) {
        Object.defineProperty(this, "settings", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: void 0
        });
        this.settings = settings;
    }
    async callTokenizeAPI(text, context) {
        return callWithRetryAndThrottle({
            retry: this.settings.retry,
            throttle: this.settings.throttle,
            call: async () => callLlamaCppTokenizeAPI({
                abortSignal: context?.abortSignal,
                text,
                ...this.settings,
            }),
        });
    }
    async tokenize(text) {
        const response = await this.callTokenizeAPI(text);
        return response.tokens;
    }
}
const llamaCppTokenizationResponseSchema = z.object({
    tokens: z.array(z.number()),
});
async function callLlamaCppTokenizeAPI({ baseUrl = "http://127.0.0.1:8080", abortSignal, text, }) {
    return postJsonToApi({
        url: `${baseUrl}/tokenize`,
        body: {
            content: text,
        },
        failedResponseHandler: failedLlamaCppCallResponseHandler,
        successfulResponseHandler: createJsonResponseHandler(llamaCppTokenizationResponseSchema),
        abortSignal,
    });
}