ai-utils.js/model-provider/openai/chat/countOpenAIChatMessageTokens.cjs

Build AI applications, chatbots, and agents with JavaScript and TypeScript.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.countOpenAIChatPromptTokens = exports.countOpenAIChatMessageTokens = exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = exports.OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = void 0;
const countTokens_js_1 = require("../../../model-function/tokenize-text/countTokens.cjs");
const TikTokenTokenizer_js_1 = require("../TikTokenTokenizer.cjs");
/**
 * Prompt tokens that are included automatically for every full
 * chat prompt (several messages) that is sent to OpenAI.
 */
exports.OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = 2;
/**
 * Prompt tokens that are included automatically for every
 * message that is sent to OpenAI.
 */
exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = 5;
async function countOpenAIChatMessageTokens({ message, model, }) {
    return (exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT +
        (await (0, countTokens_js_1.countTokens)(new TikTokenTokenizer_js_1.TikTokenTokenizer({ model }), message.content ?? "")));
}
exports.countOpenAIChatMessageTokens = countOpenAIChatMessageTokens;
async function countOpenAIChatPromptTokens({ messages, model, }) {
    let tokens = exports.OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT;
    for (const message of messages) {
        tokens += await countOpenAIChatMessageTokens({ message, model });
    }
    return tokens;
}
exports.countOpenAIChatPromptTokens = countOpenAIChatPromptTokens;