ai-utils.js
Version:
Build AI applications, chatbots, and agents with JavaScript and TypeScript.
24 lines (23 loc) • 967 B
JavaScript
import { countTokens } from "../../../model-function/tokenize-text/countTokens.js";
import { TikTokenTokenizer } from "../TikTokenTokenizer.js";
/**
* Prompt tokens that are included automatically for every full
* chat prompt (several messages) that is sent to OpenAI.
*/
export const OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = 2;
/**
* Prompt tokens that are included automatically for every
* message that is sent to OpenAI.
*/
export const OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = 5;
export async function countOpenAIChatMessageTokens({ message, model, }) {
return (OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT +
(await countTokens(new TikTokenTokenizer({ model }), message.content ?? "")));
}
export async function countOpenAIChatPromptTokens({ messages, model, }) {
let tokens = OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT;
for (const message of messages) {
tokens += await countOpenAIChatMessageTokens({ message, model });
}
return tokens;
}