hugbot/dist/cjs/components/Tokenizers/MistralTokenizer.d.ts

Chatbot maker for HuggingFace Inference API and other AI API providers and backends.

/**
 * Tokenizer function is used to calculate the length of the chatbot memory buffer so that
 * it can be truncated if it exceeds context window of the model.
 */
export declare const mistralTokenizer: (text: string) => number;
//# sourceMappingURL=MistralTokenizer.d.ts.map