hugbot
Version:
Chatbot maker for HuggingFace Inference API and other AI API providers and backends.
20 lines • 686 B
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.mistralTokenizer = void 0;
// @ts-ignore
const mistral_tokenizer_ts_1 = require("mistral-tokenizer-ts");
const mistralTokenCounter = new mistral_tokenizer_ts_1.MistralTokenizer();
/**
* Tokenizer function is used to calculate the length of the chatbot memory buffer so that
* it can be truncated if it exceeds context window of the model.
*/
const mistralTokenizer = (text) => {
if (!text) {
return 0;
}
else {
return mistralTokenCounter.encode(text, true, false).length;
}
};
exports.mistralTokenizer = mistralTokenizer;
//# sourceMappingURL=MistralTokenizer.js.map