@microsoft/teams-ai/lib/tokenizers/GPTTokenizer.js

SDK focused on building AI based applications for Microsoft Teams.

"use strict";
/**
 * @module teams-ai
 */
/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License.
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.GPTTokenizer = void 0;
const gpt_tokenizer_1 = require("gpt-tokenizer");
/**
 * Tokenizer that uses GPT's cl100k_base encoding.
 * To use GPT-3 encoding, pass in an instance of GPT3Tokenizer.
 */
class GPTTokenizer {
    decode(tokens) {
        return (0, gpt_tokenizer_1.decode)(tokens);
    }
    encode(text) {
        return (0, gpt_tokenizer_1.encode)(text);
    }
}
exports.GPTTokenizer = GPTTokenizer;
//# sourceMappingURL=GPTTokenizer.js.map