gpt-tokenizer
Version:
A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models
11 lines • 757 B
JavaScript
/* eslint-disable import/extensions */
import bpeRanks from '../bpeRanks/r50k_base.js';
import { GptEncoding } from '../GptEncoding.js';
export * from '../constants.js';
export * from '../specialTokens.js';
const api = GptEncoding.getEncodingApi('r50k_base', () => bpeRanks);
const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, vocabularySize, setMergeCacheSize, clearMergeCache, estimateCost, } = api;
export { clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, estimateCost, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
// eslint-disable-next-line import/no-default-export
export default api;
//# sourceMappingURL=r50k_base.js.map