gpt-tokenizer/esm/constants.d.ts

A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models

export declare const ALL_SPECIAL_TOKENS = "all";
export declare const DEFAULT_MERGE_CACHE_SIZE = 100000;