gpt-tokenizer
Version:
A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models
9 lines (8 loc) • 390 B
TypeScript
export declare const EndOfText = "<|endoftext|>";
export declare const FimPrefix = "<|fim_prefix|>";
export declare const FimMiddle = "<|fim_middle|>";
export declare const FimSuffix = "<|fim_suffix|>";
export declare const ImStart = "<|im_start|>";
export declare const ImEnd = "<|im_end|>";
export declare const ImSep = "<|im_sep|>";
export declare const EndOfPrompt = "<|endofprompt|>";