gpt-tokenizer
Version:
A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models
5 lines (4 loc) • 310 B
TypeScript
export declare const isAscii: (codePoint: number) => boolean;
export declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
export declare function tryConvertToString(arr: Uint8Array): string | undefined;
export declare function compareUint8Arrays(a: Uint8Array, b: Uint8Array): number;