gpt-tokenizer
Version:
A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models
17 lines • 577 B
JavaScript
import {} from '../modelParams.js';
import { EndOfText, FimMiddle, FimPrefix, FimSuffix } from '../specialTokens.js';
import { R50K_TOKEN_SPLIT_REGEX } from './constants.js';
export function P50KEdit(bytePairRankDecoder) {
const specialTokenMapping = new Map([
[EndOfText, 50_256],
[FimPrefix, 50_281],
[FimMiddle, 50_282],
[FimSuffix, 50_283],
]);
return {
tokenSplitRegex: R50K_TOKEN_SPLIT_REGEX,
bytePairRankDecoder,
specialTokensEncoder: specialTokenMapping,
};
}
//# sourceMappingURL=p50k_edit.js.map