gpt-tokenizer/esm/mapping.js

A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models

/* eslint-disable camelcase */
import { chatEnabledModels } from './modelsChatEnabled.gen.js';
import * as encodingsMap from './modelsMap.js';
import { ImSep } from './specialTokens.js';
export const cl100k_base = 'cl100k_base';
export const p50k_base = 'p50k_base';
export const p50k_edit = 'p50k_edit';
export const r50k_base = 'r50k_base';
export const o200k_base = 'o200k_base';
export const DEFAULT_ENCODING = o200k_base;
export const encodingNames = [
    p50k_base,
    r50k_base,
    p50k_edit,
    cl100k_base,
    o200k_base,
];
/**
 * maps model names to encoding names
 * if a model is not listed, it uses the default encoding for new models
 * which is `o200k_base`
 */
export const modelToEncodingMap = Object.fromEntries(Object.entries(encodingsMap).flatMap(([encodingName, models]) => models.map((modelName) => [modelName, encodingName])));
const gpt3params = {
    messageSeparator: '\n',
    roleSeparator: '\n',
};
const gpt4params = {
    messageSeparator: '',
    roleSeparator: ImSep,
};
export const chatModelParams = Object.fromEntries(chatEnabledModels.flatMap((modelName) => modelName.startsWith('gpt-3.5')
    ? [[modelName, gpt3params]]
    : [[modelName, gpt4params]]));
//# sourceMappingURL=mapping.js.map