gpt-token-utils/dist/BytePairTokenMap.mjs

Isomorphic utilities for GPT-3 tokenization and prompt building.

/**
 * @copyright Sister Software. All rights reserved.
 * @author Teffen Ellis, et al.
 * @license
 * See LICENSE file in the project root for full license information.
 */
const nodeInspectSymbol = Symbol.for('nodejs.util.inspect.custom');
/**
 * Two-way map between Unicode byte-pairs and tokens.
 * @internal
 */
export class BytePairTokenMap {
    _bpeTokenMap;
    _tokenBPEMap;
    constructor(tokenEncodings, nMergedSpaces = 0) {
        this._bpeTokenMap = new Map();
        this._tokenBPEMap = new Map();
        for (const [key, value] of Object.entries(tokenEncodings)) {
            this.addBytePair(key, value);
        }
        // add merged spaces for codex tokenizer
        const normalizeVocabLength = this._bpeTokenMap.size + nMergedSpaces;
        for (let i = 0; i < nMergedSpaces; i++) {
            const key = '\u0120'.repeat(i + 2);
            const value = normalizeVocabLength - nMergedSpaces + i;
            this.addBytePair(key, value);
        }
    }
    addBytePair(bytePair, token) {
        this._bpeTokenMap.set(bytePair, token);
        this._tokenBPEMap.set(token, bytePair);
    }
    tokenToBytePair(token) {
        const bytePair = this._tokenBPEMap.get(token);
        if (typeof bytePair === 'undefined') {
            throw new Error(`Token "${token}" was not found in the token encoder.`);
        }
        return bytePair;
    }
    bytePairToToken(bytePair) {
        const token = this._bpeTokenMap.get(bytePair);
        if (typeof token === 'undefined') {
            throw new Error(`Byte pair "${bytePair}" was not found in the token encoder.`);
        }
        return token;
    }
    get size() {
        return this._bpeTokenMap.size;
    }
    [nodeInspectSymbol]() {
        return `BytePairTokenMap(${this.size})`;
    }
}