gpt-token-utils/dist/BytePairTokenMap.d.mts

Isomorphic utilities for GPT-3 tokenization and prompt building.

/**
 * @copyright Sister Software. All rights reserved.
 * @author Teffen Ellis, et al.
 * @license
 * See LICENSE file in the project root for full license information.
 */
/**
 * A map of byte-pair encodings to their corresponding tokens.
 * @internal
 */
export type TokenEncodingsRecord = Record<string, number | undefined>;
declare const nodeInspectSymbol: unique symbol;
/**
 * Two-way map between Unicode byte-pairs and tokens.
 * @internal
 */
export declare class BytePairTokenMap {
    protected _bpeTokenMap: Map<
    /**
     * Byte paired character(s), e.g. `'!'`, `'\u00a8'`
     */
    string, 
    /**
     * The corresponding token, e.g. `0`, `101`
     */
    number>;
    protected _tokenBPEMap: Map<
    /**
     * The corresponding token, e.g. `0`, `101`
     */
    number, 
    /**
     * Byte paired character(s), e.g. `'!'`, `'\u00a8'`
     */
    string>;
    constructor(tokenEncodings: TokenEncodingsRecord, nMergedSpaces?: number);
    addBytePair(bytePair: string, token: number): void;
    tokenToBytePair(token: number): string;
    bytePairToToken(bytePair: string): number;
    get size(): number;
    [nodeInspectSymbol](): string;
}
export {};