gpt-token-utils/dist/EncoderResult.d.mts

Isomorphic utilities for GPT-3 tokenization and prompt building.

/**
 * @copyright Sister Software. All rights reserved.
 * @author Teffen Ellis, et al.
 * @license
 * See LICENSE file in the project root for full license information.
 */
declare const nodeInspectSymbol: unique symbol;
export interface IEncoderResult {
    /**
     * The tokens that were encoded.
     */
    readonly tokens: number[];
    /**
     * The BPE token pairs that were used during encoded.
     */
    readonly bpeTokenPairs: string[];
    /**
     * The original text content that was encoded.
     */
    readonly originalInput: string;
    /**
     * The matched text segments found during encoding.
     */
    readonly matchedTextSegments: string[];
}
/**
 * The `EncoderResult` includes information for post-encoding analysis such as...
 *
 * - The tokens that were encoded.
 * - The BPE token pairs that were used during encoded.
 * - Two-way maps of tokens to BPE token pairs.
 *
 * This information can be used to analyze the encoding process and to
 * reconstruct the original string from the encoded tokens.
 *
 * Note that this object is considered immutable. Consider encoding a new string
 * if you need an updated `EncoderResult`.
 *
 * @see {@linkcode BytePairEncoder}
 */
export declare class EncoderResult implements IEncoderResult {
    /**
     * A map of BPE token pairs to the corresponding token.
     */
    tokenBPEMap: ReadonlyMap<number, string>;
    /**
     * A map of tokens to the corresponding BPE token pair.
     */
    bpeTokenMap: ReadonlyMap<string, number>;
    /**
     * A map of BPE token pairs to the number of times they were used during encoding.
     * The key is the BPE token pair and the value is the number of times it appeared.
     */
    bpeCountsMap: ReadonlyMap<string, number>;
    /**
     * A map of tokens to the number of times they were used during encoding.
     * The key is the token and the value is the number of times it appeared.
     */
    tokenCountsMap: ReadonlyMap<number, number>;
    readonly tokens: number[];
    readonly bpeTokenPairs: string[];
    readonly originalInput: string;
    readonly matchedTextSegments: string[];
    segmenter: Intl.Segmenter | undefined;
    constructor({ tokens, bpeTokenPairs, originalInput, matchedTextSegments }: IEncoderResult, locale?: string);
    /**
     * Get the encoded byte-pair for a given token.
     */
    getBPE(token: number): string | undefined;
    /**
     * Get the number of times a given token appeared during encoding.
     * @see {@linkcode EncoderResult.length} if you're just trying count number of tokens.
     */
    getTokenCount(token: number): number;
    /**
     * Get the number of times a given byte-pair appeared during encoding.
     */
    getBPECount(bpe: string): number;
    /**
     * Iterate over the tokens in the result.
     */
    [Symbol.iterator](): IterableIterator<number>;
    /**
     * The number of tokens in the result.
     */
    get length(): number;
    /**
     * The number of characters in the original text.
     *
     * @see {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Segmenter Intl.Segmenter}
     */
    get characterCount(): number;
    [nodeInspectSymbol](): string;
    toString(): string;
    toJSON(): IEncoderResult;
}
export {};