gpt-token-utils/dist/BytePairDecoder.d.mts

Isomorphic utilities for GPT-3 tokenization and prompt building.

/**
 * @copyright Sister Software. All rights reserved.
 * @author Teffen Ellis, et al.
 * @license
 * See LICENSE file in the project root for full license information.
 */
import { BytePairEncoding } from './BytePairEncoding.mjs';
import type { EncoderResult } from './EncoderResult.mjs';
/**
 * Methods associated with decoding a list of tokens into a string.
 */
export interface TokenDecodeFn {
    (
    /**
     * The list of tokens to decode.
     */
    tokens: number[]): string;
    (
    /**
     * The resulting object of the {@linkcode BytePairEncoder.encode} function.
     */
    encoderResult: EncoderResult): string;
}
/**
 * GPT Token Decoder.
 *
 * Generally, you should not need to use this class directly unless you are
 * implementing a custom token decoder.
 *
 * @see {@linkcode BytePairEncoder} for the encoder.
 *
 * ```ts
 * const decoder = new BytePairDecoder({codePointByteMap, bpeTokenMap})
 * const text = decoder.decode(tokens)
 * ```
 */
export declare class BytePairDecoder {
    protected _bpe: BytePairEncoding;
    protected _textDecoder: TextDecoder;
    constructor(_bpe: BytePairEncoding, _textDecoder?: TextDecoder);
    /**
     * Converts a list of tokens into a string.
     *
     * ```ts
     * const tokens = [5211, 290, 305, 2340, 4320, 286, 5186, 15900, 30]
     * const text = decoder.decode(tokens)
     * console.log(text) // "Do androids dream of electric sheep?"
     * ```
     *
     * @returns The decoded string.
     */
    decode: TokenDecodeFn;
}