gpt-token-utils/dist/BytePairEncoder.d.mts

Isomorphic utilities for GPT-3 tokenization and prompt building.

/**
 * @copyright Sister Software. All rights reserved.
 * @author Teffen Ellis, et al.
 * @license
 * See LICENSE file in the project root for full license information.
 */
import { BytePairEncoding } from './BytePairEncoding.mjs';
import { EncoderResult } from './EncoderResult.mjs';
/**
 * A valid input for the encoder.
 * @internal
 */
export type EncoderInput = string | EncoderResult;
export interface TokenEncodeFn {
    (
    /**
     * The string to encode.
     */
    text: string): EncoderResult;
    (
    /**
     * The string to encode.
     */
    text: string, 
    /**
     * Skip post-encoding processing for a slight performance boost.
     */
    skipPostProcessing?: boolean): EncoderResult;
    (
    /**
     * A previous encoder result to use as a starting point.
     * This will simply pass back the same result.
     * Useful when batch processing a mixed list of strings and encoder results.
     */
    encoderResult: EncoderResult): EncoderResult;
    (input: EncoderInput, skipPostProcessing?: boolean): EncoderResult;
}
/**
 * GPT Token Encoder.
 *
 * Generally, you should not need to use this class directly unless you are
 * implementing a custom token encoder.
 *
 * @see {@linkcode BytePairDecoder} for the decoder.
 *
 * ```ts
 * const encoder = new BytePairEncoder(bpeTokenMap, ranksMap)
 * const tokens = encoder.encode(encoder)
 * ```
 */
export declare class BytePairEncoder {
    protected _bpe: BytePairEncoding;
    protected _textEncoder: TextEncoder;
    protected _bpeTokenCache: Map<string, string[]>;
    constructor(_bpe: BytePairEncoding, _textEncoder?: TextEncoder, _bpeTokenCache?: Map<string, string[]>);
    /**
     * Encodes a given string into a list of tokens.
     *
     * ```ts
     * const text = "Do androids dream of electric sheep?"
     * const tokens = encoder.encode(text)
     * console.log(tokens) // [5211, 290, 305, 2340, 4320, 286, 5186, 15900, 30]
     * ```
     *
     * @returns The list of encoded tokens.
     */
    encode: TokenEncodeFn;
    /**
     * Merges the pair of characters with the given values in the given word.
     *
     * @param word - An array of individual characters in the word.
     * @param first - The first character in the pair to merge.
     * @param second - The second character in the pair to merge.
     *
     * @returns The word with the pair of characters merged.
     */
    mergePair(word: string[], first: string, second: string): string[];
    /**
     * Returns an array of all possible pairs of adjacent characters in the given word.
     *
     * @param word - An array of individual characters in the word.
     * @returns An array of all possible pairs of adjacent characters in the word.
     */
    getPairs(word: string[]): string[][];
    /**
     * Applies byte pair encoding (BPE) to the given token using the provided BPE ranks and cache.
     * If the token is already in the cache, returns its value from the cache.
     *
     * @param token - The token to encode using BPE. This is derived from text passed through the `tokenizerPattern` RegExp.
     *
     * @returns The BPE-encoded token.
     */
    protected _tokenToBPE(token: string): string[];
    /**
     * Finds the pair with the lowest rank (or highest numeric value if the rank is NaN) in the given array of pairs.
     *
     * @param pairs - An array of pairs of characters.
     * @param bpeRanks - An object containing the BPE ranks for all pairs of characters.
     * @returns The pair with the lowest rank, or null if no valid pair is found.
     */
    protected _findMinRankPair(pairs: string[][]): string[] | null;
}