gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
45 lines (44 loc) • 1.15 kB
text/typescript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
declare const nodeInspectSymbol: unique symbol;
/**
* Map of byte-pair encodings according to their BPE rank
* @internal
*/
export declare class RanksMap {
protected _prefixToSuffixRankMap: Map<
/** Prefix */
string, Map<
/** Suffix */
string,
/** Rank */
number>>;
getRank(prefix: string, suffix: string): number | undefined;
constructor(vocab: VocabEntry[] | BPEVocab, mergesSpacesCount?: number);
get size(): number;
[nodeInspectSymbol](): string;
}
/**
* A parsed vocabulary entry.
* The rank of the byte-pair encoding is derived from the index of the pair in the `vocab.bpe` file.
*/
export interface VocabEntry {
/** The word stem prefix in the pair. */
prefix: string;
/** The suffix token in the pair. */
suffix: string;
}
/**
* A vocabulary of byte-pair encodings.
*
* @see {@linkcode parseBPEFile}
*/
export interface BPEVocab {
version: string;
entries: VocabEntry[];
}
export {};