gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
38 lines (37 loc) • 1.14 kB
text/typescript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
import { TokenEncodingsRecord } from '../BytePairTokenMap.mjs';
import { BPEVocab } from '../RanksMap.mjs';
/**
* Parses a BPE file into a list of bigrams
*
* The vocab.bpe file is a text file that contains a set of byte pair encoding (BPE) codes
* that are used in the tokenization process.
*
* The file should be in the following format:
*
* ```text
* #version: VERSION_STRING
* [prefix1] [suffix1]
* [prefixN] [suffixN]
* ...
* ```
*/
export declare function parseBPEFile(bpeFileContents: string): BPEVocab;
/**
* Parse a token encoder file, usually from a file named `encoder.json`
*/
export declare function parseEncoderFile(
/**
* The token encoder content, either as a string or as a parsed object.
*/
tokenEncoderContent: string | TokenEncodingsRecord): TokenEncodingsRecord;
export interface TikTokenEntry {
token: string;
rank: number;
}
export declare function parseTikTokenFile(tikTokenFileContents: string): TikTokenEntry[];