gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
20 lines (19 loc) • 727 B
text/typescript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
import { TokenEncodingsRecord } from '../BytePairTokenMap.mjs';
import { BPEVocab } from '../RanksMap.mjs';
import { BundledEncoderKeys, BundledEncoderValues, BundledVocab } from './common.mjs';
/**
* Parses a bundled vocabulary into a list of bigrams.
* @internal
*/
export declare function parseBundledVocab(bundledVocab: BundledVocab): BPEVocab;
/**
* Parses a bundled encoder into a record.
* @internal
*/
export declare function parseBundledEncoder(encoderKeys: BundledEncoderKeys, encoderValues: BundledEncoderValues): TokenEncodingsRecord;