gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
18 lines (17 loc) • 572 B
text/typescript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
import type { IBytePairEncodingOptions } from '../BytePairEncoding.mjs';
/**
* @internal
*/
export declare function createDefaultBPEOptions(): Readonly<IBytePairEncodingOptions>;
/**
* Default options for byte pair encoding.
*
* Note that referencing to this object will incur a filesize penalty when bundling.
*/
export declare const DEFAULT_BPE_OPTIONS: Readonly<IBytePairEncodingOptions>;