gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
24 lines (23 loc) • 1 kB
text/typescript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
/**
* Default contractions used by the tokenizer pattern.
* Note that order matters here, as the pattern will match the first contraction that matches.
*/
export declare const DEFAULT_CONTRACTIONS: readonly ["'s", "'t", "'re", "'ve", "'m", "'ll", "'d"];
/**
* Default tokenizer rules used to build the tokenizer pattern.
*/
export declare const DEFAULT_TOKENIZER_RULES: readonly [" ?\\p{L}+", " ?\\p{N}+", " ?[^\\s\\p{L}\\p{N}]+", "\\s+(?!\\S)", "\\s+"];
/**
* Creates a regular expression pattern used to tokenize text into individual tokens.
* @param contractions - Contractions used by the tokenizer pattern.
* @param rules - Rules used to build the tokenizer pattern.
*
* @see {@linkcode DEFAULT_TOKENIZER_RULES}
*/
export declare function createTokenizerPattern(contractions?: string[], rules?: string[]): RegExp;