gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
45 lines (44 loc) • 1.62 kB
text/typescript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
import { BytePairDecoder } from '../BytePairDecoder.mjs';
import { BytePairEncoder } from '../BytePairEncoder.mjs';
/**
* Default GPT-3 decoder.
* This is a singleton instance of {@linkcode BytePairEncoder} that is pre-configured to decode GPT-3 tokens.
*/
export declare const gptEncoder: BytePairEncoder;
/**
* Encodes a given UTF-8 string into a list of GPT-3 tokens.
*
* ```js
* const text = "Do androids dream of electric sheep?"
* const tokens = encoder.encode(text)
* console.log(tokens) // [5211, 290, 305, 2340, 4320, 286, 5186, 15900, 30]
* ```
*
* @see {@linkcode decode} for the inverse function.
* @see {@linkcode BytePairEncoder} for more information on how the tokens are decoded.
*/
export declare const encode: import("../BytePairEncoder.mjs").TokenEncodeFn;
/**
* Default GPT-3 decoder.
* This is a singleton instance of {@linkcode BytePairDecoder} that is pre-configured to decode GPT-3 tokens.
*/
export declare const gptDecoder: BytePairDecoder;
/**
* Converts a list of GPT-3 tokens into a string.
*
* ```ts
* const tokens = [5211, 290, 305, 2340, 4320, 286, 5186, 15900, 30]
* const text = decode(tokens)
* console.log(text) // "Do androids dream of electric sheep?"
* ```
*
* @see {@linkcode encode} for the inverse function.
* @see {@linkcode BytePairDecoder} for more information on how the tokens are decoded.
*/
export declare const decode: import("../BytePairDecoder.mjs").TokenDecodeFn;