gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
60 lines (59 loc) • 1.81 kB
text/typescript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
import { BytePairDecoder } from '../BytePairDecoder.mjs';
import { BytePairEncoder } from '../BytePairEncoder.mjs';
/**
* Default Codex decoder.
* This is a singleton instance of {@linkcode BytePairDecoder} that is pre-configured to decode GPT-3 tokens.
*/
export declare const codexEncoder: BytePairEncoder;
/**
* Encodes a given UTF-8 string into a list of GPT-3 tokens.
*
* ```js
* const codeText = [
* 'function deeplyNested () {',
* ' return {',
* ' the: {',
* ' quick: {',
* ' brown: {',
* '...etc'
* ].join('')
*
* const codexTokens = encodeToCodexTokens(codeText)
* ```
*
* @see {@linkcode decodeCodexTokens} for the inverse function.
* @see {@linkcode BytePairEncoder} for more information on how the tokens are decoded.
*/
export declare const encodeCodex: import("../BytePairEncoder.mjs").TokenEncodeFn;
/**
* Default Codex decoder.
* This is a singleton instance of {@linkcode BytePairDecoder} that is pre-configured to decode GPT-3 tokens.
*/
export declare const codexDecoder: BytePairDecoder;
/**
* Converts a list of Codex tokens into a string.
*
* ```ts
* // Truncated for brevity...
* const tokens = [8818, 7744, 45, 7287, 7499]
* const text = codexDecoder.decode(tokens)
* console.log(text)
*
* // `function deeplyNested () {
* // return {
* // the: {
* // quick: {
* // brown: {
* // ...`
* ```
*
* @see {@linkcode codexEncoder} for the inverse object.
* @see {@linkcode BytePairDecoder} for more information on how the tokens are decoded.
*/
export declare const decodeCodex: import("../BytePairDecoder.mjs").TokenDecodeFn;