gpt-token-utils
Version:
Isomorphic utilities for GPT-3 tokenization and prompt building.
34 lines (33 loc) • 1.2 kB
JavaScript
/**
* @copyright Sister Software. All rights reserved.
* @author Teffen Ellis, et al.
* @license
* See LICENSE file in the project root for full license information.
*/
import { BytePairTokenMap } from './BytePairTokenMap.mjs';
import { CodePointByteMap } from './CodePointByteMap.mjs';
import { createTokenizerPattern } from './patterns.mjs';
import { RanksMap } from './RanksMap.mjs';
/**
* A base class for the Byte Pair Encoding (BPE) encoder and decoder.
* @internal
*/
export class BytePairEncoding {
codePointByteMap;
mergesSpacesCount;
tokenMap;
ranksMap;
tokenizationPattern;
constructor(options) {
this.tokenizationPattern = options.tokenizationPattern ?? createTokenizerPattern();
this.codePointByteMap = new CodePointByteMap();
if (typeof options.mergeSpaces === 'string') {
this.mergesSpacesCount = options.mergeSpaces === 'codex' ? 30 : 0;
}
else {
this.mergesSpacesCount = options.mergeSpaces ?? 0;
}
this.tokenMap = new BytePairTokenMap(options.tokenEncodings, this.mergesSpacesCount);
this.ranksMap = new RanksMap(options.vocab, this.mergesSpacesCount);
}
}