@lenml/tokenizer-gpt2/dist/main.d.ts

gpt2 tokenizer for NodeJS/Browser

import { NSTokenizerJSON, NSTokenizerConfig, FromPreTrainedFn, TokenizerClassNameMapping } from '@lenml/tokenizers';
export { TokenizerLoader, tokenizers } from '@lenml/tokenizers';

declare const tokenizerJSON: NSTokenizerJSON.Root;
declare const tokenizerConfig: NSTokenizerConfig.Root;

/**
 * Build a tokenizer from a pre-trained model.
 */
declare const fromPreTrained: FromPreTrainedFn<TokenizerClassNameMapping<"GPT2Tokenizer">>;

export { fromPreTrained, tokenizerConfig, tokenizerJSON };