@lenml/tokenizer-gemini/dist/main.d.ts

gemini tokenizer for NodeJS/Browser

import { NSTokenizerJSON, NSTokenizerConfig, FromPreTrainedFn, TokenizerClassNameMapping } from '@lenml/tokenizers';
export { TokenizerLoader, tokenizers } from '@lenml/tokenizers';

declare const tokenizerJSON: NSTokenizerJSON.Root;
declare const tokenizerConfig: NSTokenizerConfig.Root;

/**
 * Build a tokenizer from a pre-trained model.
 */
declare const fromPreTrained: FromPreTrainedFn<TokenizerClassNameMapping<"GemmaTokenizer">>;

export { fromPreTrained, tokenizerConfig, tokenizerJSON };