@llamaindex/env
Version:
environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker
17 lines (14 loc) • 406 B
TypeScript
declare enum Tokenizers {
CL100K_BASE = "cl100k_base"
}
interface Tokenizer {
encode: (text: string) => Uint32Array;
decode: (tokens: Uint32Array) => string;
}
declare class TokenizerSingleton {
private defaultTokenizer;
constructor();
tokenizer(encoding?: Tokenizers): Tokenizer;
}
declare const tokenizers: TokenizerSingleton;
export { type Tokenizer, Tokenizers, tokenizers };