declare class Tokenizer {
version: string;
constructor();
_clean_text(text: string): string;
_run_split_on_punc(text: string): string[];
_run_strip_accents(text: string): string;
tokenize(text: string): string[];
lexical_diversity(text: string): number[];
}
export default Tokenizer;