UNPKG

nlp-tokenizer

Version:

tokenizer

github.com/josherich/nlp-tokenizer

josherich/nlp-tokenizer

6 lines (5 loc) • 300 B
1
2
3
4
5
6
declare function is_whitespace(char: string): boolean; declare function is_control(char: string): boolean; declare function is_punctuation(char: string): boolean; declare function whitespace_tokenize(text: string): string[]; export { is_whitespace, is_control, is_punctuation, whitespace_tokenize };