@orama/tokenizers
Version:
Additional tokenizers for Orama
9 lines • 395 B
TypeScript
import type { DefaultTokenizer, DefaultTokenizerConfig } from "@orama/orama";
declare const tokenizerLanguage = "mandarin";
type TLanguage = typeof tokenizerLanguage;
type MandarinTokenizerConfig = DefaultTokenizerConfig & {
language: TLanguage;
};
export declare function createTokenizer(config?: MandarinTokenizerConfig): DefaultTokenizer;
export {};
//# sourceMappingURL=mandarin.d.ts.map