kitoken/dist/index.js

Fast tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization

/* @ts-self-types="./index.d.ts" */
import * as wasm from "./index_bg.wasm";
import { __wbg_set_wasm } from "./index_bg.js";

__wbg_set_wasm(wasm);

export {
    Kitoken
} from "./index_bg.js";