kitoken/dist/minimal.js

Fast and versatile tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization

import * as wasm from "./minimal_bg.wasm";
export * from "./minimal_bg.js";
import { __wbg_set_wasm } from "./minimal_bg.js";
__wbg_set_wasm(wasm);