@lenml/tokenizers/src/main.ts

a lightweight no-dependency fork of transformers.js (only tokenizers)

// polyfill for Array.toSorted
import "core-js/actual/array/to-sorted";

import "./prepatch";
export * from "./TokenizerLoader";
export * as tokenizers from "./tokenizers/tokenizers";
export * from "./types";