@lenml/tokenizer-yi/src/main.ts

yi tokenizer for NodeJS/Browser

import { tokenizerJSON, tokenizerConfig } from "./data";
import {
  TokenizerLoader,
  TokenizerClassNameMapping,
  FromPreTrainedFn,
  tokenizers,
} from "@lenml/tokenizers";

/**
 * Build a tokenizer from a pre-trained model.
 */
export const fromPreTrained: FromPreTrainedFn<
  TokenizerClassNameMapping<"LlamaTokenizer">
> = (params) => {
  return TokenizerLoader.fromPreTrained({
    tokenizerJSON: {
      ...tokenizerJSON,
      ...params?.tokenizerJSON,
    },
    tokenizerConfig: {
      ...tokenizerConfig,
      ...params?.tokenizerConfig,
    },
  }) as any;
};

export { tokenizerJSON, tokenizerConfig, tokenizers, TokenizerLoader };