@lenml/tokenizers/src/prepatch.ts

a lightweight no-dependency fork of transformers.js (only tokenizers)

import { AutoTokenizer, PreTrainedTokenizer } from "./tokenizers/tokenizers";

// NOTE: Set this to turn off the warning of claude tokenizer
// @ts-ignore
AutoTokenizer.TOKENIZER_CLASS_MAPPING.ClaudeTokenizer = PreTrainedTokenizer;