@lobehub/chat/src/utils/tokenizer/tokenizer.worker.ts

Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.

addEventListener('message', async (event) => {
  const { id, str } = event.data;
  try {
    const { encode } = await import('gpt-tokenizer');

    const tokenCount = encode(str).length;

    postMessage({ id, result: tokenCount });
  } catch (error) {
    postMessage({ error: (error as Error).message, id });
  }
});