@lobehub/chat/src/utils/tokenizer/index.ts

Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.

export const encodeAsync = async (str: string): Promise<number> => {
  if (str.length === 0) return 0;

  // 30_000 is the limit of the client
  // if the string is longer than 30_000, we will use the server
  if (str.length <= 30_000) {
    const { clientEncodeAsync } = await import('./client');

    return await clientEncodeAsync(str);
  } else {
    const { serverEncodeAsync } = await import('./server');

    return await serverEncodeAsync(str);
  }
};