@lobehub/chat/src/utils/tokenizer/estimated.ts

Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.

import { approximateTokenSize } from 'tokenx';

export const estimatedEncodeAsync = async (str: string): Promise<number> =>
  approximateTokenSize(str);