tokenx/dist/index.d.mts

Fast token estimation with 95%+ average accuracy, 2kB minified and gzipped

export interface TokenEstimationOptions {
  /** Average characters per token for segments no other rule prices (default: 7). */
  defaultCharsPerToken?: number;
  /** Language configs checked before the built-in ones, so the first matching custom config wins. */
  languageConfigs?: LanguageConfig[];
}
export interface LanguageConfig {
  /** Regular expression to detect the language. */
  pattern: RegExp;
  averageCharsPerToken: number;
}
export interface SplitByTokensOptions extends TokenEstimationOptions {
  /**
   * Approximate number of tokens repeated from the end of each chunk at the start of the next (default: 0, clamped
   * below the target chunk size). The repeat consists of whole segments – more tokens when a segment crosses this
   * count, fewer when one more segment would bring the repeat up to the target chunk size.
   */
  overlap?: number;
}
export declare function isWithinTokenLimit(text: string, tokenLimit: number, options?: TokenEstimationOptions): boolean;
/** Estimates the number of tokens in a text string using heuristic rules. */
export declare function estimateTokenCount(text?: string, options?: TokenEstimationOptions): number;
/** Extracts a portion of text based on token positions, similar to `Array.prototype.slice()`. */
export declare function sliceByTokens(text: string, start?: number, end?: number, options?: TokenEstimationOptions): string;
export declare function splitByTokens(text: string, tokensPerChunk: number, options?: SplitByTokensOptions): string[];