tokenx/dist/index.d.mts

Fast token estimation with 95%+ average accuracy in a 2kB bundle

//#region src/types.d.ts
export interface TokenEstimationOptions {
  /** Average characters per token for segments no other rule prices (default: 7). */
  defaultCharsPerToken?: number;
  /** Language configs checked before the built-in ones, so the first matching custom config wins. */
  languageConfigs?: LanguageConfig[];
}
export interface LanguageConfig {
  /** Regular expression to detect the language. */
  pattern: RegExp;
  averageCharsPerToken: number;
}
export interface SplitByTokensOptions extends TokenEstimationOptions {
  /** Number of tokens to overlap between consecutive chunks (default: 0, clamped below the target chunk size). */
  overlap?: number;
}
//#endregion
//#region src/index.d.ts
export declare function isWithinTokenLimit(text: string, tokenLimit: number, options?: TokenEstimationOptions): boolean;
/** Estimates the number of tokens in a text string using heuristic rules. */
export declare function estimateTokenCount(text?: string, options?: TokenEstimationOptions): number;
/** Extracts a portion of text based on token positions, similar to `Array.prototype.slice()`. */
export declare function sliceByTokens(text: string, start?: number, end?: number, options?: TokenEstimationOptions): string;
export declare function splitByTokens(text: string, tokensPerChunk: number, options?: SplitByTokensOptions): string[];
//#endregion