tokenx
Version:
Fast token estimation with 95%+ average accuracy in a 2kB bundle
820 lines (819 loc) • 33.2 kB
JavaScript
import { parseArgs, stripVTControlCharacters, styleText } from "node:util";
import process from "node:process";
import { text } from "node:stream/consumers";
import * as fs from "node:fs";
import * as fsp from "node:fs/promises";
import * as path from "node:path";
//#region node_modules/.pnpm/utilful@3.5.1_vitest@5.0.5_@types+node@26.6.5_vite@8.3.4_@types+node@26.6.5_jiti@2.7.0_yaml@2.9.1__/node_modules/utilful/dist/cli/style.mjs
function paint(style, text, stream) {
return styleText(style, text, { stream });
}
//#endregion
//#region node_modules/.pnpm/utilful@3.5.1_vitest@5.0.5_@types+node@26.6.5_vite@8.3.4_@types+node@26.6.5_jiti@2.7.0_yaml@2.9.1__/node_modules/utilful/dist/_virtual/_rolldown/runtime.mjs
var __defProp = Object.defineProperty;
var __exportAll = (all, no_symbols) => {
let target = {};
for (var name in all) __defProp(target, name, {
get: all[name],
enumerable: true
});
if (!no_symbols) __defProp(target, Symbol.toStringTag, { value: "Module" });
return target;
};
//#endregion
//#region node_modules/.pnpm/utilful@3.5.1_vitest@5.0.5_@types+node@26.6.5_vite@8.3.4_@types+node@26.6.5_jiti@2.7.0_yaml@2.9.1__/node_modules/utilful/dist/cli/log.mjs
var log_exports = /* @__PURE__ */ __exportAll({
blankLine: () => blankLine,
error: () => error,
info: () => info,
success: () => success,
warn: () => warn
});
function error(message) {
console.error(`${paint("red", "✖", process.stderr)} ${message}`);
}
function warn(message) {
console.error(`${paint("yellow", "⚠", process.stderr)} ${message}`);
}
function info(message) {
console.error(`${paint("cyan", "●", process.stderr)} ${message}`);
}
function success(message) {
console.error(`${paint("green", "✔", process.stderr)} ${message}`);
}
function blankLine() {
console.error("");
}
//#endregion
//#region node_modules/.pnpm/utilful@3.5.1_vitest@5.0.5_@types+node@26.6.5_vite@8.3.4_@types+node@26.6.5_jiti@2.7.0_yaml@2.9.1__/node_modules/utilful/dist/cli/errors.mjs
/**
* A condition the CLI recognized and phrased for a human. Anything else
* reaching the boundary is a defect in the tool and prints its stack unasked.
*/
var CliError = class extends Error {};
/** Gets usage printed alongside its message. */
var ArgumentError = class extends CliError {};
/** Reports a failure the way the boundary does, for one that outlives `run`, such as a watch rebuild. */
function reportFailure(error$1, { verbose = false, expectedErrors = [], describe } = {}) {
const message = error$1 instanceof Error ? describe?.(error$1) ?? error$1.message : String(error$1);
const sections = [message];
const causeChain = formatCauseChain(error$1, message);
if (causeChain) sections.push(causeChain);
if ((verbose || !isExpected(error$1, expectedErrors)) && error$1 instanceof Error && error$1.stack) sections.push(error$1.stack);
error(sections.join("\n\n"));
process.exitCode = 1;
}
/**
* Reports whether the CLI raised this error deliberately rather than tripping
* over it. A system error such as `ENOENT` reaches the boundary as the honest
* answer to what the user asked for, so it reads as deliberate too, while an
* `ERR_*` error from a Node API is a wrong call and stays a defect.
*/
function isExpected(error, expectedErrors) {
if (error instanceof CliError) return true;
if (expectedErrors.some((expectedError) => error instanceof expectedError)) return true;
return error instanceof Error && /^E[A-Z0-9]+$/.test(String(error.code));
}
function formatCauseChain(error, message) {
const causeLines = [];
const seen = /* @__PURE__ */ new Set([error]);
let printedText = message;
let current = error instanceof Error ? error.cause : void 0;
while (current instanceof Error && !seen.has(current)) {
seen.add(current);
const name = current.name || "Error";
if (!current.message) causeLines.push(`Caused by: ${name}`);
else if (!printedText.includes(current.message)) {
causeLines.push(`Caused by: ${name}: ${current.message}`);
printedText += `\n${current.message}`;
}
current = current.cause;
}
return causeLines.join("\n");
}
//#endregion
//#region node_modules/.pnpm/utilful@3.5.1_vitest@5.0.5_@types+node@26.6.5_vite@8.3.4_@types+node@26.6.5_jiti@2.7.0_yaml@2.9.1__/node_modules/utilful/dist/cli/args.mjs
const verboseArg = {
type: "boolean",
description: "Print the stack trace on failure"
};
/** Parses `argv` against a definition. An absent boolean reads as `false`, and `--no-<name>` turns one off. */
function parseArgs$1(argv, argsDef, { allowExtraPositionals = false } = {}) {
let parseResult;
try {
parseResult = parseArgs({
args: joinNegativeValues(splitShortOptionValues(argv), argsDef),
options: toNodeOptions(argsDef),
strict: true,
allowPositionals: true,
allowNegative: true
});
} catch (error) {
if (isNodeArgumentError(error)) throw new ArgumentError(error.message.split(". ")[0]);
throw error;
}
const args = { _: parseResult.positionals };
const positionalNames = [];
for (const [name, definition] of Object.entries(argsDef)) {
if (definition.type === "positional") {
positionalNames.push(name);
continue;
}
const value = parseResult.values[name];
if (definition.type === "enum") assertEnumValue(name, definition, value);
if (definition.type === "boolean") args[name] = value ?? false;
else if (value === void 0 && definition.required === true) throw new ArgumentError(`Missing required argument: --${name}`);
else args[name] = value;
}
positionalNames.forEach((name, index) => {
const definition = argsDef[name];
const value = parseResult.positionals[index];
if (value === void 0 && definition.required === true) throw new ArgumentError(`Missing required positional argument: ${name.toUpperCase()}`);
args[name] = value;
});
if (!allowExtraPositionals && parseResult.positionals.length > positionalNames.length) throw new ArgumentError(`Unexpected argument: ${JSON.stringify(parseResult.positionals[positionalNames.length])}`);
return args;
}
function toNodeOptions(argsDef) {
const options = {};
for (const [name, definition] of Object.entries(argsDef)) {
if (definition.type === "positional") continue;
const option = { type: definition.type === "boolean" ? "boolean" : "string" };
if (definition.alias !== void 0) option.short = definition.alias;
if (definition.default !== void 0) option.default = definition.default;
options[name] = option;
}
return options;
}
/**
* Splits an inline value off a short option. Node's `parseArgs` splits
* `--name=value` but leaves `-n=value` whole, so `-o=report.json` would write to
* a file named `=report.json`. Past `--` every token is an operand and stays as
* it was written.
*/
function splitShortOptionValues(argv) {
const splitArguments = [];
let isTerminated = false;
for (const argument of argv) {
const match = /^(-[^-])=(.*)$/.exec(argument);
if (isTerminated || match === null) {
splitArguments.push(argument);
isTerminated ||= argument === "--";
continue;
}
splitArguments.push(match[1], match[2]);
}
return splitArguments;
}
/**
* Joins a negative number onto the value-taking option before it, as
* `--start=-3`. Node refuses `--start -3` as ambiguous, but a token that starts
* with a digit after its dash is a number wherever a value is due.
*/
function joinNegativeValues(argv, argsDef) {
const optionNamesBySpelling = /* @__PURE__ */ new Map();
for (const [name, definition] of Object.entries(argsDef)) {
if (definition.type !== "string" && definition.type !== "enum") continue;
optionNamesBySpelling.set(`--${name}`, name);
if (definition.alias !== void 0) optionNamesBySpelling.set(`-${definition.alias}`, name);
}
const joinedArguments = [];
for (let index = 0; index < argv.length; index++) {
const argument = argv[index];
const next = argv[index + 1];
const name = optionNamesBySpelling.get(argument);
if (argument === "--") {
joinedArguments.push(...argv.slice(index));
break;
}
if (name !== void 0 && next !== void 0 && /^-\d/.test(next)) {
joinedArguments.push(`--${name}=${next}`);
index++;
continue;
}
joinedArguments.push(argument);
}
return joinedArguments;
}
function assertEnumValue(name, definition, value) {
const options = definition.options.join(", ");
if (definition.default !== void 0 && !definition.options.includes(definition.default)) throw new Error(`The default of --${name}, ${JSON.stringify(definition.default)}, is not one of: ${options}`);
if (value !== void 0 && !definition.options.includes(value)) throw new ArgumentError(`Invalid value for --${name}: ${JSON.stringify(value)}. Expected one of: ${options}`);
}
function isNodeArgumentError(error) {
return error instanceof Error && String(error.code).startsWith("ERR_PARSE_ARGS");
}
//#endregion
//#region node_modules/.pnpm/utilful@3.5.1_vitest@5.0.5_@types+node@26.6.5_vite@8.3.4_@types+node@26.6.5_jiti@2.7.0_yaml@2.9.1__/node_modules/utilful/dist/cli/usage.mjs
function renderUsage(command, { commandPath: commandName = command.meta?.name ?? "", version = command.meta?.version, stream = process.stdout } = {}) {
const color = (style, text) => paint(style, text, stream);
const heading = (title) => color("bold", title);
const meta = command.meta ?? {};
const positionalLines = [];
const optionLines = [];
const commandLines = [];
const usageLine = [];
for (const [name, definition] of Object.entries(command.args ?? {})) {
const isRequired = definition.type !== "boolean" && definition.required === true;
const hasDefault = definition.type !== "positional" && definition.default !== void 0 && definition.default !== false;
const hints = [
definition.description,
isRequired ? color("gray", "(Required)") : void 0,
hasDefault ? color("gray", `(Default: ${definition.default})`) : void 0
].filter((hint) => hint !== void 0).join(" ");
if (definition.type === "positional") {
const label = name.toUpperCase();
positionalLines.push([color("cyan", label), hints]);
usageLine.push(isRequired ? `<${label}>` : `[${label}]`);
continue;
}
const spellings = [definition.alias === void 0 ? void 0 : `-${definition.alias}`, `--${name}`].filter((spelling) => spelling !== void 0).join(", ");
const value = definition.type === "boolean" ? void 0 : `<${definition.valueHint ?? (definition.type === "enum" ? definition.options.join("|") : name)}>`;
optionLines.push([color("cyan", spellings) + (value === void 0 ? "" : color("dim", `=${value}`)), hints]);
if (definition.type === "boolean" && definition.default === true) optionLines.push([color("cyan", `--no-${name}`), ""]);
if (isRequired) usageLine.push(`--${name}=${value}`);
}
for (const [name, subCommand] of Object.entries(command.subCommands ?? {})) commandLines.push([color("cyan", name), subCommand.meta?.description ?? ""]);
const hasArguments = positionalLines.length > 0 || optionLines.length > 0;
if (commandLines.length > 0 && !hasArguments) usageLine.push(Object.keys(command.subCommands).join("|"));
const lines = [
color("bold", version === void 0 ? commandName : `${commandName} v${version}`),
...meta.description === void 0 ? [] : [meta.description],
"",
`${heading("USAGE")} ${color("cyan", [
commandName,
hasArguments ? "[OPTIONS]" : void 0,
...usageLine
].filter((part) => part !== void 0).join(" "))}`,
""
];
for (const [title, rows] of [
["ARGUMENTS", positionalLines],
["OPTIONS", optionLines],
["COMMANDS", commandLines]
]) if (rows.length > 0) lines.push(heading(title), "", formatColumns(rows), "");
if (commandLines.length > 0) lines.push(`Use ${color("cyan", `${commandName} <command> --help`)} for more information about a command.`);
return lines.join("\n").trimEnd();
}
function formatColumns(rows) {
const widths = rows[0].map((_, column) => Math.max(...rows.map((row) => visibleLength(row[column]))));
return rows.map((row) => row.map((cell, column) => ` ${cell}${" ".repeat(widths[column] - visibleLength(cell))}`).join("").trimEnd()).join("\n");
}
function visibleLength(text) {
return stripVTControlCharacters(text).length;
}
//#endregion
//#region node_modules/.pnpm/utilful@3.5.1_vitest@5.0.5_@types+node@26.6.5_vite@8.3.4_@types+node@26.6.5_jiti@2.7.0_yaml@2.9.1__/node_modules/utilful/dist/cli/command.mjs
const HELP_FLAGS = /* @__PURE__ */ new Set(["--help", "-h"]);
const VERSION_FLAGS = /* @__PURE__ */ new Set(["--version", "-v"]);
function defineCommand(command) {
return command;
}
/**
* Runs the command tree and reports whatever it throws. Help someone asked for
* is the result of the run and goes to stdout; usage that follows a wrong
* argument is diagnostics and joins its message on stderr.
*/
async function runMain(command, options = {}) {
const argv = options.argv ?? process.argv.slice(2);
const beforeTerminator = argv.slice(0, argv.includes("--") ? argv.indexOf("--") : void 0);
const hasFlag = (flags) => beforeTerminator.some((argument) => flags.has(argument));
try {
if (hasFlag(HELP_FLAGS)) process.stdout.write(`${usageFor(command, argv, process.stdout)}\n`);
else if (hasFlag(VERSION_FLAGS)) process.stdout.write(`${command.meta?.version ?? ""}\n`);
else await runCommand(command, argv);
} catch (error) {
if (error instanceof ArgumentError) process.stderr.write(`${usageFor(command, argv, process.stderr)}\n\n`);
const verbose = !(error instanceof ArgumentError) && beforeTerminator.includes("--verbose");
reportFailure(error, {
...options,
verbose
});
}
}
/** Runs like `runMain`, but without the error boundary. */
async function runCommand(command, argv) {
const { commands, firstOperand, rest } = resolveCommand(command, argv);
const leaf = withRunnerArgs(commands.at(-1));
if (isDispatchOnly(leaf)) throw new ArgumentError(firstOperand === void 0 ? "Missing command" : `Unknown command: ${firstOperand}`);
const args = parseArgs$1(rest, leaf.args ?? {}, { allowExtraPositionals: leaf.allowExtraPositionals });
await leaf.run?.({ args });
}
function usageFor(command, argv, stream) {
const { commands, keys } = resolveCommand(command, argv);
return renderUsage(withRunnerArgs(commands.at(-1)), {
commandPath: [command.meta?.name, ...keys].filter((name) => name !== void 0).join(" "),
version: commands.findLast((walked) => walked.meta?.version !== void 0)?.meta?.version,
stream
});
}
function resolveCommand(command, argv) {
const commands = [command];
const keys = [];
let rest = [...argv];
while (true) {
const found = findSubCommand(commands.at(-1), rest);
if (found.name === void 0) return {
commands,
keys,
firstOperand: found.firstOperand,
rest
};
commands.push(commands.at(-1).subCommands[found.name]);
keys.push(found.name);
rest = found.rest;
}
}
function isDispatchOnly(command) {
return command.subCommands !== void 0 && command.run === void 0;
}
/** `runMain` reads `--verbose` from argv itself, so strict parsing has to accept it on every command that parses arguments. */
function withRunnerArgs(command) {
if (isDispatchOnly(command)) return command;
return {
...command,
args: {
...command.args,
verbose: command.args?.verbose ?? verboseArg
}
};
}
/**
* Finds the sub-command the first operand names, wherever it stands among the
* options. Telling an operand from the value of an option needs every
* value-taking option of every sub-command, since the command is still unknown.
*/
function findSubCommand(command, argv) {
if (command.subCommands === void 0) return { rest: [...argv] };
const options = Object.assign(toNodeOptions(command.args ?? {}), ...Object.values(command.subCommands).map((subCommand) => toNodeOptions(subCommand.args ?? {})));
const { tokens } = parseArgs({
args: [...argv],
options,
strict: false,
allowPositionals: true,
tokens: true
});
const firstOperand = tokens.find((token) => token.kind === "positional" || token.kind === "option-terminator");
if (firstOperand?.kind !== "positional") return { rest: [...argv] };
if (Object.hasOwn(command.subCommands, firstOperand.value)) return {
name: firstOperand.value,
rest: argv.toSpliced(firstOperand.index, 1)
};
return {
firstOperand: firstOperand.value,
rest: [...argv]
};
}
//#endregion
//#region node_modules/.pnpm/utilful@3.5.1_vitest@5.0.5_@types+node@26.6.5_vite@8.3.4_@types+node@26.6.5_jiti@2.7.0_yaml@2.9.1__/node_modules/utilful/dist/cli/stdin.mjs
function readStdin() {
return text(process.stdin);
}
//#endregion
//#region package.json
var package_default = {
name: "tokenx",
type: "module",
version: "2.2.0",
packageManager: "pnpm@12.14.0",
description: "Fast token estimation with 95%+ average accuracy in a 2kB bundle",
author: "Johann Schopplich <hello@johannschopplich.com>",
license: "MIT",
homepage: "https://github.com/johannschopplich/tokenx#readme",
repository: {
"type": "git",
"url": "git+https://github.com/johannschopplich/tokenx.git"
},
bugs: { "url": "https://github.com/johannschopplich/tokenx/issues" },
keywords: [
"ai",
"cli",
"llm",
"token",
"tokenizer",
"estimation",
"tiktoken",
"anthropic",
"openai"
],
sideEffects: false,
exports: { ".": {
"types": "./dist/index.d.mts",
"default": "./dist/index.mjs"
} },
types: "./dist/index.d.mts",
bin: { "tokenx": "bin/tokenx.mjs" },
files: ["bin", "dist"],
scripts: {
"build": "tsdown",
"dev": "node ./src/cli/entry.ts --help",
"bench": "node scripts/generate-bench.ts && automd",
"bench:png": "node scripts/render-bench.ts",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
"test": "vitest",
"test:types": "tsc --noEmit",
"release": "bumpp"
},
devDependencies: {
"@antfu/eslint-config": "^9.5.1",
"@types/node": "^26.6.5",
"ansis": "^4.4.0",
"automd": "^0.4.3",
"bumpp": "^12.3.0",
"eslint": "^10.12.0",
"eslint-flat-config-utils": "^3.2.0",
"gpt-tokenizer": "^4.0.3",
"tsdown": "^0.23.1",
"typescript": "^6.0.3",
"utilful": "^3.5.1",
"vitest": "^5.0.5"
}
};
//#endregion
//#region src/segments.ts
const PATTERNS = {
whitespace: /^\s+$/,
structuredWhitespace: /\n\s/,
nonAscii: /[\u0080-\uFFFF]/,
cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u30FF\uFF00-\uFFEF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
numeric: /^\d+$/,
punctuation: /[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_"-]/,
cjkPunctuation: /[\u3001-\u3003\u3008-\u3011\u3014-\u301F\u3030\u30FB\uFF01-\uFF0F\uFF1A-\uFF20\uFF3B-\uFF40\uFF5B-\uFF65]/,
lowercaseWord: /^[a-z]+$/
};
const TOKEN_SPLIT_PATTERN = new RegExp(`(\\s+|${PATTERNS.punctuation.source}+|${PATTERNS.cjkPunctuation.source}+)`);
const DEFAULT_CHARS_PER_TOKEN = 7;
const SHORT_TOKEN_THRESHOLD = 3;
const LOWERCASE_WORD_SINGLE_TOKEN_LENGTH = 8;
const PUNCTUATION_CHARS_PER_TOKEN = 6;
const KANA_CHARS_PER_TOKEN = 1.35;
const HANGUL_CHARS_PER_TOKEN = 1.65;
const HANZI_CHARS_PER_TOKEN = 1.2;
const DEFAULT_LANGUAGE_CONFIGS = [
{
pattern: /[äöüßẞ]/i,
averageCharsPerToken: 3
},
{
pattern: /[éèêëàâîïôûùüÿçœæáíóúñ]/i,
averageCharsPerToken: 4.5
},
{
pattern: /[ąćęłńóśźżěščřžýůúďťň]/i,
averageCharsPerToken: 2.5
},
{
pattern: /[\u0430-\u044F\u0451]/i,
averageCharsPerToken: 6
},
{
pattern: /[\u03AC-\u03CE]/i,
averageCharsPerToken: 3
},
{
pattern: /^\p{Extended_Pictographic}[\p{Extended_Pictographic}\p{Emoji_Component}]*$/u,
averageCharsPerToken: .9
}
];
/**
* Walks a text as (segment, estimated token count) pairs. The segments
* concatenate back to the original text; whitespace segments count zero
* tokens unless they carry structure (indentation or blank lines).
*/
function* walkSegments(text, options = {}) {
if (!text) return;
const resolvedOptions = resolveOptions(options);
let previousSegment = "";
const parts = text.split(TOKEN_SPLIT_PATTERN);
for (let i = 0; i < parts.length; i++) {
let segment = parts[i];
if (PATTERNS.cjkPunctuation.test(parts[i + 1] ?? "") && PATTERNS.cjk.test(segment)) segment += parts[++i];
if (segment) {
yield {
segment,
tokenCount: estimateSegmentTokens(segment, resolvedOptions, previousSegment)
};
previousSegment = segment;
}
}
}
function resolveOptions(options) {
return {
defaultCharsPerToken: options.defaultCharsPerToken ?? DEFAULT_CHARS_PER_TOKEN,
customLanguageConfigs: options.languageConfigs ?? []
};
}
function estimateSegmentTokens(segment, { customLanguageConfigs, defaultCharsPerToken }, previousSegment) {
if (PATTERNS.whitespace.test(segment)) {
if (PATTERNS.structuredWhitespace.test(segment)) return 1;
if (!segment.includes("\n")) return 0;
const previousCharacter = previousSegment.slice(-1);
return PATTERNS.punctuation.test(previousCharacter) || PATTERNS.cjkPunctuation.test(previousCharacter) ? 0 : 1;
}
const customCharsPerToken = findLanguageCharsPerToken(segment, customLanguageConfigs);
if (customCharsPerToken !== void 0) return Math.ceil(getCharacterCount(segment) / customCharsPerToken);
if (PATTERNS.cjk.test(segment)) return estimateCjkTokens(segment);
const builtInCharsPerToken = PATTERNS.nonAscii.test(segment) ? findLanguageCharsPerToken(segment, DEFAULT_LANGUAGE_CONFIGS) : void 0;
if (builtInCharsPerToken !== void 0) return Math.ceil(getCharacterCount(segment) / builtInCharsPerToken);
if (PATTERNS.numeric.test(segment)) return Math.ceil(segment.length / 3);
if (segment.length <= SHORT_TOKEN_THRESHOLD) return 1;
if (segment.length <= LOWERCASE_WORD_SINGLE_TOKEN_LENGTH && PATTERNS.lowercaseWord.test(segment)) return 1;
if (PATTERNS.punctuation.test(segment)) return Math.ceil(segment.length / PUNCTUATION_CHARS_PER_TOKEN);
return Math.ceil(segment.length / defaultCharsPerToken);
}
function findLanguageCharsPerToken(segment, languageConfigs) {
for (const config of languageConfigs) if (segment.search(config.pattern) !== -1) return config.averageCharsPerToken;
}
function getCharacterCount(text) {
return Array.from(text).length;
}
function estimateCjkTokens(segment) {
let kanaCount = 0;
let hangulCount = 0;
let hanziCount = 0;
for (const character of segment) {
const codePoint = character.codePointAt(0);
if (codePoint >= 12352 && codePoint <= 12543) kanaCount++;
else if (isHangulCodePoint(codePoint)) hangulCount++;
else hanziCount++;
}
return Math.ceil(hanziCount / HANZI_CHARS_PER_TOKEN + kanaCount / KANA_CHARS_PER_TOKEN + hangulCount / HANGUL_CHARS_PER_TOKEN);
}
function isHangulCodePoint(codePoint) {
return codePoint >= 44032 && codePoint <= 55215 || codePoint >= 4352 && codePoint <= 4607 || codePoint >= 12592 && codePoint <= 12687 || codePoint >= 43360 && codePoint <= 43391 || codePoint >= 55216 && codePoint <= 55295;
}
//#endregion
//#region src/index.ts
/** Estimates the number of tokens in a text string using heuristic rules. */
function estimateTokenCount(text, options = {}) {
if (!text) return 0;
let tokenCount = 0;
for (const segmentEstimate of walkSegments(text, options)) tokenCount += segmentEstimate.tokenCount;
return tokenCount;
}
/** Extracts a portion of text based on token positions, similar to `Array.prototype.slice()`. */
function sliceByTokens(text, start = 0, end, options = {}) {
if (!text) return "";
let segmentEstimates = walkSegments(text, options);
let totalTokens = 0;
if (start < 0 || end !== void 0 && end < 0) {
const bufferedEstimates = Array.from(segmentEstimates);
for (const { tokenCount } of bufferedEstimates) totalTokens += tokenCount;
segmentEstimates = bufferedEstimates;
}
const normalizedStart = start < 0 ? Math.max(0, totalTokens + start) : Math.max(0, start);
const normalizedEnd = end === void 0 ? Infinity : end < 0 ? Math.max(0, totalTokens + end) : end;
if (normalizedStart >= normalizedEnd) return "";
const parts = [];
let currentTokenPos = 0;
for (const { segment, tokenCount } of segmentEstimates) {
if (currentTokenPos >= normalizedEnd) break;
const extracted = extractSegmentPart(segment, currentTokenPos, tokenCount, normalizedStart, normalizedEnd);
if (extracted) parts.push(extracted);
currentTokenPos += tokenCount;
}
return parts.join("");
}
function splitByTokens(text, tokensPerChunk, options = {}) {
if (!text || tokensPerChunk <= 0) return [];
const overlap = Math.min(options.overlap ?? 0, tokensPerChunk - 1);
const chunks = [];
let currentChunk = [];
let currentTokenCount = 0;
let hasUnchunkedSegments = false;
for (const segmentEstimate of walkSegments(text, options)) {
currentChunk.push(segmentEstimate);
currentTokenCount += segmentEstimate.tokenCount;
hasUnchunkedSegments = true;
if (currentTokenCount >= tokensPerChunk) {
chunks.push(joinSegments(currentChunk));
hasUnchunkedSegments = false;
if (overlap > 0) {
const overlapSegments = [];
let overlapTokenCount = 0;
for (let i = currentChunk.length - 1; i >= 0 && overlapTokenCount < overlap; i--) {
const overlapCandidate = currentChunk[i];
if (overlapTokenCount + overlapCandidate.tokenCount >= tokensPerChunk) break;
overlapSegments.unshift(overlapCandidate);
overlapTokenCount += overlapCandidate.tokenCount;
}
currentChunk = overlapSegments;
currentTokenCount = overlapTokenCount;
} else {
currentChunk = [];
currentTokenCount = 0;
}
}
}
if (currentChunk.length > 0 && hasUnchunkedSegments) chunks.push(joinSegments(currentChunk));
return chunks;
}
function joinSegments(segmentEstimates) {
let joined = "";
for (const { segment } of segmentEstimates) joined += segment;
return joined;
}
function extractSegmentPart(segment, segmentTokenStart, segmentTokenCount, targetStart, targetEnd) {
if (segmentTokenCount === 0) return segmentTokenStart >= targetStart && segmentTokenStart < targetEnd ? segment : "";
const segmentTokenEnd = segmentTokenStart + segmentTokenCount;
if (segmentTokenStart >= targetEnd || segmentTokenEnd <= targetStart) return "";
const overlapStart = Math.max(0, targetStart - segmentTokenStart);
const overlapEnd = Math.min(segmentTokenCount, targetEnd - segmentTokenStart);
if (overlapStart === 0 && overlapEnd === segmentTokenCount) return segment;
const characters = Array.from(segment);
const toCharacterIndex = (tokenOffset) => Math.round(tokenOffset / segmentTokenCount * characters.length);
return characters.slice(toCharacterIndex(overlapStart), toCharacterIndex(overlapEnd)).join("");
}
//#endregion
//#region src/cli/input.ts
const GLOB_PATTERN_RE = /[*?[{]/;
/** Reads each path, treating no paths – or a lone `-` – as stdin. */
async function readInputs(paths) {
if (paths.length === 0 || paths.length === 1 && paths[0] === "-") return [{
label: "stdin",
text: await readStdin()
}];
if (paths.includes("-")) throw new CliError("Cannot read stdin alongside file paths");
const filePaths = (await Promise.all(paths.map(expandPattern))).flat();
return Promise.all(filePaths.map(readFileInput));
}
/** Expands a pattern the shell passed through verbatim, as Windows shells do with `*.md`. */
async function expandPattern(inputPath) {
if (!GLOB_PATTERN_RE.test(inputPath) || fs.existsSync(inputPath)) return [inputPath];
const filePaths = (await Array.fromAsync(fsp.glob(inputPath, { withFileTypes: true }))).filter((entry) => !entry.isDirectory()).map((entry) => path.join(entry.parentPath, entry.name));
if (filePaths.length === 0) throw new CliError(`No files match \`${inputPath}\``);
return filePaths.sort();
}
async function readFileInput(inputPath) {
const resolvedPath = path.resolve(inputPath);
const label = path.relative(process.cwd(), resolvedPath) || path.basename(resolvedPath);
try {
return {
label,
text: await fsp.readFile(resolvedPath, "utf-8")
};
} catch (error) {
if (error.code === "EISDIR") throw new CliError(`\`${inputPath}\` is a directory – pass a pattern like "${path.join(inputPath, "*")}"`);
throw new CliError(`Cannot read \`${label}\`: ${Error.isError(error) ? error.message : String(error)}`);
}
}
//#endregion
//#region src/cli/index.ts
const { name, version } = package_default;
/** The boundary exits with `1` for any failure, which leaves `2` free to mean "ran fine, but the input is over the limit". */
const EXIT_OVER_LIMIT = 2;
/** The one default the CLI owns – every other default belongs to the library. */
const DEFAULT_CHUNK_SIZE = 1e3;
const INTEGER_PATTERN = /^-?\d+$/;
const DECIMAL_PATTERN = /^\d+(?:\.\d+)?$/;
const countArgs = {
"input": {
type: "positional",
description: "File path or glob pattern (omit or use \"-\" to read from stdin)",
required: false
},
"chars-per-token": {
type: "string",
description: "Average characters per token for text no other rule prices"
},
"limit": {
type: "string",
description: `Exit with code ${EXIT_OVER_LIMIT} when the total exceeds this many tokens`
},
"json": {
type: "boolean",
description: "Print a JSON object instead of plain numbers"
}
};
const sliceArgs = {
"input": countArgs.input,
"chars-per-token": countArgs["chars-per-token"],
"start": {
type: "string",
description: "Start token index, inclusive (negative counts from the end)"
},
"end": {
type: "string",
description: "End token index, exclusive (negative counts from the end)"
}
};
const splitArgs = {
"input": countArgs.input,
"chars-per-token": countArgs["chars-per-token"],
"size": {
type: "string",
description: `Target tokens per chunk (default: ${DEFAULT_CHUNK_SIZE})`
},
"overlap": {
type: "string",
description: "Tokens repeated from the end of the previous chunk"
}
};
const countCommand = defineCommand({
meta: {
name: "count",
description: "Estimate the token count of one or more inputs"
},
args: countArgs,
allowExtraPositionals: true,
async run({ args }) {
const options = resolveEstimationOptions(args["chars-per-token"]);
const limit = parseInteger("limit", args.limit, 0);
const counts = (await readInputs(args._)).map(({ label, text }) => ({
label,
tokenCount: estimateTokenCount(text, options)
}));
const total = counts.reduce((sum, { tokenCount }) => sum + tokenCount, 0);
if (args.json) process.stdout.write(`${JSON.stringify({
inputs: counts,
total
})}\n`);
else if (counts.length === 1) process.stdout.write(`${total}\n`);
else process.stdout.write(formatCountTable(counts, total));
if (limit !== void 0 && total > limit) {
log_exports.info(`${total} tokens exceeds the limit of ${limit}`);
process.exitCode = EXIT_OVER_LIMIT;
}
}
});
const sliceCommand = defineCommand({
meta: {
name: "slice",
description: "Extract a token range from an input, like Array.prototype.slice()"
},
args: sliceArgs,
allowExtraPositionals: true,
async run({ args }) {
const options = resolveEstimationOptions(args["chars-per-token"]);
const start = parseInteger("start", args.start);
const end = parseInteger("end", args.end);
const text = await readSingleInput(args._);
process.stdout.write(`${sliceByTokens(text, start, end, options)}\n`);
}
});
const splitCommand = defineCommand({
meta: {
name: "split",
description: "Split an input into token-sized chunks, printed as a JSON array"
},
args: splitArgs,
allowExtraPositionals: true,
async run({ args }) {
const options = resolveEstimationOptions(args["chars-per-token"]);
const size = parseInteger("size", args.size, 1) ?? DEFAULT_CHUNK_SIZE;
const overlap = parseInteger("overlap", args.overlap, 0);
const chunks = splitByTokens(await readSingleInput(args._), size, {
...options,
overlap
});
process.stdout.write(`${JSON.stringify(chunks)}\n`);
}
});
/** An input on its own is counted, so `count` doubles as the run of the tree itself. */
const mainCommand = defineCommand({
...countCommand,
meta: {
name,
description: "Estimate, slice and split text by LLM token count",
version
},
subCommands: {
count: countCommand,
slice: sliceCommand,
split: splitCommand
}
});
async function readSingleInput(paths) {
const documents = await readInputs(paths);
if (documents.length > 1) throw new CliError("Expected a single input");
return documents[0].text;
}
function resolveEstimationOptions(charsPerToken) {
const defaultCharsPerToken = parseNumber("chars-per-token", charsPerToken, DECIMAL_PATTERN, (value) => value > 0);
return defaultCharsPerToken === void 0 ? {} : { defaultCharsPerToken };
}
function parseInteger(option, rawValue, minimum) {
return parseNumber(option, rawValue, INTEGER_PATTERN, (value) => minimum === void 0 || value >= minimum);
}
function parseNumber(option, rawValue, pattern, isValid) {
if (rawValue === void 0) return void 0;
if (rawValue === "") throw new CliError(`Missing --${option} value`);
const value = pattern.test(rawValue) ? Number(rawValue) : NaN;
if (!Number.isFinite(value) || !isValid(value)) throw new CliError(`Invalid --${option} value: ${rawValue}`);
return value;
}
function formatCountTable(counts, total) {
const rows = [...counts, {
label: "total",
tokenCount: total
}];
const labelWidth = Math.max(...rows.map(({ label }) => label.length));
const countWidth = Math.max(...rows.map(({ tokenCount }) => String(tokenCount).length));
return `${rows.map(({ label, tokenCount }) => `${label.padEnd(labelWidth)} ${String(tokenCount).padStart(countWidth)}`).join("\n")}\n`;
}
//#endregion
//#region src/cli/entry.ts
runMain(mainCommand);
//#endregion
export {};