kestrel.markets/src/render/tokens.ts

A typed, token-efficient language + runtime for agentic trading: agents author bounded plans, the runtime fires them at the tick. CLI + typed library + MCP server.

/**
 * # render/tokens — the per-model token-count ORACLE (kestrel-4gl.11)
 *
 * A Rendering has two parameters: a **format** and the **tokenizer its token costs are
 * measured under** (CONTEXT.md / {@link ./index.ts}) — "which glyphs are cheap is an
 * empirical, per-model fact." This module IS that tokenizer: it measures how many BPE
 * tokens a rendered string costs a given model. It is NOT `src/tooling/highlight.ts`
 * `tokenize` — that is a source HIGHLIGHTER (what colour is this lexeme); this answers a
 * different question (how many tokens does this string cost).
 *
 * Three backends, and every result is LABELLED with the {@link CountMethod} that produced it,
 * so a consumer always knows whether it holds a precise BPE count, a provider-authoritative
 * native count, or a documented estimate — never a silently stubbed, fake-precise number:
 *   - `tiktoken-cl100k` — precise BPE via the OPTIONAL, LAZY `js-tiktoken` backend (a frozen
 *     merge table; a pure function of the input bytes).
 *   - `anthropic-count-tokens` — the CLOSED Claude/Fable tokenizer (`fable-native`), measured via
 *     the Anthropic `POST /v1/messages/count_tokens` endpoint with a base+fragment DIFFERENCING
 *     protocol, served OFFLINE from a committed `sha256(bytes) → input_tokens` cache fixture. A
 *     cache miss FAILS CLOSED (see {@link NativeTokenCacheMissError}) — it NEVER hits the network
 *     at test/CI time, so it stays as deterministic as the open backends. Only an explicit
 *     maintainer refresh (networked, needs `ANTHROPIC_API_KEY`) writes the fixture.
 *   - `chars-approx`    — the dependency-free `ceil(len / CHARS_PER_TOKEN)` fallback, always
 *     available. It is deliberately allowed to diverge from the true BPE count; the label is
 *     exactly what tells a consumer not to trust it as precise.
 *
 * FOUR STANDING CONSTRAINTS (this module is a CI-testable contract for each):
 *   - DETERMINISTIC: same text + same tokenizer id ⇒ byte-identical {@link TokenCount}. No
 *     wall clock, no RNG, NO NETWORK on the read path — `encode()`, `ceil(len/N)`, and the native
 *     cache lookup are all pure functions of committed bytes.
 *   - NEVER SILENTLY STUBBED: if `js-tiktoken` is present the oracle MUST use it (no silent
 *     downgrade); if absent it reports `chars-approx` and logs ONCE that the precise backend
 *     is unavailable (visible, not silent). The native backend NEVER falls back to an estimate on
 *     a miss — it throws {@link NativeTokenCacheMissError} rather than emit a fake-precise number.
 *   - OFF THE RUNTIME/RECORD PATH: this is a MEASUREMENT tool for renderer/benchmark cost. A
 *     closed model's token count is an external estimate, scored into `docs/results`, never
 *     stamped onto the deterministic Blotter/Grade or the bus. It is imported by benchmark /
 *     renderer code, never by the engine, session, bus, or any canonical receipt.
 *   - ISOLATED FROM LIBRARY EXPORTS: a leaf module. `src/render/index.ts` homes the lightweight
 *     render tokenizer contract (a pure `count(text): number` budget type) but NOT this oracle;
 *     `src/index.ts` / `src/protocol` / `src/lang` never reach THIS module. `js-tiktoken`,
 *     `node:fs`, and `node:crypto` are reached ONLY through lazy dynamic imports here, so the
 *     Bun/Node/Workers library surface (dist/index.js) stays dependency-free and Workers-safe.
 */

/**
 * Tokenizers the oracle can measure a Rendering's cost under. Extensible: add an id here and
 * a backend to {@link REGISTRY}. Two open tokenizers today: `cl100k_base` (GPT-3.5/4, older
 * GPT family) and `o200k_base` (GPT-4o / o-series / Codex family — the tokenizer the `codex`
 * agent's model reads under). Open-weight tokenizers are one registry entry each. A CLOSED
 * model registers one of two labelled backends: a fallback-only `chars-approx` backend when
 * nothing better exists (Gemini's SentencePiece), OR — when the provider ships an authoritative
 * count endpoint — a NATIVE backend serving precise counts from a committed offline cache.
 * `fable-native` is the second case: Claude/Fable's proprietary BPE, measured via the Anthropic
 * count_tokens endpoint (model {@link NATIVE_MODEL}). Never a fake-precise count.
 */
export type TokenizerId = "cl100k_base" | "o200k_base" | "fable-native";

/**
 * The honest label of HOW a count was produced — this is the anti-fake-precise guarantee. A
 * consumer reads `method` to know whether it holds a precise BPE count (`tiktoken-cl100k`), a
 * provider-authoritative count read from the committed native cache (`anthropic-count-tokens`), or
 * a documented approximation (`chars-approx`). Never invent an unlabelled path.
 */
export type CountMethod =
  | "tiktoken-cl100k"
  | "tiktoken-o200k"
  | "anthropic-count-tokens"
  | "chars-approx";

/** The model whose tokenizer `fable-native` measures — the `model` on every count_tokens request
 * AND the label stamped (with {@link NATIVE_METHOD} + `measured_at`) into every emitted artifact
 * (ADR-0043 declared-tokenizer). */
export const NATIVE_MODEL = "claude-fable-5";

/** The {@link CountMethod} label for the native backend: the count came from the Anthropic
 * count_tokens endpoint, not a locally reproducible BPE table. */
export const NATIVE_METHOD = "anthropic-count-tokens" as const;

/** The Anthropic token-count endpoint the native cache is measured against (stamped in artifacts). */
export const NATIVE_ENDPOINT = "/v1/messages/count_tokens";

/**
 * The FIXED, COMMITTED base message the native DIFFERENCING protocol subtracts. A count_tokens
 * response includes per-message/wrapper overhead; measuring `count(base + fragment) − count(base)`
 * cancels that overhead and isolates the fragment's true native cost. Editing this string
 * invalidates EVERY cached differenced count — the cache stores `base_sha256` so a drift is caught
 * at load ({@link NativeTokenCacheBaseDriftError}), never silently mis-differenced. Generic
 * wording — carries no application/strategy (ARCHITECTURE §7).
 */
export const NATIVE_DIFF_BASE =
  "kestrel native-token differencing base: a fixed committed anchor message. " +
  "Do not edit — editing invalidates every cached differenced count.";

/** The maintainer command that (networked, with ANTHROPIC_API_KEY) measures + rewrites the cache.
 * Named verbatim in the fail-closed miss error so the repair path is discoverable from the error. */
export const NATIVE_REFRESH_COMMAND =
  "bun scripts/percept-lab/refresh-native-cache.ts --refresh-native-cache";

/**
 * A measured token cost of a Rendering under one tokenizer. Lives OFF the record path: a
 * closed model's count is an external estimate, never a canonical fact on the Blotter/Grade.
 */
export interface TokenCount {
  readonly count: number;
  /** Precise BPE vs documented approximation — the count's provenance, always present. */
  readonly method: CountMethod;
  readonly tokenizer: TokenizerId;
}

/**
 * A resolved counter: pure + sync + deterministic once its backend is loaded. The async
 * dynamic-import cost is paid once by {@link loadTokenizer}; then `count()` is a pure function,
 * which is exactly what a tournament that counts many Renderings under one tokenizer needs.
 *
 * NAMED distinctly from the render barrel's `Tokenizer` ({@link ./index.ts}) on purpose (kestrel-2fab):
 * this ORACLE returns a LABELLED {@link TokenCount} (precise BPE vs documented estimate), a richer
 * contract than the render path's lightweight budget tokenizer (`count(text): number`). One name per
 * contract so neither can silently drift into the other.
 */
export interface TokenCounter {
  readonly id: TokenizerId;
  readonly method: CountMethod;
  count(text: string): TokenCount;
}

/**
 * Documented approximation ratio: ~4 chars/token for English (OpenAI's own rule of thumb).
 * The `chars-approx` fallback is `ceil(text.length / CHARS_PER_TOKEN)`.
 */
export const CHARS_PER_TOKEN = 4;

/**
 * How a {@link TokenizerId} maps to a backend — a discriminated union, so a closed model can
 * register a `native` backend (count_tokens + committed cache) alongside the open `tiktoken`
 * backends WITHOUT any fake-precise or null-encoding shim. Extending the oracle = adding a row.
 */
type Backend =
  | { readonly kind: "tiktoken"; readonly encoding: string; readonly method: "tiktoken-cl100k" | "tiktoken-o200k" }
  | { readonly kind: "native"; readonly model: string; readonly method: typeof NATIVE_METHOD };

/** The registry. One entry per known tokenizer; extending the oracle = adding a row. */
const REGISTRY: Readonly<Record<TokenizerId, Backend>> = {
  cl100k_base: { kind: "tiktoken", encoding: "cl100k_base", method: "tiktoken-cl100k" },
  o200k_base: { kind: "tiktoken", encoding: "o200k_base", method: "tiktoken-o200k" },
  "fable-native": { kind: "native", model: NATIVE_MODEL, method: NATIVE_METHOD },
};

/** Minimal structural view of the parts of `js-tiktoken` the oracle uses. Kept local (not an
 *  import) so this module typechecks and ships WITHOUT the optional dependency installed. The
 *  interop union covers both the ESM named export and a CJS default-wrapped shape. */
type Encoder = { encode(text: string): readonly number[] };
type TiktokenModule = {
  getEncoding?: (name: string) => Encoder;
  default?: { getEncoding?: (name: string) => Encoder };
};

/**
 * The always-available, dependency-free fallback. `method` is always `chars-approx`, so it can
 * never masquerade as a precise BPE count; `count = ceil(len / CHARS_PER_TOKEN)` (0 for "").
 */
export function approxTokenizer(id: TokenizerId): TokenCounter {
  return {
    id,
    method: "chars-approx",
    count(text: string): TokenCount {
      return {
        count: Math.ceil(text.length / CHARS_PER_TOKEN),
        method: "chars-approx",
        tokenizer: id,
      };
    },
  };
}

/** Wrap a resolved js-tiktoken encoder as a precise, per-id-labelled TokenCounter (the `method` names
 * the EXACT BPE table — `tiktoken-cl100k` or `tiktoken-o200k` — never a generic "tiktoken"). */
function tiktokenTokenizer(id: TokenizerId, enc: Encoder, method: "tiktoken-cl100k" | "tiktoken-o200k"): TokenCounter {
  return {
    id,
    method,
    count(text: string): TokenCount {
      return {
        count: enc.encode(text).length,
        method,
        tokenizer: id,
      };
    },
  };
}

/** Ids we have already logged a fallback for — so the "backend unavailable" line prints ONCE. */
const warned = new Set<TokenizerId>();
/** Memoised loads: the dynamic import + encoding build is paid once per id, then reused. */
const cache = new Map<TokenizerId, Promise<TokenCounter>>();

async function resolveTokenizer(id: TokenizerId): Promise<TokenCounter> {
  const backend = REGISTRY[id];
  // A closed model with a provider-authoritative count endpoint: served offline from the committed
  // native cache. NEVER falls back to an estimate — a miss THROWS (fail-closed) rather than fake it.
  if (backend.kind === "native") return (await loadNativeTokenizer()).tokenizer;
  try {
    // Lazy + indirected specifier: the `: string` type stops tsc from resolving `js-tiktoken`
    // at build time, so the package typechecks/builds/packs with the optional dep uninstalled.
    const spec: string = "js-tiktoken";
    const mod = (await import(spec)) as TiktokenModule;
    const getEncoding = mod.getEncoding ?? mod.default?.getEncoding;
    if (typeof getEncoding !== "function") throw new Error("js-tiktoken: getEncoding not found");
    return tiktokenTokenizer(id, getEncoding(backend.encoding), backend.method);
  } catch {
    // Backend absent/unloadable: fall back to the LABELLED approximation — never a silent
    // downgrade, never a fake-precise count. Log once so the degrade is visible, not silent.
    if (!warned.has(id)) {
      warned.add(id);
      console.warn(
        `token oracle: precise backend 'js-tiktoken' unavailable for '${id}'; ` +
          `falling back to labelled 'chars-approx' (ceil(len/${CHARS_PER_TOKEN})).`,
      );
    }
    return approxTokenizer(id);
  }
}

/**
 * Lazily resolve the counter for `id`: try the js-tiktoken backend via a dynamic import; on ANY
 * load failure fall back to {@link approxTokenizer}. Never throws. Memoised, so the async cost
 * is paid once and the returned {@link TokenCounter.count} is thereafter pure + sync.
 */
export function loadTokenizer(id: TokenizerId): Promise<TokenCounter> {
  const hit = cache.get(id);
  if (hit) return hit;
  const loading = resolveTokenizer(id);
  cache.set(id, loading);
  return loading;
}

/** Convenience: load the tokenizer for `id` and count `text` in one call. */
export async function countTokens(text: string, id: TokenizerId): Promise<TokenCount> {
  const tokenizer = await loadTokenizer(id);
  return tokenizer.count(text);
}

// ─────────────────────────────────────────────────────────────────────────────
// The `fable-native` backend — Anthropic count_tokens, DIFFERENCED, served OFFLINE from a
// committed cache. This is the ADR-0043 declared-tokenizer coordinate for a closed model.
// ─────────────────────────────────────────────────────────────────────────────

/** One measured count: the authoritative `input_tokens` the count_tokens endpoint returned for a
 * specific byte string, stamped with the `model` measured under and the `measured_at` timestamp
 * (ADR-0043 label). Keyed in {@link NativeTokenCache.entries} by `sha256(utf8(bytes))`. */
export interface NativeTokenCacheEntry {
  readonly model: string;
  readonly input_tokens: number;
  /** ISO-8601 instant the measurement was taken (maintainer refresh time — off the read path). */
  readonly measured_at: string;
}

/**
 * The committed offline cache the native backend reads. `entries` maps `sha256(utf8(measured
 * string))` → its measured count; the measured strings are the DIFFERENCING pairs — the base
 * ({@link NATIVE_DIFF_BASE}) and every `base + fragment`. `base_sha256` pins the base so an edit to
 * {@link NATIVE_DIFF_BASE} is caught at load rather than silently mis-differenced. An EMPTY
 * `entries` object is schema-valid (the Day-0 unmeasured state): every count then fails closed with
 * a repair-guiding {@link NativeTokenCacheMissError}, never a fabricated number.
 */
export interface NativeTokenCache {
  readonly tokenizer: "fable-native";
  readonly method: typeof NATIVE_METHOD;
  readonly model: string;
  readonly endpoint: string;
  readonly base_sha256: string;
  readonly note: string;
  readonly entries: Readonly<Record<string, NativeTokenCacheEntry>>;
}

/**
 * Fail-closed miss: the native backend was asked to count a byte string whose measurement is not in
 * the committed cache, in offline (read) mode. Sibling in spirit to a KernelHonestyError — the
 * oracle refuses to fabricate a precise-looking count. Carries the missing `sha` and the refresh
 * command so the repair path is discoverable straight from the thrown error.
 */
export class NativeTokenCacheMissError extends Error {
  readonly sha: string;
  constructor(sha: string, which: string) {
    super(
      `native token cache MISS: ${which} (sha256=${sha}) is not in the committed cache. ` +
        `The '${"fable-native"}' backend is offline-deterministic and will NOT hit the network at ` +
        `read/CI time. To measure and commit it, run (maintainer, needs ANTHROPIC_API_KEY): ` +
        `${NATIVE_REFRESH_COMMAND}`,
    );
    this.name = "NativeTokenCacheMissError";
    this.sha = sha;
  }
}

/** The committed cache's `base_sha256` disagrees with `sha256(NATIVE_DIFF_BASE)` — the base message
 * was edited without a re-measure, so every differenced count would be wrong. Fail closed at load. */
export class NativeTokenCacheBaseDriftError extends Error {
  constructor(cachedBase: string, computedBase: string) {
    super(
      `native token cache base drift: cache.base_sha256=${cachedBase} but sha256(NATIVE_DIFF_BASE)=` +
        `${computedBase}. NATIVE_DIFF_BASE was edited without a re-measure — every differenced count ` +
        `would be wrong. Re-run: ${NATIVE_REFRESH_COMMAND}`,
    );
    this.name = "NativeTokenCacheBaseDriftError";
  }
}

/** Is the committed cache in the Day-0 unmeasured (empty-but-schema-valid) state? Used by artifact
 * renderers to print a single "cache empty — run refresh" reason instead of a per-sha miss. */
export function isNativeCacheEmpty(cacheData: NativeTokenCache): boolean {
  return Object.keys(cacheData.entries).length === 0;
}

/**
 * Construct the native {@link TokenCounter} from a cache + a sha256 function (injected so this stays a
 * pure function — the caller supplies `node:crypto` off the runtime path, and tests can supply the
 * same hash to exercise the DIFFERENCING arithmetic without touching the network).
 *
 * `count(text)` = `entries[sha(base+text)].input_tokens − entries[base].input_tokens`. The base
 * subtraction cancels the count_tokens message/wrapper overhead, isolating the fragment's true
 * native cost. A missing entry (base or base+text) throws {@link NativeTokenCacheMissError}.
 */
export function nativeTokenizer(cacheData: NativeTokenCache, sha256Hex: (s: string) => string): TokenCounter {
  const baseKey = sha256Hex(NATIVE_DIFF_BASE);
  if (cacheData.base_sha256 !== baseKey) throw new NativeTokenCacheBaseDriftError(cacheData.base_sha256, baseKey);
  return {
    id: "fable-native",
    method: NATIVE_METHOD,
    count(text: string): TokenCount {
      const base = cacheData.entries[baseKey];
      if (base === undefined) throw new NativeTokenCacheMissError(baseKey, "differencing base");
      const fullKey = sha256Hex(NATIVE_DIFF_BASE + text);
      const full = cacheData.entries[fullKey];
      if (full === undefined) throw new NativeTokenCacheMissError(fullKey, `base+fragment (${text.length} chars)`);
      return { count: full.input_tokens - base.input_tokens, method: NATIVE_METHOD, tokenizer: "fable-native" };
    },
  };
}

/** The committed cache fixture path (relative to repo root). The default the loader reads. */
export const NATIVE_TOKEN_CACHE_REL_PATH = "docs/percept-lab/native-token-cache.json";

/** Lazily read `node:crypto` (off the library surface) and return a sync sha256-hex function. */
async function nativeSha256(): Promise<(s: string) => string> {
  const { createHash } = await import("node:crypto");
  return (s: string) => createHash("sha256").update(s, "utf8").digest("hex");
}

/**
 * Read the committed native-token cache fixture (default {@link NATIVE_TOKEN_CACHE_REL_PATH}, two
 * levels up from this module). Lazy `node:fs` import keeps the library surface Workers-safe. Throws
 * if the file is missing (the fixture must be committed) — but an EMPTY `entries` object is valid.
 */
export async function loadNativeTokenCache(path?: string): Promise<NativeTokenCache> {
  const { readFileSync } = await import("node:fs");
  const target = path ?? new URL(`../../${NATIVE_TOKEN_CACHE_REL_PATH}`, import.meta.url).pathname;
  const raw = readFileSync(target, "utf8");
  const parsed = JSON.parse(raw) as NativeTokenCache;
  if (parsed.tokenizer !== "fable-native" || parsed.method !== NATIVE_METHOD || typeof parsed.entries !== "object") {
    throw new Error(`native token cache at ${target} is not schema-valid (tokenizer/method/entries)`);
  }
  return parsed;
}

/** Memoised native load — the fixture read + crypto import is paid once. */
let nativeLoad: Promise<{ tokenizer: TokenCounter; cacheData: NativeTokenCache }> | null = null;

/**
 * Resolve the native tokenizer AND the cache it reads (the cache is returned too so artifact
 * renderers can stamp `model`/`measured_at` and detect the empty state). Offline + deterministic.
 */
export function loadNativeTokenizer(path?: string): Promise<{ tokenizer: TokenCounter; cacheData: NativeTokenCache }> {
  if (nativeLoad !== null && path === undefined) return nativeLoad;
  const loading = (async () => {
    const [sha, cacheData] = await Promise.all([nativeSha256(), loadNativeTokenCache(path)]);
    return { tokenizer: nativeTokenizer(cacheData, sha), cacheData };
  })();
  if (path === undefined) nativeLoad = loading;
  return loading;
}