kestrel.markets/src/ledger/hosted.ts

A typed, token-efficient language + runtime for agentic trading: agents author bounded plans, the runtime fires them at the tick. CLI + typed library + MCP server.

/**
 * # ledger/hosted — the LOCAL receipt of a HOSTED `sim` run (bead kestrel-pvlc).
 *
 * The durable {@link ./index.ts run/plan/lineage ledger} is `bun:sqlite` and records only LOCAL
 * `run`/`day` sessions. A hosted `sim` (`../cli/commands/sim.ts`) is LIGHT by construction — it must
 * run under plain node for every agent who copies the snippet, so it can touch NEITHER `bun:sqlite`
 * NOR the network beyond the funnel it already drove. Yet the doctrine that every state change leaves
 * a LEGIBLE, RETRIEVABLE receipt applies to hosted sims too: without a local ledger of its own
 * attempts, a first-party agent iterating a strategy across sims has no way to re-list its operations,
 * grades, or proof URLs (they print exactly once on stdout).
 *
 * This module is that sibling lineage: an append-only JSONL file that sim appends one row to per
 * completed hosted run, and `runs list` reads. It imports NOTHING Bun-only (node `fs`/`path`/`os`
 * only), so it loads on the LIGHT sim path.
 *
 * ## Where it lives — HOME, not CWD (kestrel-fq78 / kestrel-markets-p3af)
 * The receipt store is IDENTITY-BOUND: it lives beside the agent's credential + operator secrets in
 * `~/.kestrel/hosted-runs.jsonl` ({@link defaultHostedStorePath}, the `KESTREL_HOME`-overridable home
 * established by `cli/credentials.ts`, kestrel-jh9w.1), NOT in a CWD-relative `./data/`. Identity
 * already follows the agent globally; the run ledger now follows it too, so `runs list`/`runs show`/
 * `runs compare` resolve the same authenticated agent's prior runs — and their `art_`/`op_` handles —
 * from ANY working directory. The old CWD-local path ({@link LEGACY_CWD_HOSTED_STORE}) survives ONLY
 * as a READ-COMPAT fallback ({@link readHostedRunsMerged}): receipts an older CLI wrote into an
 * originating folder stay visible there and are never dropped, but new writes always go home. This is
 * a merged READ VIEW, never a migrate-on-touch — a query verb never mints or mutates a store.
 *
 * ## The rules it holds (mirroring the sqlite ledger)
 * - **Idempotent by operation id.** Re-listing collapses rows on `operation_id` (the hosted run's
 *   identity), keeping the LAST write — a re-run of the same Operation is one receipt, never a
 *   duplicate (ADR-0007 idempotent-upsert, held at read time since append-only can't rewrite in place).
 * - **No wall clock.** The row's `issued_at` is the platform's signed-proof timestamp (server-provided,
 *   part of the grade), never `Date.now()` — the hosted receipt stays reproducible (RUNTIME §0).
 * - **Fail-closed on corruption.** A present-but-unparseable store line is a loud error, never a
 *   silently-dropped receipt — the same discipline `runs list` holds for a corrupt sqlite store.
 */

import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";

/** The store's basename, so a caller can derive it as a sibling of a ledger `--db` path or the home dir. */
export const HOSTED_STORE_BASENAME = "hosted-runs.jsonl";

/**
 * The LEGACY CWD-local hosted-lineage store — a sibling of the default ledger `data/kestrel.db`. Kept
 * ONLY as a READ-COMPAT fallback ({@link readHostedRunsMerged}): older CLIs wrote receipts here, so they
 * must stay visible in the originating folder. New writes go to {@link defaultHostedStorePath} (home).
 */
export const LEGACY_CWD_HOSTED_STORE = `data/${HOSTED_STORE_BASENAME}`;

/** The env this module reads (only `KESTREL_HOME`, mirroring `cli/credentials.ts` — for test isolation). */
interface HostedStoreEnv {
  readonly [k: string]: string | undefined;
}

/**
 * The default (WRITE + primary READ) hosted-run store: `$KESTREL_HOME/.kestrel/hosted-runs.jsonl` (else
 * `~/.kestrel/hosted-runs.jsonl`). IDENTITY-BOUND — it lives in the SAME owner-scoped home dir the
 * credential + operator-secrets store establish (kestrel-jh9w.1), so the run ledger follows the agent
 * across working directories instead of being trapped in the folder a run happened to start in
 * (kestrel-fq78 / kestrel-markets-p3af). A pure path: no wall clock, no RNG (RUNTIME §0). `KESTREL_HOME`
 * overrides the base dir so tests stay hermetic and CI/containers can redirect it.
 */
export function defaultHostedStorePath(env: HostedStoreEnv = process.env): string {
  const home = env.KESTREL_HOME ?? homedir();
  return join(home, ".kestrel", HOSTED_STORE_BASENAME);
}

/**
 * One hosted-run receipt row. `source` is a literal discriminant so a merged `runs list` can label
 * each row's lineage; the rest is the print-once story made retrievable — the scenario slug, the
 * Operation id (its identity), the grade digest (order/fill/pnl), the shareable proof URL, the durable
 * evidence artifact ids, and the platform's signed timestamp.
 */
export interface HostedRunRow {
  readonly source: "hosted";
  readonly slug: string;
  readonly requested_slug: string;
  readonly operation_id: string;
  readonly grade_artifact_id: string;
  readonly bus_artifact_id: string;
  readonly manifest_artifact_id: string;
  readonly order_count: number;
  readonly fill_count: number;
  readonly realized_pnl: number;
  readonly proof_url: string;
  readonly issued_at: string;
  /**
   * The human LABEL of the plan/strategy that produced this run — the `--plans <file>` path, or the
   * label of the stand-down/bundled default when no `--plans` was supplied (kestrel-etyu). For
   * DISPLAY/legibility; it is NOT a content identity — an in-place edit of one `--plans` path keeps
   * the same label (see {@link strategy_digest} for the identity that moves with the content).
   *
   * ADDITIVE-OPTIONAL. Receipts written before this field lack it; a missing (or non-string) value
   * reads as UNKNOWN and is a NORMAL back-compat state, never a refusal. This is a LEGIBILITY field
   * on the DISPLAY path, NOT the runtime trust path, so its read fails SOFT (the required grade
   * fields still fail closed). Historical records are never rewritten.
   */
  readonly strategy?: string;
  /**
   * The deterministic CONTENT identity of the plan/strategy — `sha256` of the canonical plan text,
   * 64-char lowercase hex (kestrel-etyu). THIS is what makes two runs distinguishable in the
   * returning-user workflow ("edit my plan file in place and re-run the same `--plans` path; did my
   * tweak help?"): the {@link strategy} label is identical across such an edit, but the digest moves
   * with the content, so `runs list` / `runs compare` can tell the two runs apart. Same ADDITIVE-
   * OPTIONAL, fail-soft, never-rewritten discipline as {@link strategy}.
   */
  readonly strategy_digest?: string;
  /**
   * The OPAQUE copy-token this run continued from, recorded VERBATIM (never interpreted — repo
   * boundary) so a `--copy-token` continuation threads back to its origin instead of being an
   * island (kestrel-gu9h). Optional: a run started without a copy-token carries none, and an older
   * receipt line that predates this field still parses (it is not required by the row validator).
   */
  readonly copy_token?: string;
  /**
   * The grader's certified explanation of WHY this run placed no orders (kestrel-kglw) — the
   * single highest-value teaching moment of a 0-order hosted run. The hosted `sim` face states it
   * ONCE on stdout (the `why` line); without it on the receipt, a returning user who `runs show`s
   * the SAME run re-reads a bare `order_count=0` with the reason lost — the exact silent-gate class
   * the product claims to have closed, one surface downstream. Recorded so the reason is DURABLE and
   * re-openable, not print-once. Surfaced VERBATIM (the CLI never recomputes a grader verdict).
   *
   * ADDITIVE-OPTIONAL, fail-SOFT (a DISPLAY field, like {@link strategy}): a receipt written before
   * this field lacks it; a run that TRADED (order_count > 0) never carries it; an empty/blank value
   * NORMALIZES to absent rather than refusing the row (the required grade fields still fail closed).
   * Historical records are never rewritten.
   */
  readonly never_fired_reason?: string;
}

/**
 * Append one hosted-run receipt to the JSONL store at `storePath`, creating the parent dir if absent
 * (a fresh cwd has no `data/` yet). One row per line, JSON, newline-terminated — an append-only ledger
 * a later `runs list` reads. Zero network; a pure local fs write.
 */
export function appendHostedRun(storePath: string, row: HostedRunRow): void {
  const dir = dirname(storePath);
  if (dir.length > 0 && !existsSync(dir)) mkdirSync(dir, { recursive: true });
  appendFileSync(storePath, JSON.stringify(row) + "\n");
}

/**
 * Read the hosted-run receipts at `storePath`, recent-first. An ABSENT store is a NORMAL empty state
 * ([]), never a fault — a query verb never mints a store. A PRESENT store with an unparseable line is a
 * loud error (fail-closed: a corrupt local ledger is never silently read as empty).
 *
 * Determinism: rows are collapsed idempotently on `operation_id` (LAST write wins, mirroring the
 * sqlite ledger's `INSERT OR REPLACE`), then ordered `issued_at` desc, then `operation_id` asc — a
 * total, wall-clock-free order (the timestamp is the server's signed proof time).
 */
export function readHostedRuns(storePath: string): HostedRunRow[] {
  return collapseHostedRuns(readRawHostedRows(storePath));
}

/**
 * Read the hosted receipts as a MERGED VIEW across the identity-bound HOME store (the primary, and the
 * only write target) and any number of legacy CWD-local FALLBACK stores (read-compat for receipts an
 * older CLI wrote into an originating folder). This is what makes `runs list`/`runs show`/`runs compare`
 * resolve the same authenticated agent's runs from ANY cwd (kestrel-fq78 / kestrel-markets-p3af).
 *
 * Purely a READ view: it NEVER writes or migrates a store (no migrate-on-touch), holding the invariant
 * that a query verb never mints/mutates a store. Rows are deduped by `operation_id` — the HOME store
 * WINS a collision (it is the durable, identity-bound record; a run present in both places is the same
 * Operation, so this only affects which byte-identical copy is kept) — then the same wall-clock-free
 * total order as {@link readHostedRuns} is applied. A corrupt store on ANY source faults LOUDLY
 * (fail-closed), never a silently-dropped receipt.
 */
export function readHostedRunsMerged(opts: { home: string; fallbacks?: readonly string[] }): HostedRunRow[] {
  // Fallbacks first, HOME last: `collapseHostedRuns` keeps the LAST write per Operation id, so a home
  // row overrides a same-id CWD-local row. De-dupe the source paths so a fallback that resolves to the
  // home path (e.g. cwd == home) is not read twice.
  const seen = new Set<string>();
  const rows: HostedRunRow[] = [];
  for (const path of [...(opts.fallbacks ?? []), opts.home]) {
    if (seen.has(path)) continue;
    seen.add(path);
    rows.push(...readRawHostedRows(path));
  }
  return collapseHostedRuns(rows);
}

/** Parse + validate one store's receipt lines in FILE ORDER (no collapse). An absent store is a NORMAL
 * empty state ([]). A present-but-unparseable line, or a line that is not a hosted-run row, is a LOUD
 * fail-closed error — a corrupt local ledger is never silently read as empty. */
function readRawHostedRows(storePath: string): HostedRunRow[] {
  if (!existsSync(storePath)) return [];
  const text = readFileSync(storePath, "utf8");
  const out: HostedRunRow[] = [];
  const lines = text.split("\n");
  for (let i = 0; i < lines.length; i += 1) {
    const line = lines[i]!.trim();
    if (line.length === 0) continue;
    let parsed: unknown;
    try {
      parsed = JSON.parse(line);
    } catch (e) {
      throw new Error(
        `ledger/hosted: ${JSON.stringify(storePath)} line ${i + 1} is not valid JSON — refusing to read a corrupt hosted store (fail-closed): ${e instanceof Error ? e.message : String(e)}`,
      );
    }
    const row = asHostedRunRow(parsed);
    if (row === null)
      throw new Error(
        `ledger/hosted: ${JSON.stringify(storePath)} line ${i + 1} is not a hosted-run row — refusing to read a corrupt hosted store (fail-closed)`,
      );
    out.push(row);
  }
  return out;
}

/** Collapse receipts idempotently on `operation_id` (LAST in array order wins, mirroring the sqlite
 * ledger's `INSERT OR REPLACE`), then order `issued_at` desc, then `operation_id` asc — a total,
 * wall-clock-free order (the timestamp is the server's signed proof time). */
function collapseHostedRuns(rows: readonly HostedRunRow[]): HostedRunRow[] {
  const byOp = new Map<string, HostedRunRow>();
  for (const row of rows) byOp.set(row.operation_id, row); // last write wins (idempotent upsert by Operation id)
  return [...byOp.values()].sort(
    (a, b) => (a.issued_at < b.issued_at ? 1 : a.issued_at > b.issued_at ? -1 : a.operation_id < b.operation_id ? -1 : 1),
  );
}

/**
 * The ONE definition of a KNOWN plan digest (kestrel-etyu): a 64-char lowercase-hex sha256. Anything
 * else — absent, EMPTY STRING, wrong length/case, non-hex, a non-string — is UNKNOWN.
 *
 * This predicate is shared by the store's normalization ({@link asHostedRunRow}) and every renderer, so
 * "unknown" cannot mean two different things in two places. It exists because it once did: an empty-string
 * digest passed a bare `typeof === "string"` check and read as KNOWN, so `runs compare` answered
 * `same_plan: true` ("A and B ran the SAME plan content") for two runs whose digests `runs list` was
 * simultaneously rendering as `—`. Fabricating a MATCH from UNKNOWN is the dangerous direction: it asserts
 * "your tweak changed nothing" when the truth is unknown. Unknown is never a match.
 */
export function isPlanDigest(v: unknown): v is string {
  return typeof v === "string" && /^[0-9a-f]{64}$/.test(v);
}

/** Narrow an untrusted parsed line to a {@link HostedRunRow}, or `null` (→ the caller's loud refusal).
 * The row is REBUILT from the known field set — an unknown extra field on a store line must never leak
 * verbatim into the `kestrel.runs/v1` / `kestrel.hosted-run/v1` machine payloads (the store is local
 * but still untrusted input; the wire schemas name exactly these fields). */
function asHostedRunRow(v: unknown): HostedRunRow | null {
  if (typeof v !== "object" || v === null || Array.isArray(v)) return null;
  const r = v as Record<string, unknown>;
  if (r["source"] !== "hosted") return null;
  const str = (k: string): boolean => typeof r[k] === "string";
  const num = (k: string): boolean => typeof r[k] === "number" && Number.isFinite(r[k] as number);
  if (!(str("slug") && str("requested_slug") && str("operation_id") && str("grade_artifact_id"))) return null;
  if (!(str("bus_artifact_id") && str("manifest_artifact_id") && str("proof_url") && str("issued_at"))) return null;
  if (!(num("order_count") && num("fill_count") && num("realized_pnl"))) return null;
  // `strategy` + `strategy_digest` are ADDITIVE-OPTIONAL (kestrel-etyu): a receipt written before the
  // fields existed lacks them, and an unusable value is NORMALIZED TO ABSENT rather than refused — these
  // are display fields, so they fail SOFT (the required grade fields above still fail closed). Never
  // rewritten onto old rows.
  //
  // Normalization is the point: this is the ONE place an untrusted store line becomes a typed row, so
  // "unknown" is decided HERE and nowhere else. An empty-string label and a non-64-hex digest are UNKNOWN
  // — not "known and blank". Letting `""` through as a known digest is what once made `runs compare`
  // report `same_plan: true` for two runs `runs list` was rendering as `—`.
  const strategyRaw = r["strategy"];
  const strategy = typeof strategyRaw === "string" && strategyRaw !== "" ? strategyRaw : undefined;
  const strategyDigest = isPlanDigest(r["strategy_digest"]) ? r["strategy_digest"] : undefined;
  // `copy_token` is OPTIONAL: carried through verbatim when present and a non-empty string, absent
  // otherwise (an older receipt predating the field, or a run with no continuation). A present-but-
  // non-string value is a malformed row → reject (the caller's loud, fail-closed refusal).
  const rawCopyToken = r["copy_token"];
  if (rawCopyToken !== undefined && typeof rawCopyToken !== "string") return null;
  // `never_fired_reason` is ADDITIVE-OPTIONAL and fail-SOFT (a display field, kestrel-kglw): a non-
  // string or blank value normalizes to ABSENT (no reason to surface), never a row refusal — the same
  // discipline as `strategy`. Only a non-empty, trimmed string is a reason worth re-showing.
  const rawNeverFired = r["never_fired_reason"];
  const neverFiredReason =
    typeof rawNeverFired === "string" && rawNeverFired.trim().length > 0 ? rawNeverFired : undefined;
  return {
    source: "hosted",
    slug: r["slug"] as string,
    requested_slug: r["requested_slug"] as string,
    operation_id: r["operation_id"] as string,
    grade_artifact_id: r["grade_artifact_id"] as string,
    bus_artifact_id: r["bus_artifact_id"] as string,
    manifest_artifact_id: r["manifest_artifact_id"] as string,
    order_count: r["order_count"] as number,
    fill_count: r["fill_count"] as number,
    realized_pnl: r["realized_pnl"] as number,
    proof_url: r["proof_url"] as string,
    issued_at: r["issued_at"] as string,
    ...(strategy !== undefined ? { strategy } : {}),
    ...(strategyDigest !== undefined ? { strategy_digest: strategyDigest } : {}),
    ...(typeof rawCopyToken === "string" && rawCopyToken.length > 0 ? { copy_token: rawCopyToken } : {}),
    ...(neverFiredReason !== undefined ? { never_fired_reason: neverFiredReason } : {}),
  };
}