kestrel.markets/src/provenance/orphan-store.ts

A typed, token-efficient language + runtime for agentic trading: agents author bounded plans, the runtime fires them at the tick. CLI + typed library + MCP server.

/**
 * # provenance/orphan-store — the no-orphan-store registry + fail-closed guard (OSS-ADR-0010)
 *
 * OSS-ADR-0010 (*legible text is truth; binary stores are regenerable projections*) makes one
 * consequence structural here: **"every new artifact must name the text truth it projects from and
 * ship a regeneration path (no orphan stores)."** A store you cannot regenerate from legible text is
 * a store you cannot certify — and certification-by-re-projection is the recomputability the whole
 * KestrelBench flywheel rests on. An *orphan store* is a durable write with no named text truth and
 * no byte-for-byte regeneration path: exactly the thing that silently becomes a second truth.
 *
 * This module is the executable manifest of every durable store the package persists. Each entry
 * NAMES the store, classifies it under OSS-ADR-0010's four persistence classes, states the legible
 * **text truth** it derives from, and states its **regeneration** path. The companion gate
 * (`scripts/check-orphan-stores.ts`) scans `src/**` for durable-write call-sites and refuses (fail
 * closed) any write-site that is not covered here — so a NEW orphan store cannot land unannounced,
 * the same way `scripts/check-nul-delimiters.ts` keeps the grep-blind NUL set known.
 *
 * ## The four persistence classes (OSS-ADR-0010)
 * - **recorded-truth** — the append-only Bus JSONL (and its sibling append-only lineages). The
 *   canonical serialization certification hashes are computed over. It IS the text truth: it derives
 *   from nothing and regenerates from nothing (self-truth).
 * - **authored-truth** — versioned `.kestrel` documents (modules). Also a text truth; a persisted
 *   serialization of one is a cache of the authored text.
 * - **projection** — a regenerable function of a truth: the Blotter (JSON projection of one Bus), the
 *   Ledger (regenerable index across Blotters), rendered Frames (glyph projections), parquet columnar
 *   caches. A projection MUST name a non-empty text truth AND a non-empty regeneration path — delete
 *   it, rebuild it, byte-for-byte answers — or it is an orphan.
 * - **control-plane** — leases, authority grants, commerce evidence, credentials/keypairs. Authority,
 *   not derivation: NOT certifiable by re-projection, so exempt from the regeneration requirement, but
 *   it must still be NAMED and carry its own honesty rule (append-only / owner-only / content-addressed).
 *   Nothing on the deterministic Session path may be load-bearing on a control-plane record.
 *
 * ## Fail-closed
 * {@link validateRegistry} refuses loudly on the two orphan shapes: a projection with a blank text
 * truth or a blank regeneration path (an unregenerable projection is an orphan), and a dead row whose
 * declared write-site the gate cannot confirm. The guard NEVER silently passes a blank.
 */

/** OSS-ADR-0010's four persistence classes. `projection` is the only class the regeneration rule binds. */
export type StoreClass = "recorded-truth" | "authored-truth" | "projection" | "control-plane";

/**
 * One durable store's provenance record. `textTruth` names the legible text it derives from;
 * `regeneration` names the byte-for-byte path that rebuilds it (for a `projection`). For
 * recorded/authored truth these say it IS the truth; for control-plane they name the honesty rule that
 * stands in for re-projection. `writeSites` are the `src/**` files whose durable-write call-sites persist it.
 */
export interface StoreProvenance {
  /** Stable store id (the manifest key). */
  readonly id: string;
  /** What is persisted on disk — the concrete artifact (a path, a file family, a store name). */
  readonly artifact: string;
  /** Which of OSS-ADR-0010's four classes this store is. */
  readonly class: StoreClass;
  /** The legible text truth it derives from. For recorded/authored truth: it IS the truth. */
  readonly textTruth: string;
  /** The byte-for-byte regeneration path. For a projection this MUST be a real path, never blank. */
  readonly regeneration: string;
  /** The `src/**` file(s) whose durable-write call-sites persist this store. */
  readonly writeSites: readonly string[];
}

/** Raised when a store is an orphan (an unregenerable projection, or an unbacked registry row). Throwing
 * is fail-closed: an orphan store is never silently tolerated (OSS-ADR-0010). */
export class OrphanStoreError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "OrphanStoreError";
  }
}

/**
 * The registry: every durable store the package persists, each naming its text truth + regeneration
 * path. The gate maps each scanned durable-write call-site to the entry whose `writeSites` include its
 * file; a write-site with no entry is an orphan store.
 */
export const STORE_REGISTRY: readonly StoreProvenance[] = [
  {
    id: "session-bus",
    artifact: "the session Bus JSONL (a local file today; archive/db adapters must reproduce the canonical bytes)",
    class: "recorded-truth",
    textTruth: "itself — the append-only Bus JSONL IS the recorded truth; determinism hashes are computed over it (CONTEXT.md · RUNTIME §1)",
    regeneration: "n/a — recorded truth is not projected from anything; serializeBus(events) is byte-stable so replay reproduces it exactly",
    writeSites: ["src/bus/write.ts"],
  },
  {
    id: "hosted-runs-lineage",
    artifact: "~/.kestrel/hosted-runs.jsonl — the identity-bound append-only receipt log of hosted sim runs (kestrel-pvlc / kestrel-fq78)",
    class: "recorded-truth",
    textTruth: "itself — an append-only sibling lineage of the Bus; each row is a signed hosted-run receipt (issued_at is the server's proof time, never a wall clock)",
    regeneration: "n/a — append-only recorded truth; idempotent-by-operation_id at read time, never rewritten in place",
    writeSites: ["src/ledger/hosted.ts"],
  },
  {
    id: "plan-fixture-cache",
    artifact: "the plan-fixture cache JSON under the fixtures dir (a decode cache of an authored plan)",
    class: "authored-truth",
    textTruth: "the authored `.kestrel` plan text the fixture serializes (the brief's content hash binds the authored thesis)",
    regeneration: "serializePlanFixture(fixture) — a pure 2-space JSON serialization; delete the cache and re-serialize the authored plan for byte-identical bytes",
    writeSites: ["src/session/harness/plan-fixture.ts"],
  },
  {
    id: "session-report",
    artifact: "the EpisodeReport JSON emitted by a sim run's `out` (a projection of one graded Session; retired in favour of the Blotter, ADR-0039)",
    class: "projection",
    textTruth: "the session Bus — the graded run is a deterministic function of its recorded tape",
    regeneration: "re-grade the same Bus: runSimSession over the same tape re-derives the report; JSON.stringify(report, null, 2) is byte-stable",
    writeSites: ["src/session/sim.ts"],
  },
  {
    id: "day-session-handoff",
    artifact: "the day-session handoff / report files written through DayHooks.write (per-day driver output)",
    class: "projection",
    textTruth: "the session Bus / JOURNAL of that day — a pure function of the day's recorded tape",
    regeneration: "re-run the day session over the same tape; the write hook is injectable, so a test drives the same bytes deterministically",
    writeSites: ["src/session/day.ts"],
  },
  {
    id: "handshake-frames",
    artifact: "the frame-<ord>.json + frame-<ord>.txt handshake evidence written per turn (glyph projection, OFF the graded path)",
    class: "projection",
    textTruth: "the session Bus — the frame bytes are a pure, matched-interface function of the bus, never of the harness (see file-handshake.ts)",
    regeneration: "serializeFrameEnvelope(env) / renderFrameText(env) — pure functions of the FrameEnvelope; re-render for byte-identical frame + text",
    writeSites: ["src/session/harness/file-handshake.ts"],
  },
  {
    id: "paper-session-output",
    artifact: "the paper-session output JSON written by `paper --out` (events/ledger/positions of one paper Session)",
    class: "projection",
    textTruth: "the paper session Bus — modeled fills over a live feed, still a deterministic projection of the recorded events",
    regeneration: "re-project events/ledger/positions from the same Bus; JSON.stringify(result, null, 2) is byte-stable",
    writeSites: ["src/cli/commands/paper.ts"],
  },
  {
    id: "sqlite-ledger",
    artifact: "data/kestrel.db — the bun:sqlite run/plan/lineage Ledger (the queryable, regenerable index across Blotters)",
    class: "projection",
    textTruth: "the graded Blotters / Buses it indexes — run_id = sha256(bus_sha256 + plans_sha256 + fill_model + r_usd), a pure function of inputs",
    regeneration: "re-record: delete the db and replay each Bus through recordRun (INSERT OR REPLACE is an idempotent upsert); same inputs rebuild the same rows",
    writeSites: ["src/ledger/index.ts"],
  },
  {
    id: "lake-segment-cache",
    artifact:
      "the lake Parquet segment cache under the configured cacheDir (`<cacheDir>/<sha256(objectUrl)[0:40]>.parquet`) — the client-side, content-addressed mirror of the tape lake, LRU-evicted within cacheMaxBytes",
    class: "projection",
    textTruth:
      "the remote corpus in object storage (R2/S3) — object storage is the ONLY source of truth (adapters/lake §'Object storage is the ONLY source of truth'); a segment is a transparent mirror of ONE immutable lake object, and a read served warm is still stamped source:\"s3\" because the canonical source never became local",
    regeneration:
      "delete the segment (or the whole cacheDir) and re-read: the cold path re-materializes it from the s3() object at the same URL, and the segment is keyed BY the sha256 of that URL, so the same key always re-fetches the same immutable object's bytes. A lake object at a given key is immutable (versioned lakes rev the path) — a changed body is a NEW key, never a silently different segment.",
    writeSites: ["src/adapters/lake/index.ts"],
  },
  {
    id: "lake-object-cache",
    artifact:
      "the lake raw-object byte cache under `<cacheDir>/objects/` — the byte-level half of the same mirror (the benchmark's .jsonl tape corpus, .csv.zst vendor pulls), LRU-evicted within cacheMaxBytes",
    class: "projection",
    textTruth:
      "the sha256-PINNED remote corpus in object storage — the pin IS the binding: a fetched body is hashed BEFORE it may enter the cache, and a body that does not match its committed pin is deleted and thrown, never served (adapters/lake/objects §'The integrity contract'). A changed object is a NEW object.",
    regeneration:
      "delete the segment and re-ensure: the cold path re-fetches the object from storage and re-verifies it against the SAME committed sha256 pin before committing it — re-fetch + re-pin by content hash. Content-addressing is what makes the rebuild byte-for-byte: bytes that hash to the pin ARE the bytes, or they never land.",
    writeSites: ["src/adapters/lake/objects.ts"],
  },
  {
    id: "credentials",
    artifact: "~/.kestrel/*.json — the stored credential + agent keypair (bearer capability / private key)",
    class: "control-plane",
    textTruth: "n/a — a credential is authority, not a derivation of any text truth; nothing on the deterministic Session path is load-bearing on it (OSS-ADR-0010)",
    regeneration: "n/a — control-plane authority is not certifiable by re-projection; its honesty rule is owner-only 0600 in a 0700 dir, re-minted from the auth flow, never rebuilt from a projection",
    writeSites: ["src/cli/credentials.ts"],
  },
  {
    id: "live-singleton-lock",
    artifact:
      "~/.kestrel/live/<sha256(venue,account)>.lock — the per-host live singleton lock: one live pump per (venue, account) per host, so two processes cannot double every clip on one broker account (kestrel-7o2 Wave A; the SPIKE FLOOR under the control-plane lease kestrel-7o2.2)",
    class: "control-plane",
    textTruth:
      "n/a — a lock is authority (the right to pump live against an account), not a derivation of any text truth. Nothing on the deterministic Session path reads it: it gates whether a live driver may START, and never enters the graded fold (OSS-ADR-0010)",
    regeneration:
      "n/a — control-plane authority is not certifiable by re-projection. Its honesty rule is owner-only 0600 in a 0700 dir, minted by exclusive create (O_EXCL decides the winner, never last-writer-wins), and RECOVERABLE only against a provably-dead pid on the same host — never rebuilt, never inferred from age",
    writeSites: ["src/session/live-lock.ts"],
  },
];

/**
 * Validate a store-provenance registry, fail closed. Refuses on the two orphan shapes:
 *   1. a `projection` (or `authored-truth` cache) with a blank `textTruth` or blank `regeneration` —
 *      an unregenerable projection is an orphan store (OSS-ADR-0010);
 *   2. an entry that declares no `writeSites`, or a blank `id`/`artifact` — an unbacked/unnamed row.
 *
 * A `recorded-truth` / `control-plane` entry is exempt from the regeneration path (it IS the truth, or
 * it is authority not derivation), but it must still NAME a non-empty text-truth clause (self / n/a-with-
 * reason) so no store is left blank. Returns the registry on success; throws {@link OrphanStoreError}
 * with a logged reason otherwise. Pure — no I/O, no clock.
 */
export function validateRegistry(registry: readonly StoreProvenance[]): readonly StoreProvenance[] {
  const seen = new Set<string>();
  for (const entry of registry) {
    const where = entry.id.length > 0 ? entry.id : entry.artifact;
    if (entry.id.trim().length === 0)
      throw new OrphanStoreError(`orphan store: an entry has a blank id (artifact ${JSON.stringify(entry.artifact)}) — every store must be named (OSS-ADR-0010)`);
    if (seen.has(entry.id)) throw new OrphanStoreError(`orphan store: duplicate id ${JSON.stringify(entry.id)} — store ids must be unique`);
    seen.add(entry.id);
    if (entry.artifact.trim().length === 0)
      throw new OrphanStoreError(`orphan store ${JSON.stringify(where)}: blank artifact — every store must name what it persists`);
    if (entry.textTruth.trim().length === 0)
      throw new OrphanStoreError(`orphan store ${JSON.stringify(where)}: blank text-truth — every store must name the legible text it derives from (or say it IS the truth) (OSS-ADR-0010)`);
    if (entry.writeSites.length === 0)
      throw new OrphanStoreError(`orphan store ${JSON.stringify(where)}: no write-sites declared — a registry row with no durable-write call-site is a dead row`);
    if (entry.class === "projection" && entry.regeneration.trim().length === 0)
      throw new OrphanStoreError(
        `orphan store ${JSON.stringify(where)}: a projection with a blank regeneration path is one you cannot certify — name the byte-for-byte rebuild (OSS-ADR-0010)`,
      );
  }
  return registry;
}

/**
 * Map a scanned durable-write file to its registry entry, or `undefined` if it is an orphan store.
 * The gate uses this to refuse (fail closed) any `src/**` durable-write call-site not covered by the
 * manifest above. `file` is a repo-relative POSIX path (e.g. `src/ledger/hosted.ts`).
 */
export function provenanceForWriteSite(
  file: string,
  registry: readonly StoreProvenance[] = STORE_REGISTRY,
): StoreProvenance | undefined {
  return registry.find((e) => e.writeSites.includes(file));
}