kestrel.markets/src/adapters/lake/objects.ts

A typed, token-efficient language + runtime for agentic trading: agents author bounded plans, the runtime fires them at the tick. CLI + typed library + MCP server.

/**
 * # adapters/lake/objects — byte-level object access, with a BOUNDED local cache
 *
 * The sibling {@link ./index.ts Lake} accessor is Parquet-over-chDB: it answers SQL. This module
 * answers the other half of the same question — *"give me the BYTES of one lake object"* — for the
 * artifacts that are not Parquet and never will be: the benchmark's `.jsonl` tape corpus, the raw
 * vendor pulls (`.csv.zst`, `.json`), the normalized sidecars. Same {@link LakeConfig}, same
 * env, same doctrine:
 *
 * > **Object storage is the ONLY source of truth. Local disk is a bounded, evictable cache.**
 *
 * That doctrine is not decoration here — it is the fix for a real incident. The 14 GiB tape corpus
 * used to live on the developer's disk under `data/tape-corpus/`, and a single Databento pull filled
 * the disk. The bytes now live on R2; this module is how the benchmark reads them back **without
 * ever reconstituting that 14 GiB pile**: the cache is LRU-evicted to `LAKE_CACHE_MAX_BYTES`, so the
 * working set is bounded by construction no matter how many objects are touched.
 *
 * ## Why synchronous
 * Every corpus loader in the benchmark is a pure, synchronous function of on-disk bytes
 * (`loadRealFomcTape`, `realTapePathFor`, the `.gen.ts` converters) and is called at module scope to
 * decide whether a suite runs or SKIPs. Threading `await` through all of that would be a large,
 * risk-bearing refactor of code whose determinism the grade path depends on. So {@link ObjectStore}
 * is **sync**: the network fetch is a `spawnSync` of `curl` with SigV4 (`--aws-sigv4`), creds fed on
 * **stdin** (never argv — they would otherwise be visible in `ps`). No new dependency: `curl` is
 * already the transport the pull scripts use, and it is the only sync HTTP the runtime has.
 *
 * ## The integrity contract (unweakened)
 * A caller that knows an object's committed `sha256` pin passes it. The fetched body is hashed
 * **before** it is committed to the cache; a mismatch is deleted and thrown ({@link LakeReadError}) —
 * a corrupt or drifted object can never enter the cache, and can never be silently served. This is
 * *additional* to (never a replacement for) the pin check the corpus loaders already perform on the
 * bytes they read: the tape is verified at fetch AND at load.
 *
 * ## Absence is a SKIP, corruption is a THROW
 * A genuinely absent object (HTTP 404), an unreachable store (no creds, no network, no `curl`), or
 * an explicitly `offline` store all resolve to `null` — the corpus-dependent suites then SKIP
 * visibly, exactly as they did when the corpus was simply missing from disk. Only a **sha mismatch**
 * throws. Unreachability is reported once on `stderr` and retained on {@link ObjectStore.lastError}
 * so a mis-set credential is legible rather than a silent universal skip.
 *
 * ## Never hangs
 * `curl` is bounded by `--connect-timeout` and `--max-time`, and `spawnSync` carries its own hard
 * `timeout` on top. (Note for the reader who reaches for the shell: **macOS has no `timeout`
 * binary** — the `coreutils` one is `gtimeout`. Wrapping a probe in `timeout` there silently no-ops.
 * The bounds must come from `curl`'s own flags, as they do here.)
 */

import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import {
  closeSync,
  mkdirSync,
  openSync,
  readSync,
  readdirSync,
  renameSync,
  rmSync,
  statSync,
  utimesSync,
} from "node:fs";
import { join } from "node:path";

import { resolveLakeConfig, type LakeConfig, type LakeConfigInput, type LakeS3Config } from "./config.ts";
import { LakeConfigError, LakeReadError, type LakeSource } from "./errors.ts";

/** Default per-object connect budget (seconds). */
const DEFAULT_CONNECT_TIMEOUT_S = 15;
/** Default per-object total transfer budget (seconds) — the corpus holds multi-hundred-MB objects. */
const DEFAULT_MAX_TIME_S = 900;

/** One resolved lake object: a local path the caller may read with plain `fs`. */
export interface LakeObject {
  /** Absolute local path to the object's bytes (a cache segment, or the dev-local file itself). */
  readonly path: string;
  /** Size in bytes. */
  readonly bytes: number;
  /** True when it was served warm — no network touched. */
  readonly cached: boolean;
  /** The canonical source. `"s3"` even when served warm: the cache is transparent, not a source. */
  readonly source: LakeSource;
}

/** Per-fetch options. */
export interface EnsureOptions {
  /**
   * The object's committed sha256 pin. When given, a fetched body is verified against it BEFORE it
   * is committed to the cache; a mismatch throws {@link LakeReadError} and caches nothing.
   */
  readonly sha256?: string;
}

/** Construction options: the whole {@link LakeConfigInput} surface, plus an offline latch. */
export interface ObjectStoreOptions extends LakeConfigInput {
  /** Never touch the network: serve only the dev-local root and an already-warm cache. */
  readonly offline?: boolean;
  /** Per-object connect budget in seconds (default {@link DEFAULT_CONNECT_TIMEOUT_S}). */
  readonly connectTimeoutS?: number;
  /** Per-object total transfer budget in seconds (default {@link DEFAULT_MAX_TIME_S}). */
  readonly maxTimeS?: number;
}

/** The byte-level object accessor over the lake. */
export interface ObjectStore {
  /** The resolved lake configuration (holds secrets in memory; never serialize it). */
  readonly config: LakeConfig;
  /** Where cache segments live (`<cacheDir>/objects`). */
  readonly cacheRoot: string;
  /** False when constructed `offline`, or when no S3 remote is configured. */
  readonly online: boolean;
  /** The last unreachability error (bad creds, network, missing `curl`), if any. Never a 404. */
  readonly lastError: Error | undefined;

  /**
   * Resolve one lake-relative key to local bytes: dev-local root → warm cache → fetch from object
   * storage. Returns `null` when the object is absent or the store is unreachable (the caller
   * SKIPs). Throws {@link LakeReadError} only on a **sha mismatch** — the integrity contract.
   */
  ensure(rel: string, opts?: EnsureOptions): LakeObject | null;

  /** Total bytes currently held in the cache. */
  cacheBytes(): number;

  /**
   * LRU-evict (by access time) until the cache is within `config.cacheMaxBytes`. `protect` is never
   * evicted — it is the object being served right now. Returns the bytes freed.
   */
  evict(protect?: string): number;
}

// ─────────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────────

/** Reject a relative key that is empty or escapes the lake root; return it normalized. */
function checkRel(rel: string): string {
  const r = rel.trim();
  if (r.length === 0) throw new LakeReadError("empty relative key");
  if (r.startsWith("/") || r.split("/").includes("..")) {
    throw new LakeReadError(`relative key must stay inside the lake: ${JSON.stringify(rel)}`, { rel });
  }
  return r;
}

/** `${endpoint}/${bucket}/${prefix}/${rel}`, each key segment percent-encoded, no doubled slashes. */
function objectUrl(s3: LakeS3Config, rel: string): string {
  const key = [s3.prefix, rel]
    .filter((p) => p.length > 0)
    .join("/")
    .split("/")
    .map(encodeURIComponent)
    .join("/");
  return `${s3.endpoint}/${encodeURIComponent(s3.bucket)}/${key}`;
}

const isFile = (abs: string): boolean => {
  try {
    return statSync(abs).isFile();
  } catch {
    return false;
  }
};

const sizeOf = (abs: string): number => {
  try {
    return statSync(abs).size;
  } catch {
    return 0;
  }
};

/**
 * sha256 of a FILE, read in bounded chunks — never buffers the whole object. The corpus holds
 * objects in the hundreds of MB (and raw pulls in the GB); hashing must not itself be the thing that
 * exhausts memory.
 */
export function sha256File(abs: string): string {
  const h = createHash("sha256");
  const buf = Buffer.allocUnsafe(1 << 20); // 1 MiB
  const fd = openSync(abs, "r");
  try {
    for (;;) {
      const n = readSync(fd, buf, 0, buf.length, null);
      if (n <= 0) break;
      h.update(buf.subarray(0, n));
    }
  } finally {
    closeSync(fd);
  }
  return h.digest("hex");
}

/** A stable, secret-free, collision-free cache-segment name for an object URL. */
function segmentName(url: string, rel: string): string {
  const digest = createHash("sha256").update(url).digest("hex").slice(0, 40);
  // Keep a legible suffix so a human can see what a cache file IS, without letting the key drive the path.
  const leaf = (rel.split("/").pop() ?? "object").replace(/[^A-Za-z0-9._-]/g, "_").slice(-64);
  return `${digest}-${leaf}`;
}

/** curl config-file syntax is quote-delimited; a credential carrying `"`/newline could break out. */
function assertCredSafe(value: string, what: string): void {
  if (/["\r\n\\]/.test(value)) {
    throw new LakeConfigError(`${what} contains a character that cannot be passed to curl safely`);
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// Factory
// ─────────────────────────────────────────────────────────────────────────────

/** {@link resolveLakeConfig} with the rclone remote suppressed — for when that remote won't resolve. */
function withoutRcloneRemote(input: ObjectStoreOptions): LakeConfig {
  const env = input.env ?? process.env;
  return resolveLakeConfig({ ...input, rcloneRemote: "", env: { ...env, LAKE_RCLONE_REMOTE: "" } });
}

/**
 * Resolve the config for an object store **without the rclone lookup being able to throw**.
 *
 * `resolveLakeConfig` is fail-closed for the SQL accessor: it throws when a named rclone remote is
 * unreadable. That is right for `lake()`, and wrong here. Resolving a remote READS A CREDENTIAL
 * FILE, and this module's contract is that no creds ⇒ unreachable ⇒ `null` ⇒ the caller SKIPs — so
 * the throw fired from inside probes whose only documented answers are an object or `null`, and a
 * machine with no `~/.config/rclone/rclone.conf` (i.e. CI) could never reach its own skip path.
 *
 * A remote that DOES resolve is honoured in full, so a developer keeps their warm cache (the cache
 * segment name is derived from the endpoint). One that does not is reported, not swallowed: the
 * caller re-surfaces it through {@link ObjectStore.lastError} + a one-shot stderr line, so a
 * mis-set credential stays legible rather than becoming a silent universal skip.
 */
function resolveForStore(input: ObjectStoreOptions): {
  config: LakeConfig;
  configError: LakeConfigError | undefined;
} {
  try {
    return { config: resolveLakeConfig(input), configError: undefined };
  } catch (err) {
    if (!(err instanceof LakeConfigError)) throw err;
    return { config: withoutRcloneRemote(input), configError: err };
  }
}

/** Construct an {@link ObjectStore} from an optional override (falls back to the environment). */
export function objectStore(input: ObjectStoreOptions = {}): ObjectStore {
  // The offline latch is legible BEFORE anything that could fail: `offline` promises "no network, no
  // credential", so an absent credential file must be a non-event rather than the first thing to
  // throw. Resolution below cannot throw at all ({@link resolveForStore}) — that is what lets this
  // latch do what it documents and force the skip path deterministically.
  const offline = input.offline === true;
  const { config, configError } = resolveForStore(input);
  const env = input.env ?? process.env;
  const cacheRoot = join(config.cacheDir, "objects");
  const connectTimeoutS = input.connectTimeoutS ?? DEFAULT_CONNECT_TIMEOUT_S;
  const maxTimeS = input.maxTimeS ?? DEFAULT_MAX_TIME_S;
  const region = env.LAKE_S3_REGION !== undefined && env.LAKE_S3_REGION !== "" ? env.LAKE_S3_REGION : "auto";

  const online = !offline && config.s3 !== undefined;
  let lastError: Error | undefined;
  let warned = false;

  /** Report unreachability ONCE, loudly enough that a mis-set credential is not a silent skip. */
  function unreachable(err: Error): null {
    lastError = err;
    if (!warned) {
      warned = true;
      console.error(`[lake] object storage unreachable — corpus reads will SKIP: ${err.message}`);
    }
    return null;
  }

  // An rclone remote that would not resolve, and nothing else supplied a remote: this store is
  // unreachable. Say so ONCE, now — the alternative is a suite that skips universally and silently
  // because of a typo'd credential path. Silent when `offline`: no credential was ever wanted.
  if (configError !== undefined && !offline && config.s3 === undefined) unreachable(configError);

  function listSegments(): { path: string; size: number; atime: number }[] {
    try {
      return (readdirSync(cacheRoot) as string[])
        .filter((f) => !f.endsWith(".tmp"))
        .map((f) => {
          const p = join(cacheRoot, f);
          const st = statSync(p);
          return { path: p, size: st.size, atime: st.atimeMs };
        })
        .filter((e) => e.size >= 0);
    } catch {
      return [];
    }
  }

  function cacheBytes(): number {
    return listSegments().reduce((a, e) => a + e.size, 0);
  }

  /**
   * LRU-evict to the cap. `protect` (the object being served right now) is never evicted, so the
   * cache is bounded by `cacheMaxBytes` PLUS at most that one in-flight object — the bound holds
   * even for an object larger than the whole cap, which is served but never accumulates.
   */
  function evict(protect?: string): number {
    const cap = config.cacheMaxBytes;
    const entries = listSegments();
    let total = entries.reduce((a, e) => a + e.size, 0);
    if (total <= cap) return 0;
    entries.sort((a, b) => a.atime - b.atime); // oldest-accessed evicted first
    let freed = 0;
    for (const e of entries) {
      if (total <= cap) break;
      if (protect !== undefined && e.path === protect) continue;
      try {
        rmSync(e.path, { force: true });
        total -= e.size;
        freed += e.size;
      } catch {
        /* a racing reader may hold it; skip */
      }
    }
    return freed;
  }

  /** Fetch `url` to `dest` with SigV4. Returns null (unreachable/absent) or throws nothing. */
  function fetchTo(s3: LakeS3Config, url: string, dest: string, rel: string): "ok" | "absent" | "unreachable" {
    assertCredSafe(s3.accessKeyId, "access key id");
    assertCredSafe(s3.secretAccessKey, "secret access key");

    const res = spawnSync(
      "curl",
      [
        "--silent",
        "--show-error",
        "--fail", // any HTTP >= 400 is a failure, not a 0-byte "success"
        "--location",
        "--config",
        "-", // credentials arrive on STDIN, never on argv (argv is world-readable in `ps`)
        "--aws-sigv4",
        `aws:amz:${region}:s3`,
        "--connect-timeout",
        String(connectTimeoutS),
        "--max-time",
        String(maxTimeS),
        "--retry",
        "2",
        "--retry-connrefused",
        "--write-out",
        "%{http_code}",
        "--output",
        dest,
        url,
      ],
      {
        input: `user = "${s3.accessKeyId}:${s3.secretAccessKey}"\n`,
        encoding: "utf8",
        timeout: (maxTimeS + 30) * 1000, // a hard backstop over curl's own budget — never hang
        maxBuffer: 1 << 20,
      },
    );

    if (res.error !== undefined && res.error !== null) {
      rmSync(dest, { force: true });
      unreachable(new LakeReadError(`could not run curl to reach object storage: ${res.error.message}`, { rel, source: "s3", cause: res.error }));
      return "unreachable";
    }
    if (res.status === 0) return "ok";

    rmSync(dest, { force: true });
    const code = (res.stdout ?? "").trim();
    if (code === "404") return "absent"; // a legitimately absent object → the caller SKIPs
    const detail = (res.stderr ?? "").trim();
    unreachable(
      new LakeReadError(
        `object storage read failed (curl exit ${String(res.status)}, http ${code || "n/a"}): ${rel}${detail === "" ? "" : ` — ${detail}`}`,
        { rel, source: "s3" },
      ),
    );
    return "unreachable";
  }

  function ensure(rel: string, opts: EnsureOptions = {}): LakeObject | null {
    const key = checkRel(rel);

    // 1. The explicit dev/fixture root (creds-free tests/CI, and the warm local corpus).
    if (config.devLocalRoot !== undefined) {
      const abs = join(config.devLocalRoot, key);
      if (isFile(abs)) return { path: abs, bytes: sizeOf(abs), cached: true, source: "dev-local" };
    }

    if (config.s3 === undefined) return null; // nothing to reach; SKIP
    const s3 = config.s3;
    const url = objectUrl(s3, key);
    const segment = join(cacheRoot, segmentName(url, key));

    // 2. A warm cache segment — no network, and no re-hash: it was verified when it was written.
    //    (A pinned caller re-verifies on read regardless, so on-disk rot is still caught loudly.)
    if (isFile(segment)) {
      try {
        const now = new Date();
        utimesSync(segment, now, now); // bump recency for the LRU
      } catch {
        /* best-effort */
      }
      return { path: segment, bytes: sizeOf(segment), cached: true, source: "s3" };
    }

    if (!online) return null; // offline latch, cache missed → SKIP

    // 3. Cold: fetch once from object storage, verify, commit, then bring the cache back under cap.
    try {
      mkdirSync(cacheRoot, { recursive: true });
    } catch (cause) {
      return unreachable(new LakeReadError(`cache dir unusable: ${cacheRoot}`, { rel: key, source: "s3", cause }));
    }

    const tmp = `${segment}.${String(process.pid)}.${String(Date.now())}.tmp`;
    const outcome = fetchTo(s3, url, tmp, key);
    if (outcome !== "ok") return null;

    // The integrity contract: verify BEFORE the object can ever be served from the cache.
    const pin = opts.sha256;
    if (pin !== undefined && pin !== "") {
      const got = sha256File(tmp);
      if (got !== pin) {
        rmSync(tmp, { force: true });
        throw new LakeReadError(
          `lake object ${key} sha256 ${got} != pinned ${pin} — the object drifted from its committed pin; ` +
            "NOT cached. Re-vet the corpus before re-pinning (a changed object is a NEW object).",
          { rel: key, source: "s3" },
        );
      }
    }

    try {
      renameSync(tmp, segment);
    } catch (cause) {
      rmSync(tmp, { force: true });
      throw new LakeReadError(`could not commit cache segment for ${key}`, { rel: key, source: "s3", cause });
    }

    evict(segment);
    return { path: segment, bytes: sizeOf(segment), cached: false, source: "s3" };
  }

  return {
    config,
    cacheRoot,
    online,
    get lastError(): Error | undefined {
      return lastError;
    },
    ensure,
    cacheBytes,
    evict,
  };
}