UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

961 lines (889 loc) • 36.2 kB
import { chmodSync, closeSync, constants, copyFileSync, existsSync, fsyncSync, linkSync, lstatSync, mkdirSync, openSync, opendirSync, readFileSync, unlinkSync, writeSync, type Stats, } from "node:fs"; import { createHash, randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; import { getConfigDir } from "../config"; import { forgetEphemeralSecretPath, forgetHardenedSecretPath, hardenSecretDir, hardenSecretDirAsync, hardenSecretPath, hardenSecretPathAsync, windowsSecretAclApplies, } from "../lib/windows-secret-acl"; import { isValidProviderContinuationOwner } from "./provider-continuation"; import type { OcxProviderContinuationState } from "../types"; export const RESPONSE_SPILL_VERSION = 1; export const RESPONSE_SPILL_DIR_NAME = "responses-state-spill"; export const RESPONSE_SPILL_ORPHAN_GRACE_MS = 15 * 60_000; export const RESPONSE_SPILL_SCAN_MAX = 4_096; export const RESPONSE_SPILL_CLEANUP_MAX = 512; // The liveness-tick sweep runs synchronously on the serving event loop every // 60 s, so it gets a tighter bound than the startup pass that blocks one call. export const PERIODIC_SPILL_SWEEP_OPTS = { scanMax: 512, cleanupMax: 64, deadlineMs: 25, } as const; /** Open directory iterator the next liveness-tick sweep resumes from; null * after a full pass, an iteration error, or a directory change. Holding the * real iterator (instead of an offset re-skipped from a fresh opendir) keeps * each tick's cost bounded by its own budget, so a slow enumeration cannot * pin the cursor at the same prefix forever. */ let periodicSweepCursor: { dir: string; scan: SpillDirScan } | null = null; const RESPONSE_SPILL_PUBLISH_RETRIES = 64; const OWNED_SPILL_NAME = /^([A-Za-z0-9._-]{1,80})\.([0-9a-f]{12})\.([0-9a-f]{24})\.(\d+)\.(\d+)\.spill\.json$/; const OWNED_SPILL_TEMP_NAME = /^\.response-spill\.[0-9]+\.[0-9a-f]{16}\.tmp$/; export interface ResponseSpillPayload { version: 1; responseId: string; createdAt: number; clientThreadId?: string; items: unknown[]; /** * Index in `items` where the provider output begins, used by replay-overlap detection * in state.ts. Optional so a payload written before this field still loads (it simply * never authorizes a skip). * * Compatibility is FORWARD-ONLY: `validPayload` is a strict key allowlist, so a build * predating this field rejects a payload carrying it as corrupt rather than ignoring * it. Rolling back across this change invalidates spilled entries, which degrades to a * replay miss — an already-handled path — not to corrupted live state. */ providerOutputStart?: number; providers?: OcxProviderContinuationState; } export interface ResponseSpillRef { version: 1; fileName: string; digest: string; payloadBytes: number; } /** * Hard ceiling for one spill payload, enforced BOTH at direct-spill admission * (state.ts refuses to durably retain a larger candidate) and at replay read * (below). 256 MiB keeps the replay transient under the process-wide * APP_OWNED_WORST_CASE_PINNED_BYTES ceiling (512 MiB); without an admission * ceiling the read ceiling would strand write-only spills on disk. */ export const MAX_RESPONSE_SPILL_PAYLOAD_BYTES = 256 * 1024 * 1024; let spillPayloadCapOverride: number | null = null; /** Test-only: lower/restore the single-spill payload ceiling (null restores). */ export function setResponseSpillPayloadCapForTests(bytes: number | null): void { spillPayloadCapOverride = bytes; } export function responseSpillPayloadCap(): number { return spillPayloadCapOverride ?? MAX_RESPONSE_SPILL_PAYLOAD_BYTES; } export type ResponseSpillReadResult = | { ok: true; payload: ResponseSpillPayload } | { ok: false; reason: "missing" | "corrupt" | "too_large" }; export interface ResponseSpillCleanupResult { scanned: number; removed: number; failed: number; bytesRemoved: number; /** The scan reached the end of the directory. */ exhausted?: boolean; } export interface ResponseSpillIoForTest { write?: (fd: number, bytes: Uint8Array) => void; fsync?: (fd: number) => void; /** Directory-handle fsync seam for `fsyncDirectoryBestEffort` only. */ fsyncDir?: (fd: number) => void; /** Directory-handle open seam for `fsyncDirectoryBestEffort` only. */ openDir?: (dir: string) => number; /** Directory-handle close seam for `fsyncDirectoryBestEffort` only. */ closeDir?: (fd: number) => void; link?: (tempPath: string, destinationPath: string) => void; copyFileExcl?: (tempPath: string, destinationPath: string) => void; unlink?: (path: string) => void; /** Orphan-GC enumeration seam: returns the next directory entry NAME or * null for end-of-directory. Lets tests prove the scan cap binds without * materializing thousands of real files. */ readdirEntry?: () => string | null; record?: (event: "write" | "fsync" | "close" | "harden" | "publish" | "dir-fsync" | "stub-swap") => void; } let spillIoForTest: ResponseSpillIoForTest | null = null; let spillGeneration = 0; let spillNowOverride: (() => number) | null = null; interface ResponseSpillWriteOptions { retryTimedOutOnce?: boolean; /** Total caller-owned ACL budget shared by every harden in this publication. */ aclBudgetMs?: number; publicationControl?: ResponseSpillPublicationControl; } export interface ResponseSpillPublicationControl { superseded: boolean; tempPath: string | null; destinationPath: string | null; } interface SpillAclBudget { deadline: number; perCallMs: number; } export function createResponseSpillPublicationControl(): ResponseSpillPublicationControl { return { superseded: false, tempPath: null, destinationPath: null }; } export function markResponseSpillPublicationSuperseded(control: ResponseSpillPublicationControl): void { control.superseded = true; } export function setSpillIoForTest(io: ResponseSpillIoForTest | null): void { spillIoForTest = io; } /** Test-only: inject the spill deadline clock. */ export function setResponseSpillNowForTests(now: (() => number) | null): void { spillNowOverride = now; } function spillNow(): number { return spillNowOverride?.() ?? Date.now(); } /** * The spill deadline clock, shared with the shutdown drain in `state/spill-queue.ts`. * * Every deadline the shutdown path enforces has to read the same clock the work it * budgets reads. When the drain measured its reserve on `Date.now()` while the ACL * harden it was budgeting ran on this injected clock, a test could freeze the clock, * believe it had removed wall time from the case, and still lose an 80 ms reserve to * real elapsed time on a loaded runner — which is what turned * `shutdown fallback prices the job-owned superseded generation before publishing` * red on macOS 2/2 in run 35137850114 while the assertion it was written for never ran. * * Production is unchanged: with no override installed this is `Date.now()`. */ export function responseSpillNow(): number { return spillNow(); } function record(event: "write" | "fsync" | "close" | "harden" | "publish" | "dir-fsync" | "stub-swap"): void { spillIoForTest?.record?.(event); } function fsyncDirectoryBestEffort(dir: string): void { let fd: number | null = null; // Record the durability attempt before opening the directory: Windows cannot // open/fsync directory handles this way, but callers still cross this seam. record("dir-fsync"); try { fd = spillIoForTest?.openDir ? spillIoForTest.openDir(dir) : openSync(dir, "r"); try { if (spillIoForTest?.fsyncDir) spillIoForTest.fsyncDir(fd); else if (spillIoForTest?.fsync) spillIoForTest.fsync(fd); else fsyncSync(fd); } catch { // Windows and some filesystems do not support fsync on directory handles. } } catch { // Directory missing or unreadable — nothing further to sync. } finally { if (fd !== null) { try { if (spillIoForTest?.closeDir) spillIoForTest.closeDir(fd); else closeSync(fd); } catch { /* best effort */ } } } } export function noteStubSwapForTest(): void { record("stub-swap"); } export function responseSpillDirectory(dir = getConfigDir()): string { return join(dir, RESPONSE_SPILL_DIR_NAME); } function sha256(value: string | Uint8Array): string { return createHash("sha256").update(value).digest("hex"); } function sanitizeResponseId(responseId: string): string { const visible = responseId .normalize("NFC") .replace(/[^A-Za-z0-9._-]/g, "_") .replace(/_+/g, "_") .slice(0, 80); return visible || "response"; } function isErrno(error: unknown, code: string): boolean { return !!error && typeof error === "object" && (error as NodeJS.ErrnoException).code === code; } function canUseExclusiveCopyFallback(error: unknown): boolean { // Same platform seam as harden(): a fixture pinned to the POSIX lane on a Windows host must // see a link failure as a failure, not as a cue to copy. return windowsSecretAclApplies() || ["EPERM", "EACCES", "ENOSYS", "ENOTSUP", "EOPNOTSUPP", "EXDEV"] .some(code => isErrno(error, code)); } function spillAclBudget(totalMs: number | undefined): SpillAclBudget | undefined { if (totalMs === undefined) return undefined; const bounded = Math.max(1, Math.floor(totalMs)); return { deadline: spillNow() + bounded, perCallMs: Math.max(1, Math.floor(bounded / 2)) }; } function nextSpillHardenDeadlineMs(budget: SpillAclBudget | undefined): number | undefined { if (!budget) return undefined; const remaining = budget.deadline - spillNow(); if (remaining <= 0) { throw Object.assign(new Error("Response spill ACL budget exhausted"), { code: "ETIMEDOUT" }); } return Math.min(budget.perCallMs, remaining); } function harden(path: string, mode: number, budget?: SpillAclBudget): void { // One predicate for both lanes: the test seam that forces a platform must reach the // sync harden too, or a fixture pinned to "linux" on a Windows host still spawns icacls. const aclApplies = windowsSecretAclApplies(); try { chmodSync(path, mode); } catch { if (!aclApplies) throw new Error("Response spill permission hardening failed"); } if (aclApplies) { const deadlineMs = nextSpillHardenDeadlineMs(budget); const options = { required: true, ...(deadlineMs !== undefined ? { deadlineMs } : {}), }; const result = mode === 0o700 ? hardenSecretDir(path, options) : hardenSecretPath(path, options); if (!result.ok) throw new Error("Response spill permission hardening failed"); } } async function hardenAsync( path: string, mode: number, budget: SpillAclBudget, retryTimedOutOnce = false, ): Promise<void> { try { chmodSync(path, mode); } catch { if (!windowsSecretAclApplies()) throw new Error("Response spill permission hardening failed"); } if (windowsSecretAclApplies()) { const deadlineMs = nextSpillHardenDeadlineMs(budget); const options = { required: true, retryTimedOutOnce, ...(deadlineMs !== undefined ? { deadlineMs } : {}), }; const result = mode === 0o700 ? await hardenSecretDirAsync(path, options) : await hardenSecretPathAsync(path, options); if (!result.ok) throw new Error("Response spill permission hardening failed"); } } function writeAll(fd: number, bytes: Uint8Array): void { if (spillIoForTest?.write) spillIoForTest.write(fd, bytes); else { let offset = 0; while (offset < bytes.byteLength) offset += writeSync(fd, bytes, offset, bytes.byteLength - offset); } record("write"); } function fsyncFile(fd: number): void { if (spillIoForTest?.fsync) spillIoForTest.fsync(fd); else fsyncSync(fd); record("fsync"); } function closeFile(fd: number): void { closeSync(fd); record("close"); } function unlink(path: string, ephemeral = false): void { try { if (spillIoForTest?.unlink) spillIoForTest.unlink(path); else unlinkSync(path); // Ephemeral release only for publish temps; stable spill files keep their // destination-keyed timeout memos (anti-restall) and drop just the // success memo for the now-deleted file. if (ephemeral) forgetEphemeralSecretPath(path); else forgetHardenedSecretPath(path); } catch (error) { if (isErrno(error, "ENOENT")) { if (ephemeral) forgetEphemeralSecretPath(path); else forgetHardenedSecretPath(path); } throw error; } } function unlinkEphemeral(path: string): void { unlink(path, true); } function supersededPublicationError(): NodeJS.ErrnoException { return Object.assign(new Error("Response spill publication superseded"), { code: "ECANCELED" }); } function throwIfPublicationSuperseded(control: ResponseSpillPublicationControl | undefined): void { if (control?.superseded) throw supersededPublicationError(); } function clearOwnedPath( control: ResponseSpillPublicationControl, key: "tempPath" | "destinationPath", ephemeral: boolean, ): unknown { const path = control[key]; if (!path) return null; try { if (ephemeral) unlinkEphemeral(path); else unlink(path); control[key] = null; return null; } catch (error) { if (isErrno(error, "ENOENT")) { control[key] = null; return null; } return error; } } /** Claim and remove every path still owned by an abandoned async publication. */ export function cleanupSupersededResponseSpillPublication( control: ResponseSpillPublicationControl, ): Error | null { control.superseded = true; const ownedDir = control.destinationPath ? dirname(control.destinationPath) : control.tempPath ? dirname(control.tempPath) : null; const destinationError = clearOwnedPath(control, "destinationPath", false); const tempError = clearOwnedPath(control, "tempPath", true); if (ownedDir) fsyncDirectoryBestEffort(ownedDir); const cleanupError = destinationError ?? tempError; return cleanupError ? responseSpillWriteError(cleanupError) : null; } function publishNoReplace( tempPath: string, destinationPath: string, budget?: SpillAclBudget, ): void { try { if (spillIoForTest?.link) spillIoForTest.link(tempPath, destinationPath); else linkSync(tempPath, destinationPath); } catch (error) { if (isErrno(error, "EEXIST")) throw error; if (!canUseExclusiveCopyFallback(error)) throw error; let copied = false; try { if (spillIoForTest?.copyFileExcl) spillIoForTest.copyFileExcl(tempPath, destinationPath); else copyFileSync(tempPath, destinationPath, constants.COPYFILE_EXCL); copied = true; harden(destinationPath, 0o600, budget); // "r+": a read-only handle cannot be fsynced on Windows (EPERM). const copyFd = openSync(destinationPath, "r+"); try { if (spillIoForTest?.fsync) spillIoForTest.fsync(copyFd); else fsyncSync(copyFd); } finally { closeSync(copyFd); } } catch (copyError) { if (copied) { try { unlink(destinationPath); } catch { /* startup GC reclaims an incomplete publication */ } } throw copyError; } } record("publish"); } async function publishNoReplaceAsync( tempPath: string, destinationPath: string, budget: SpillAclBudget, retryTimedOutOnce: boolean, publicationControl?: ResponseSpillPublicationControl, ): Promise<void> { throwIfPublicationSuperseded(publicationControl); try { if (spillIoForTest?.link) spillIoForTest.link(tempPath, destinationPath); else linkSync(tempPath, destinationPath); } catch (error) { if (isErrno(error, "EEXIST")) throw error; if (!canUseExclusiveCopyFallback(error)) throw error; let copied = false; try { if (spillIoForTest?.copyFileExcl) spillIoForTest.copyFileExcl(tempPath, destinationPath); else copyFileSync(tempPath, destinationPath, constants.COPYFILE_EXCL); copied = true; await hardenAsync(destinationPath, 0o600, budget, retryTimedOutOnce); throwIfPublicationSuperseded(publicationControl); const copyFd = openSync(destinationPath, "r+"); try { if (spillIoForTest?.fsync) spillIoForTest.fsync(copyFd); else fsyncSync(copyFd); } finally { closeSync(copyFd); } } catch (copyError) { if (copied) { try { unlink(destinationPath); } catch { /* startup GC reclaims an incomplete publication */ } } throw copyError; } } throwIfPublicationSuperseded(publicationControl); record("publish"); } function serializedSpill( responseId: string, state: Omit<ResponseSpillPayload, "version" | "responseId">, ): { bytes: Buffer; digest: string; idDigest: string; contentDigest: string; } { const payload: ResponseSpillPayload = { version: 1, responseId, createdAt: state.createdAt, ...(state.clientThreadId ? { clientThreadId: state.clientThreadId } : {}), items: state.items, ...(state.providerOutputStart !== undefined ? { providerOutputStart: state.providerOutputStart } : {}), ...(state.providers ? { providers: state.providers } : {}), }; const serialized = JSON.stringify(payload); if (serialized === undefined) throw new Error("Response spill serialization failed"); const bytes = Buffer.from(serialized, "utf8"); const digest = sha256(bytes); return { bytes, digest, idDigest: sha256(responseId).slice(0, 12), contentDigest: digest.slice(0, 24), }; } /** * Exact on-disk payload size this spill WOULD occupy, measured before publication. * * Callers that reserve disk against a cap need the real envelope, not the resident * measurement: the resident figure omits the `version` field the published payload * carries, so pricing an admission by it undercounts and lets a request that sits exactly * at the cap still exceed it. Shares `serializedSpill` rather than describing it, so the * two cannot drift. */ export function prospectiveResponseSpillBytes( responseId: string, state: Omit<ResponseSpillPayload, "version" | "responseId">, ): number | null { try { return serializedSpill(responseId, state).bytes.byteLength; } catch { return null; } } function responseSpillWriteError(cause: unknown): NodeJS.ErrnoException { const error = new Error("Response spill write failed", { cause }) as NodeJS.ErrnoException; if (cause && typeof cause === "object" && "code" in cause) { error.code = String((cause as { code?: unknown }).code); } return error; } function validSpillRef(ref: ResponseSpillRef): boolean { return ref.version === 1 && OWNED_SPILL_NAME.test(ref.fileName) && /^[0-9a-f]{64}$/.test(ref.digest) && Number.isSafeInteger(ref.payloadBytes) && ref.payloadBytes >= 0; } function validPayload(value: unknown, responseId: string): value is ResponseSpillPayload { if (!value || typeof value !== "object" || Array.isArray(value)) return false; const payload = value as Record<string, unknown>; const keys = Object.keys(payload); if (keys.some(key => !["version", "responseId", "createdAt", "clientThreadId", "items", "providerOutputStart", "providers"].includes(key))) return false; if (payload.version !== 1 || payload.responseId !== responseId) return false; if (typeof payload.createdAt !== "number" || !Number.isFinite(payload.createdAt)) return false; if (payload.clientThreadId !== undefined && (typeof payload.clientThreadId !== "string" || payload.clientThreadId.trim().length === 0)) return false; if (!Array.isArray(payload.items)) return false; // A malformed boundary must degrade to "never skip", never to a bad index: reject the // payload outright so materialization treats it as corrupt rather than trusting it. if (payload.providerOutputStart !== undefined) { const anchor = payload.providerOutputStart; if (typeof anchor !== "number" || !Number.isSafeInteger(anchor) || anchor < 0 || anchor > payload.items.length) return false; } if (payload.providers !== undefined) { if (!payload.providers || typeof payload.providers !== "object" || Array.isArray(payload.providers)) return false; const providers = payload.providers as Record<string, unknown>; if (providers.__ocxOwner !== undefined && !isValidProviderContinuationOwner(providers.__ocxOwner)) return false; for (const [provider, providerState] of Object.entries(providers)) { if (provider === "__ocxOwner") continue; if (!providerState || typeof providerState !== "object" || Array.isArray(providerState)) return false; } } return true; } export function writeResponseSpillDurably( responseId: string, state: Omit<ResponseSpillPayload, "version" | "responseId">, options: ResponseSpillWriteOptions = {}, ): ResponseSpillRef { let tempPath: string | null = null; let fd: number | null = null; try { const aclBudget = spillAclBudget(options.aclBudgetMs); const { bytes, digest, idDigest, contentDigest } = serializedSpill(responseId, state); const dir = responseSpillDirectory(); mkdirSync(dir, { recursive: true, mode: 0o700 }); harden(dir, 0o700, aclBudget); tempPath = join(dir, `.response-spill.${process.pid}.${randomBytes(8).toString("hex")}.tmp`); fd = openSync(tempPath, "wx", 0o600); writeAll(fd, bytes); fsyncFile(fd); closeFile(fd); fd = null; harden(tempPath, 0o600, aclBudget); record("harden"); const publishTempPath = tempPath; for (let attempt = 0; attempt < RESPONSE_SPILL_PUBLISH_RETRIES; attempt++) { spillGeneration += 1; const fileName = `${sanitizeResponseId(responseId)}.${idDigest}.${contentDigest}.${spillGeneration}.${bytes.byteLength}.spill.json`; if (!OWNED_SPILL_NAME.test(fileName)) throw new Error("Response spill name allocation failed"); const destinationPath = join(dir, fileName); try { publishNoReplace(publishTempPath, destinationPath, aclBudget); fsyncDirectoryBestEffort(dir); unlinkEphemeral(publishTempPath); tempPath = null; return { version: 1, fileName, digest, payloadBytes: bytes.byteLength }; } catch (error) { if (isErrno(error, "EEXIST")) continue; throw error; } } throw new Error("Response spill publication retries exhausted"); } catch (cause) { if (fd !== null) { try { closeSync(fd); } catch { /* best effort */ } } if (tempPath) { try { unlinkEphemeral(tempPath); } catch { /* best effort */ } } throw responseSpillWriteError(cause); } } /** * Windows runtime counterpart of `writeResponseSpillDurably`. * * The filesystem publication contract stays identical, but required NTFS ACL subprocesses are * awaited through Bun.spawn instead of Bun.spawnSync. State ownership and serialization remain in * `state.ts`; callers must compare the resident generation again before installing the returned * reference because another response can replace it while ACL hardening is pending. */ export async function writeResponseSpillDurablyAsync( responseId: string, state: Omit<ResponseSpillPayload, "version" | "responseId">, options: ResponseSpillWriteOptions & { aclBudgetMs: number }, ): Promise<ResponseSpillRef> { const publicationControl = options.publicationControl; const aclBudget = spillAclBudget(options.aclBudgetMs); if (!aclBudget) throw new Error("Response spill async ACL budget is required"); let tempPath: string | null = null; let fd: number | null = null; try { throwIfPublicationSuperseded(publicationControl); const { bytes, digest, idDigest, contentDigest } = serializedSpill(responseId, state); const dir = responseSpillDirectory(); mkdirSync(dir, { recursive: true, mode: 0o700 }); await hardenAsync(dir, 0o700, aclBudget, options.retryTimedOutOnce === true); throwIfPublicationSuperseded(publicationControl); tempPath = join(dir, `.response-spill.${process.pid}.${randomBytes(8).toString("hex")}.tmp`); fd = openSync(tempPath, "wx", 0o600); if (publicationControl) publicationControl.tempPath = tempPath; writeAll(fd, bytes); fsyncFile(fd); closeFile(fd); fd = null; await hardenAsync(tempPath, 0o600, aclBudget, options.retryTimedOutOnce === true); throwIfPublicationSuperseded(publicationControl); record("harden"); const publishTempPath = tempPath; for (let attempt = 0; attempt < RESPONSE_SPILL_PUBLISH_RETRIES; attempt++) { throwIfPublicationSuperseded(publicationControl); spillGeneration += 1; const fileName = `${sanitizeResponseId(responseId)}.${idDigest}.${contentDigest}.${spillGeneration}.${bytes.byteLength}.spill.json`; if (!OWNED_SPILL_NAME.test(fileName)) throw new Error("Response spill name allocation failed"); const destinationPath = join(dir, fileName); if (publicationControl) publicationControl.destinationPath = destinationPath; try { throwIfPublicationSuperseded(publicationControl); await publishNoReplaceAsync( publishTempPath, destinationPath, aclBudget, options.retryTimedOutOnce === true, publicationControl, ); throwIfPublicationSuperseded(publicationControl); fsyncDirectoryBestEffort(dir); unlinkEphemeral(publishTempPath); tempPath = null; if (publicationControl) { publicationControl.tempPath = null; publicationControl.destinationPath = null; } return { version: 1, fileName, digest, payloadBytes: bytes.byteLength }; } catch (error) { if (publicationControl?.superseded) throw error; if (publicationControl) publicationControl.destinationPath = null; if (isErrno(error, "EEXIST")) continue; throw error; } } throw new Error("Response spill publication retries exhausted"); } catch (cause) { if (fd !== null) { try { closeSync(fd); } catch { /* best effort */ } } if (tempPath) { try { unlinkEphemeral(tempPath); if (publicationControl?.tempPath === tempPath) publicationControl.tempPath = null; } catch (error) { if (isErrno(error, "ENOENT") && publicationControl?.tempPath === tempPath) { publicationControl.tempPath = null; } } } if (publicationControl) { const destinationPath = publicationControl.destinationPath; if (destinationPath) { try { unlink(destinationPath); publicationControl.destinationPath = null; } catch (error) { if (isErrno(error, "ENOENT")) publicationControl.destinationPath = null; } } } throw responseSpillWriteError(cause); } } export function readResponseSpill(responseId: string, ref: ResponseSpillRef): ResponseSpillReadResult { if (!validSpillRef(ref)) return { ok: false, reason: "corrupt" }; // Refuse before any read/parse: an oversized declared payload would otherwise // materialize an unbounded transient (readFileSync + utf8 + JSON.parse). if (ref.payloadBytes > responseSpillPayloadCap()) return { ok: false, reason: "too_large" }; const match = OWNED_SPILL_NAME.exec(ref.fileName); if (!match || match[2] !== sha256(responseId).slice(0, 12) || match[3] !== ref.digest.slice(0, 24) || !Number.isSafeInteger(Number(match[4])) || Number(match[4]) <= 0 || Number(match[5]) !== ref.payloadBytes) return { ok: false, reason: "corrupt" }; const path = join(responseSpillDirectory(), ref.fileName); let bytes: Buffer; try { const stat = lstatSync(path); if (!stat.isFile() || stat.isSymbolicLink()) return { ok: false, reason: "corrupt" }; if (stat.size !== ref.payloadBytes) return { ok: false, reason: "corrupt" }; bytes = readFileSync(path); } catch (error) { return { ok: false, reason: isErrno(error, "ENOENT") ? "missing" : "corrupt" }; } if (bytes.byteLength !== ref.payloadBytes || sha256(bytes) !== ref.digest) return { ok: false, reason: "corrupt" }; try { const payload = JSON.parse(bytes.toString("utf8")) as unknown; return validPayload(payload, responseId) ? { ok: true, payload } : { ok: false, reason: "corrupt" }; } catch { return { ok: false, reason: "corrupt" }; } } export function deleteResponseSpill(ref: ResponseSpillRef): void { if (!validSpillRef(ref)) return; const dir = responseSpillDirectory(); try { unlink(join(dir, ref.fileName)); fsyncDirectoryBestEffort(dir); } catch { /* best effort */ } } type SpillDirNameKind = "spill" | "temp"; /** Directory-iteration plumbing shared by reclaim and the dry-run inspector. * ONE loop serves both the real directory handle and the injected test seam * (review C2-2: two duplicated loops let the test prove only its own copy). * Callers must request the next name strictly AFTER their scan-cap check, so * entry SCAN_MAX+1 is never requested from either source. */ interface SpillDirScan { nextName: () => string | null; close: () => void } function openSpillDirScan(dir: string): SpillDirScan | null { let handle: ReturnType<typeof opendirSync> | null = null; const injected = spillIoForTest?.readdirEntry; if (!injected) { try { handle = opendirSync(dir); } catch { return null; } } return { nextName: () => { if (injected) return injected(); const entry = handle!.readSync(); return entry ? entry.name : null; }, close: () => { try { handle?.closeSync(); } catch { /* best effort */ } }, }; } /** Orphan predicate, name half: a spill-shaped name nobody references. Shared by * reclaim and the dry-run inspector so the report and the unlink cannot disagree. */ function orphanSpillNameKind(name: string, referencedFileNames: ReadonlySet<string>): SpillDirNameKind | null { if (referencedFileNames.has(name)) return null; if (OWNED_SPILL_NAME.test(name)) return "spill"; if (OWNED_SPILL_TEMP_NAME.test(name)) return "temp"; return null; } /** Orphan predicate, stat half: a regular file (never a symlink) past the grace window. */ function spillEntryPastGrace(stat: Stats, graceMs: number): boolean { return stat.isFile() && !stat.isSymbolicLink() && Date.now() - stat.mtimeMs >= graceMs; } export interface ResponseSpillDirInspection { scanned: number; /** The walk stopped at RESPONSE_SPILL_SCAN_MAX, so every count describes a * prefix of the directory and the real totals are higher. */ truncated: boolean; /** Every regular file present, spill-shaped or not — the honest disk total. */ files: number; bytes: number; /** Regular files named in the caller's referenced set. */ ownedFiles: number; ownedBytes: number; /** Spill/temp-shaped regular files unreferenced and past the orphan grace. */ orphanFiles: number; orphanBytes: number; } /** * Dry-run counterpart of recoverOrphanedResponseSpills: same directory walk and * the same orphan predicate, but it counts and never unlinks. It stats every * entry (bounded by RESPONSE_SPILL_SCAN_MAX) because `bytes` is the disk total * an operator sizes the directory by — reclaim only stats orphan candidates. */ export function inspectResponseSpillDir( referencedFileNames: ReadonlySet<string>, dir = responseSpillDirectory(), opts?: { graceMs?: number }, ): ResponseSpillDirInspection { const result: ResponseSpillDirInspection = { scanned: 0, truncated: false, files: 0, bytes: 0, ownedFiles: 0, ownedBytes: 0, orphanFiles: 0, orphanBytes: 0, }; const graceMs = opts?.graceMs ?? RESPONSE_SPILL_ORPHAN_GRACE_MS; const scan = openSpillDirScan(dir); if (!scan) return result; try { while (result.scanned < RESPONSE_SPILL_SCAN_MAX) { const name = scan.nextName(); if (name === null) break; result.scanned += 1; const path = join(dir, name); let stat: Stats; try { stat = lstatSync(path); } catch { continue; } if (!stat.isFile() || stat.isSymbolicLink()) continue; result.files += 1; result.bytes += stat.size; if (referencedFileNames.has(name)) { result.ownedFiles += 1; result.ownedBytes += stat.size; } else if (orphanSpillNameKind(name, referencedFileNames) !== null && spillEntryPastGrace(stat, graceMs)) { result.orphanFiles += 1; result.orphanBytes += stat.size; } } } finally { scan.close(); } result.truncated = result.scanned >= RESPONSE_SPILL_SCAN_MAX; return result; } export function recoverOrphanedResponseSpills( referencedFileNames: ReadonlySet<string>, dir = responseSpillDirectory(), opts?: { graceMs?: number; scanMax?: number; cleanupMax?: number; deadlineMs?: number }, ): ResponseSpillCleanupResult { const scan = openSpillDirScan(dir); if (!scan) return { scanned: 0, removed: 0, failed: 0, bytesRemoved: 0 }; try { return reclaimFromSpillDirScan(scan, referencedFileNames, dir, opts); } finally { scan.close(); } } /** Reclaim loop over an already-open scan. It never reads a name it will not * process, so a caller that keeps the scan open resumes at the first entry the * previous call left untouched. Throws only if the iterator itself throws. */ function reclaimFromSpillDirScan( scan: SpillDirScan, referencedFileNames: ReadonlySet<string>, dir: string, opts?: { graceMs?: number; scanMax?: number; cleanupMax?: number; deadlineMs?: number }, ): ResponseSpillCleanupResult { const result: ResponseSpillCleanupResult = { scanned: 0, removed: 0, failed: 0, bytesRemoved: 0 }; const graceMs = opts?.graceMs ?? RESPONSE_SPILL_ORPHAN_GRACE_MS; const scanMax = opts?.scanMax ?? RESPONSE_SPILL_SCAN_MAX; const cleanupMax = opts?.cleanupMax ?? RESPONSE_SPILL_CLEANUP_MAX; const deadline = opts?.deadlineMs === undefined ? Infinity : Date.now() + opts.deadlineMs; while (result.scanned < scanMax && result.removed + result.failed < cleanupMax && Date.now() < deadline) { const name = scan.nextName(); if (name === null) { result.exhausted = true; break; } result.scanned += 1; const orphanKind = orphanSpillNameKind(name, referencedFileNames); if (orphanKind === null) continue; const path = join(dir, name); let stat: Stats; try { stat = lstatSync(path); } catch { continue; } if (!spillEntryPastGrace(stat, graceMs)) continue; try { // Orphaned publish temps get the full ephemeral release; stable // orphaned spills keep destination-keyed timeout memos. if (orphanKind === "temp") unlinkEphemeral(path); else unlink(path); result.removed += 1; result.bytesRemoved += stat.size; } catch { result.failed += 1; } } return result; } export function responseSpillExistsForTests(ref: ResponseSpillRef): boolean { return validSpillRef(ref) && existsSync(join(responseSpillDirectory(), ref.fileName)); } /** * Bounded liveness-tick reclaim that resumes where the previous tick stopped, * so orphans behind the first `scanMax` owned entries are still reached. The * directory iterator stays open between ticks; it is dropped at end of * directory, on an iteration error, or when the spill directory changes. */ export function sweepOrphanedResponseSpillsPeriodically( referencedFileNames: ReadonlySet<string>, dir = responseSpillDirectory(), ): ResponseSpillCleanupResult { if (periodicSweepCursor && periodicSweepCursor.dir !== dir) closePeriodicSweepCursor(); if (!periodicSweepCursor) { const scan = openSpillDirScan(dir); if (!scan) return { scanned: 0, removed: 0, failed: 0, bytesRemoved: 0 }; periodicSweepCursor = { dir, scan }; } let result: ResponseSpillCleanupResult; try { result = reclaimFromSpillDirScan(periodicSweepCursor.scan, referencedFileNames, dir, PERIODIC_SPILL_SWEEP_OPTS); } catch { closePeriodicSweepCursor(); return { scanned: 0, removed: 0, failed: 0, bytesRemoved: 0 }; } // A full pass restarts from a fresh listing so files created mid-pass are seen. if (result.exhausted) closePeriodicSweepCursor(); return result; } function closePeriodicSweepCursor(): void { periodicSweepCursor?.scan.close(); periodicSweepCursor = null; } export function resetPeriodicSpillSweepCursorForTests(): void { closePeriodicSweepCursor(); }