@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
1,365 lines • 65.8 kB
text/typescript
import { chmodSync, existsSync, mkdirSync, rmSync, statSync, unlinkSync } from "node:fs";
import { dirname, join } from "node:path";
import { atomicWriteFileAsync, getConfigDir, resolveWriteTarget } from "../config";
import { appOwnedBytesSnapshot, enforceAppOwnedMemoryBudget, type RetainedStoreSnapshot } from "../lib/app-owned-memory";
import { windowsSecretAclApplies } from "../lib/windows-secret-acl";
import type { OcxProviderContinuationState } from "../types";
import type { ResidentResponseState, SpilledResponseState, SpillFailedResponseState, StoredResponseState, ResidentInput, PreviousResponseReplayFailure } from "./state/entry-types";
export type { ResidentResponseState, SpilledResponseState, SpillFailedResponseState, StoredResponseState, ResidentInput, PreviousResponseReplayFailure } from "./state/entry-types";
import {
deleteResponseSpill,
inspectResponseSpillDir,
noteStubSwapForTest,
readResponseSpill,
recoverOrphanedResponseSpills,
resetPeriodicSpillSweepCursorForTests,
responseSpillDirectory,
responseSpillPayloadCap,
sweepOrphanedResponseSpillsPeriodically,
type ResponseSpillDirInspection,
type ResponseSpillRef,
writeResponseSpillDurably,
} from "./spill-store";
import { collectReferencedSpillFileNames, snapshotReferencedSpillFileNames } from "./state/spill-inspect";
import { selectSnapshotEntries, snapshotPayload } from "./state/snapshot-select";
import { expandStoredResponseInput } from "./state/replay-input";
import { normalizedClientThreadId, replayScopeMatches, validReplayOwnerKey } from "./state/replay-owner";
import { bindConnectionScopeStore, connectionReplayScopeAcceptsEntry, connectionScopeEntryExpired, forgetConnectionScopeEntry, isConnectionOwnedResponseState, noteConnectionScopeEntry, oldestConnectionScopeResident, planConnectionScopeEvictions, type ConnectionReplayScope } from "./state/connection-scope";
export { mintConnectionReplayScope, connectionReplayScopeLive, connectionReplayScopeForThread, connectionReplayScopeOwnerKey, releaseConnectionReplayScope, type ConnectionReplayScope } from "./state/connection-scope";
import { hasPendingClientToolCall } from "./state/unforced-store-false";
import { computeResponseStateMetrics, type ResponseStateMetrics } from "./state/metrics";
export type { ResponseStateMetrics } from "./state/metrics";
export type { ResponseStateTempRecoveryResult, ResponseStateTempRecoveryOptions } from "./state/temp-recovery";
export type { ResponseSpillDirInspection } from "./spill-store";
export { recoverStaleResponseStateTemps, reclaimAbandonedResponseStateTemps, inspectAbandonedResponseStateTemps, sweepAbandonedResponseStateTemps } from "./state/temp-recovery";
import { recoverStaleResponseStateTemps } from "./state/temp-recovery";
export type { ResponseSpillWriteFailureCode, ResponseSpillWriteStatus, ResponseSpillWriteFailureOrigin } from "./state/spill-failure";
import type { ResponseSpillWriteFailureCode, ResponseSpillWriteStatus, ResponseSpillWriteFailureOrigin } from "./state/spill-failure";
export { responseAdmissionCountersForTests } from "./state/spill-failure";
import { admissionCounters, noteSpillWriteFailure, noteSpillWriteSuccess, resetSpillCountersForTests, spillCounters, spillWriteHealth } from "./state/spill-failure";
import { loadSnapshotEntries, snapshotFileFingerprint, stageSnapshotEntries, type SnapshotLoadStore } from "./state/snapshot-codec";
import { isBodyNonPersistable } from "./state/body-policy";
export { isBodyNonPersistable, markBodyNonPersistable } from "./state/body-policy";
export { flushPendingResponseSpillsForTests, awaitResponseSpillPublicationTailForTests, pendingResponseSpillMetricsForTests, setResponseSpillShutdownBudgetForTests, setResponseSpillAsyncAclAttemptBudgetForTests, setResponseSpillShutdownTerminalizationPassLimitForTests } from "./state/spill-queue";
import {
bindSpillQueueStore,
cancelPendingResponseSpill,
drainResponseSpillPublications,
queuePendingResponseSpill,
replaceWithPendingResponseSpill,
resetSpillQueueForTests,
spillQueueAccounting,
spillQueueHoldsResidentCandidate,
spillQueuePendingBytes,
spillQueueReferencedSpillFileNames,
spillQueueResidentCandidates,
spillQueueSupersededSpillFor,
} from "./state/spill-queue";
const MAX_STORED_RESPONSES = 1_000;
/**
* Retention for locally replayed continuation state.
*
* A Codex client chained by `previous_response_id` sends ONLY the new turn and expects this
* process to hold everything before it, so this constant is the practical memory span of every
* conversation that does not go to the canonical ChatGPT backend. At the original one hour, a
* session resumed after lunch expanded to nothing and the delta — one user line — was all the
* provider ever saw, which reads to the operator as the model losing the conversation.
*
* A day is safe to hold because retention is no longer what bounds this store: the resident cap
* (MAX_STORED_RESPONSE_BYTES), the spill ceiling (MAX_SPILLED_RESPONSE_BYTES) and the entry count
* evict older entries first within each retention class. Every turn re-stores the chain under a fresh id, so the live
* conversation is the last thing any of those three caps would drop. Raising the TTL therefore
* moves eviction from the clock to those budgets rather than growing the ceiling.
*/
export const RESPONSE_TTL_MS = 24 * 60 * 60 * 1_000;
const SNAPSHOT_DEBOUNCE_MS = 2_000;
/** Snapshot size below which the debounce stays at its base value. */
const SNAPSHOT_DEBOUNCE_SCALE_FROM_BYTES = 1 * 1024 * 1024;
/** Ceiling for the stretched debounce. Continuation state is only read after a
* restart, and a graceful shutdown flushes, so the exposure a longer debounce adds
* is bounded by a hard kill — paid against rewriting the whole snapshot every 2 s. */
const SNAPSHOT_DEBOUNCE_MAX_MS = 30_000;
/** In-memory high-water byte cap across all entries. Forced store:false retention (kiro/cursor
* continuation chains) stores the full expanded input each turn — ~quadratic bytes per chain —
* so a count cap alone cannot bound memory. Oldest-first eviction applies past this mark. */
export const MAX_STORED_RESPONSE_BYTES = 64 * 1024 * 1024;
/**
* Aggregate ceiling for the durable spill directory: the disk-side counterpart to
* the RAM ceiling above. Without it the spilled set is bounded only per-file
* (MAX_RESPONSE_SPILL_PAYLOAD_BYTES, 256 MiB) and per-entry (MAX_STORED_RESPONSES,
* 1000), whose product is 250 GiB — larger than the disk of any host this runs on.
* The only effective bound was therefore RESPONSE_TTL_MS, which makes disk use a
* function of client request rate rather than of anything this process controls.
*
* Measured on one macOS host, 2026-08-30: a client spilling ~150 MB payloads at
* ~1.4/min held 6.8 GB after 44 minutes, still climbing toward the ~12 GB an
* hour-long window implies, and filled the volume. Retention itself was correct
* throughout — the TTL evicted that whole cohort an hour later — so what was
* missing is a budget, not a sweep.
*
* 1 GiB comes from the same sample (n=31), whose spilled sizes are strongly
* bimodal: median 1.1 MiB against a p90 of 198.7 MiB, near the per-file ceiling.
* At that median the count cap and this ceiling bind within 8% of each other
* (1000 x 1.1 MiB = 1.07 GiB), so ordinary traffic sees no eviction it would not
* already have seen and only the large tail is cut. Erring small is the safe
* direction: too low costs a replay miss, an already-handled path surfaced as
* previous_response_not_found, while too high costs the host's disk and every
* unrelated process on it.
*/
export const MAX_SPILLED_RESPONSE_BYTES = 1024 * 1024 * 1024;
/** Legacy snapshot selection only. Spill demotion is governed solely by the RAM cap above. */
const SNAPSHOT_ENTRY_MAX_BYTES = 2 * 1024 * 1024;
const SNAPSHOT_TOTAL_MAX_BYTES = 24 * 1024 * 1024;
/** Refuse-to-parse ceiling for an existing snapshot file (above the 24 MiB write
* bound, so anything we wrote ourselves always loads; guards against externally
* planted or pre-cap unbounded files being parsed whole). */
const SNAPSHOT_FILE_MAX_BYTES = 32 * 1024 * 1024;
const MAX_SNAPSHOT_REWRITE_ATTEMPTS = 4;
const RESPONSE_SPILL_SHUTDOWN_TERMINALIZATION_MAX_PASSES = MAX_STORED_RESPONSES + 1;
const states = new Map<string, StoredResponseState>();
let storedResponseBytes = 0;
let residentResponseBytes = 0;
let budgetCandidate: [string, ResidentResponseState] | undefined;
let byteCapOverride: number | null = null;
let snapshotTotalCapOverride: number | null = null;
let durableRevision = 0;
/** Byte length and digest of the last snapshot actually written, for the
* identical-payload skip and the size-scaled debounce. The payload itself is not
* retained: at the 24 MiB bound that would double the snapshot's memory cost. */
let lastSnapshotBytes = 0;
let lastSnapshotDigest: string | null = null;
// The resolved file the digest above describes. Keeping it means a config-dir
// change or a retargeted symlink is a miss rather than a false "unchanged".
let lastSnapshotTarget: string | null = null;
/**
* Is the snapshot on disk still byte-for-byte what we last wrote?
*
* The cached digest proves what this process wrote, not what is there now. Size is
* checked first so the common mismatch costs a `stat`, and the content comparison
* only runs when the size already agrees. Any read failure answers "no" and the
* caller rewrites — the safe direction.
*/
async function snapshotOnDiskMatches(path: string, payload: string, payloadBytes: number): Promise<boolean> {
try {
const file = Bun.file(path);
if (file.size !== payloadBytes) return false;
if (await file.text() !== payload) return false;
// Content matching is not the whole invariant. This file holds persisted request
// and response bodies, and `atomicWriteFileAsync` writes it owner-only; the
// unconditional rewrite used to restore that on every mutation. Skipping without
// checking would let a broadened mode persist indefinitely, so treat a widened
// file as "does not match" and let the caller rewrite it through the hardening
// path. POSIX only — Windows ACLs are re-applied by that same write path.
if (process.platform !== "win32") {
const mode = statSync(path).mode & 0o777;
if (mode !== 0o600) return false;
}
return true;
} catch {
return false;
}
}
let replayScopeMismatchDrops = 0;
// Superseded spill generations awaiting a durable snapshot before unlink
// (review C1-1: unlinking at swap time races a crash against the debounced
// snapshot — the reloaded OLD stub would point at a deleted file).
const pendingSpillUnlinks: ResponseSpillRef[] = [];
// The queue itself must stay bounded (review C2-2: repeated replacements with
// a persistently failing snapshot write would otherwise grow it without
// limit). Beyond the cap the OLDEST superseded generation is unlinked
// immediately: the accepted worst case is that a crash inside that window
// reloads a stub whose file is gone, which fails replay with the explicit
// structured 400 — bounded-loss, never silent corruption or unbounded disk.
const PENDING_SPILL_UNLINKS_MAX = 128;
function deferSupersededSpill(ref: ResponseSpillRef | undefined): void {
if (!ref) return;
pendingSpillUnlinks.push(ref);
noteDurableMutation();
while (pendingSpillUnlinks.length > PENDING_SPILL_UNLINKS_MAX) {
deleteResponseSpill(pendingSpillUnlinks.shift()!);
}
}
function byteCap(): number {
return byteCapOverride ?? MAX_STORED_RESPONSE_BYTES;
}
/** Test-only: lower/restore the in-memory byte cap (null restores the default). */
export function setResponseStateByteCapForTests(bytes: number | null): void {
byteCapOverride = bytes;
}
function snapshotTotalBytes(): number {
return snapshotTotalCapOverride ?? SNAPSHOT_TOTAL_MAX_BYTES;
}
/** Test-only: lower/restore the durable snapshot byte budget (null restores the default). */
export function setResponseStateSnapshotByteCapForTests(bytes: number | null): void {
snapshotTotalCapOverride = bytes;
}
/** Test-only: current in-memory byte accounting (proves evictions release their bytes). */
export function getStoredResponseBytesForTests(): number {
return storedResponseBytes;
}
let spillByteCapOverride: number | null = null;
function spillByteCap(): number {
return spillByteCapOverride ?? MAX_SPILLED_RESPONSE_BYTES;
}
/**
* Live total of durable spill payloads. Recomputed per call rather than carried as
* a running counter: spilled entries reach `states` through several insertion paths
* (demotion swap, direct oversized admission, snapshot reload), and one missed
* increment there would silently disable the cap, where an O(MAX_STORED_RESPONSES)
* walk cannot drift.
*/
function spilledResponseBytes(): number {
let total = 0;
for (const entry of states.values()) {
if (entry.kind === "spill") total += entry.spill.payloadBytes;
}
// Superseded generations awaiting a durable snapshot are still files on disk.
// Counting only `states` would let PENDING_SPILL_UNLINKS_MAX of them sit outside
// the budget while it reports itself satisfied.
for (const ref of pendingSpillUnlinks) total += ref.payloadBytes;
return total;
}
/**
* Accounted on-disk bytes: files that exist, plus the peak footprint of publications
* already in flight.
*
* The cap is enforced against this rather than against `spilledResponseBytes()` alone,
* because a publication that has not finished is still consuming the volume. On Windows
* the gap between "queued" and "installed" is however long `icacls` takes, and the
* measured incident this cap answers accumulated 6.8 GiB in 44 minutes.
*/
function accountedResponseSpillBytes(): number {
// Superseded generations a pending job still owns are files on disk too. A same-id
// replacement removes the old spill from `states` and hands its ref to the job, so
// counting only `states` plus `pendingSpillUnlinks` loses it for the whole publication
// — during a copy fallback that is old generation + new temp + new destination, three
// envelopes priced as two.
const accounting = spillQueueAccounting();
return spilledResponseBytes() + accounting.reservedBytes + accounting.jobOwnedBytes
+ accounting.unreclaimableBytes;
}
/** Test-only: lower/restore the durable spill cap (null restores the default). */
export function setSpilledResponseByteCapForTests(bytes: number | null): void {
spillByteCapOverride = bytes;
}
/** Test-only: current durable spill accounting (proves evictions unlink their files). */
export function getSpilledResponseBytesForTests(): number {
return spilledResponseBytes();
}
/** Test-only: on-disk bytes plus in-flight publication reservations. */
export function getAccountedResponseSpillBytesForTests(): number {
return accountedResponseSpillBytes();
}
function serializedBytes(value: unknown): number | null {
try {
const serialized = JSON.stringify(value);
return serialized === undefined ? null : Buffer.byteLength(serialized, "utf8");
} catch {
return null;
}
}
function measureResidentEntry(id: string, entry: ResidentInput): ResidentResponseState | null {
const sizeBytes = serializedBytes({
responseId: id,
createdAt: entry.createdAt,
...(entry.clientThreadId ? { clientThreadId: entry.clientThreadId } : {}),
...(entry.replayOwnerKey ? { replayOwnerKey: entry.replayOwnerKey } : {}),
items: entry.items,
...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}),
...(entry.providers ? { providers: entry.providers } : {}),
...(entry.unforcedStoreFalse ? { unforcedStoreFalse: entry.unforcedStoreFalse } : {}),
});
return sizeBytes === null ? null : { kind: "resident", ...entry, sizeBytes };
}
function considerBudgetCandidate(id: string, entry: StoredResponseState): void {
if (entry.kind !== "resident" || spillQueueHoldsResidentCandidate(id, entry)) return;
const oldest = budgetCandidate?.[1];
// Prefer memory-only residents, then the oldest eligible resident within that class.
if (!oldest || (entry.connectionOwned === oldest.connectionOwned
? entry.createdAt < oldest.createdAt : entry.connectionOwned === true)) budgetCandidate = [id, entry];
}
function recomputeOldestResident(): void {
budgetCandidate = undefined;
for (const [id, entry] of states) considerBudgetCandidate(id, entry);
}
function replaceMapEntry(id: string, next: StoredResponseState, expected?: StoredResponseState): boolean {
const existing = states.get(id);
if (expected && existing !== expected) return false;
storedResponseBytes -= existing?.sizeBytes ?? 0;
storedResponseBytes += next.sizeBytes;
if (existing?.kind === "resident") {
residentResponseBytes -= existing.sizeBytes;
forgetConnectionScopeEntry(id, existing);
}
if (next.kind === "resident") {
residentResponseBytes += next.sizeBytes;
}
if (storedResponseBytes < 0) storedResponseBytes = 0;
if (residentResponseBytes < 0) residentResponseBytes = 0;
if (existing) states.delete(id);
states.set(id, next);
if (budgetCandidate?.[0] === id) recomputeOldestResident();
else considerBudgetCandidate(id, next);
if (!isConnectionOwnedResponseState(next) || (existing && !isConnectionOwnedResponseState(existing))) noteDurableMutation();
return true;
}
function stubSize(id: string, entry: Omit<SpilledResponseState, "sizeBytes">): number {
return serializedBytes({ responseId: id, ...entry }) ?? 0;
}
function tombstone(id: string, createdAt: number): SpillFailedResponseState {
const base = { kind: "spill-failed" as const, createdAt };
return { ...base, sizeBytes: serializedBytes({ responseId: id, ...base }) ?? 0 };
}
function deleteOwnedSpills(entry: StoredResponseState): void {
if (entry.kind === "spill") deleteResponseSpill(entry.spill);
}
/** The ONLY deletion point: TTL, count, byte, and explicit deletes all route here. */
function deleteEntry(id: string, options: { deleteSpill?: boolean } = {}): void {
const existing = states.get(id);
if (!existing) return;
const scoped = isConnectionOwnedResponseState(existing);
const supersededSpill = scoped ? undefined : cancelPendingResponseSpill(id);
storedResponseBytes -= existing.sizeBytes;
if (existing.kind === "resident") {
residentResponseBytes -= existing.sizeBytes;
forgetConnectionScopeEntry(id, existing);
}
if (storedResponseBytes < 0) storedResponseBytes = 0;
if (residentResponseBytes < 0) residentResponseBytes = 0;
states.delete(id);
if (budgetCandidate?.[0] === id) recomputeOldestResident();
if (!scoped) noteDurableMutation();
if (options.deleteSpill !== false) deleteOwnedSpills(existing);
if (options.deleteSpill !== false && supersededSpill) deleteResponseSpill(supersededSpill);
}
function replaceWithSpillFailure(
id: string,
expected?: StoredResponseState,
options: { deferSpillUnlink?: boolean } = {},
): void {
const existing = states.get(id);
if (expected && existing !== expected) return;
const failed = tombstone(id, expected?.createdAt ?? existing?.createdAt ?? now());
if (replaceMapEntry(id, failed, expected)) {
if (existing) {
if (options.deferSpillUnlink && existing.kind === "spill") {
// Crash consistency (same rule as replaceSpillEntryAtomically): the old
// durable snapshot still references this generation until the tombstone
// itself is durable — queue the unlink for the next stable persist.
deferSupersededSpill(existing.spill);
} else {
deleteOwnedSpills(existing);
}
}
}
}
function swapResidentForSpill(id: string, expected: ResidentResponseState, ref: ResponseSpillRef): boolean {
if (expected.connectionOwned === true) {
// Memory-only entries can never become stubs. A queued write reaching this sink is
// already off-invariant: unlink the file it produced and keep the resident — the
// scope's own eviction or release removes it, never a spill swap.
deleteResponseSpill(ref);
return false;
}
const base: Omit<SpilledResponseState, "sizeBytes"> = {
kind: "spill",
createdAt: expected.createdAt,
...(expected.clientThreadId ? { clientThreadId: expected.clientThreadId } : {}),
...(expected.replayOwnerKey ? { replayOwnerKey: expected.replayOwnerKey } : {}),
...(expected.providerOutputStart !== undefined ? { providerOutputStart: expected.providerOutputStart } : {}),
...(expected.providers ? { providers: expected.providers } : {}),
...(expected.unforcedStoreFalse ? { unforcedStoreFalse: expected.unforcedStoreFalse } : {}),
spill: ref,
};
const next: SpilledResponseState = { ...base, sizeBytes: stubSize(id, base) };
if (!replaceMapEntry(id, next, expected)) {
deleteResponseSpill(ref);
return false;
}
noteStubSwapForTest();
return true;
}
function replaceSpillEntryAtomically(
id: string,
expected: SpilledResponseState,
candidate: ResidentResponseState,
): void {
// A memory-only candidate never displaces a durable spill entry: the foreign stub is
// left exactly as it was and the scoped write is refused.
if (candidate.connectionOwned === true) return;
try {
const ref = writeResponseSpillDurably(id, {
createdAt: candidate.createdAt,
...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}),
...(candidate.replayOwnerKey ? { replayOwnerKey: candidate.replayOwnerKey } : {}),
items: candidate.items,
...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}),
...(candidate.providers ? { providers: candidate.providers } : {}),
...(candidate.unforcedStoreFalse ? { unforcedStoreFalse: candidate.unforcedStoreFalse } : {}),
});
const base: Omit<SpilledResponseState, "sizeBytes"> = {
kind: "spill",
createdAt: candidate.createdAt,
...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}),
...(candidate.replayOwnerKey ? { replayOwnerKey: candidate.replayOwnerKey } : {}),
...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}),
...(candidate.providers ? { providers: candidate.providers } : {}),
...(candidate.unforcedStoreFalse ? { unforcedStoreFalse: candidate.unforcedStoreFalse } : {}),
spill: ref,
};
const next: SpilledResponseState = { ...base, sizeBytes: stubSize(id, base) };
if (!replaceMapEntry(id, next, expected)) {
deleteResponseSpill(ref);
return;
}
noteSpillWriteSuccess();
noteStubSwapForTest();
// The old generation is NOT unlinked here (review C1-1): the new stub is
// only durable once the debounced snapshot flushes — a crash before that
// reloads the OLD stub, which must still find its file. Queue the unlink;
// persistNow() drains the queue only after the snapshot write succeeds.
deferSupersededSpill(expected.spill);
} catch (error) {
noteSpillWriteFailure(error);
// deferSpillUnlink: the durable snapshot may still reference the old
// generation; deleting it now would strand the old stub after a crash.
replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
}
}
function setResidentEntry(id: string, entry: ResidentInput): void {
const expected = states.get(id);
const candidate = measureResidentEntry(id, entry);
const scope = entry.connectionScope;
if (scope) {
if (!candidate || (expected && (!isConnectionOwnedResponseState(expected) || expected.connectionScope !== scope))) return;
if (candidate.sizeBytes > byteCap()) {
admissionCounters.oversizedDrops += 1;
return;
}
const plan = (coldBytes: number, coldCount: number) => {
const { budgetBytes, retainedBytes } = appOwnedBytesSnapshot();
const headroom = Math.min(byteCap() - storedResponseBytes, budgetBytes - retainedBytes) - coldBytes;
const countToFree = states.size + coldCount + (expected ? 0 : 1) - MAX_STORED_RESPONSES;
return planConnectionScopeEvictions(states, scope, id, countToFree,
candidate.sizeBytes - (expected?.sizeBytes ?? 0) - headroom);
};
const path = loaded ? undefined : snapshotPath();
const cutoff = responseExpiryCutoff(now());
const cached = coldSnapshotAdmission;
// Cache only bounded admission facts, never decoded payloads outside the memory budget.
// A changed file or the next durable expiry invalidates them before they can reject a write.
if (path && cached?.path === path && cutoff <= cached.oldestCreatedAt
&& snapshotFileFingerprint(path) === cached.fingerprint) {
if (cached.conflictingId === id || !plan(cached.bytes, cached.count)) return;
}
// Live cold rows protect durable response IDs and count against both byte budgets.
const staged = path ? stageSnapshotEntries(path, SNAPSHOT_FILE_MAX_BYTES, snapshotLoadStore, cutoff) : undefined;
if (staged?.failed) return;
if (staged?.fingerprint !== undefined) coldSnapshotAdmission = {
path: staged.path, fingerprint: staged.fingerprint, bytes: staged.bytes, count: staged.entries.size,
oldestCreatedAt: staged.oldestCreatedAt, ...(staged.entries.has(id) ? { conflictingId: id } : {}),
};
if (staged?.entries.has(id)) return;
const evictions = plan(staged?.bytes ?? 0, staged?.entries.size ?? 0);
if (!evictions || !connectionReplayScopeAcceptsEntry(scope, id, entry.replayOwnerKey, entry.clientThreadId)) return;
if (staged) {
recoverSnapshotTemps(staged.path);
for (const [rowId, row] of staged.entries) replaceMapEntry(rowId, row);
finishSnapshotLoad(staged.oversized);
}
for (const evictedId of evictions) deleteEntry(evictedId);
replaceMapEntry(id, candidate, expected);
noteConnectionScopeEntry(scope, id);
return;
}
if (!candidate) {
replaceWithSpillFailure(id, expected);
// A tombstone is tiny but still resident state: the hard-cap invariant
// must hold on EVERY mutation path (review C2-1 — with a test cap below
// tombstone size, skipping the prune leaves the store over cap).
pruneResponses();
return;
}
if (candidate.sizeBytes > byteCap()) {
admitOversizedCandidate(id, candidate, expected);
pruneResponses();
return;
}
if (windowsSecretAclApplies() && (expected?.kind === "spill" || spillQueueSupersededSpillFor(id))) {
replaceWithPendingResponseSpill(id, candidate, expected);
pruneResponses();
return;
}
if (expected?.kind === "spill") {
replaceSpillEntryAtomically(id, expected, candidate);
pruneResponses();
return;
}
if (windowsSecretAclApplies()) cancelPendingResponseSpill(id);
if (!replaceMapEntry(id, candidate, expected)) return;
pruneResponses();
}
/**
* Admission boundary for candidates that can never fit as resident (larger
* than the whole resident-map cap). Writes them DIRECTLY to durable spill and
* installs only the stub — the oversized candidate never becomes resident and
* no unrelated resident is demoted to make room for it. Candidates above the
* single-spill payload ceiling are tombstoned instead: retaining a spill the
* replay ceiling would refuse to read is write-only waste.
*/
function admitOversizedCandidate(
id: string,
candidate: ResidentResponseState,
expected?: StoredResponseState,
): void {
if (candidate.sizeBytes > responseSpillPayloadCap()) {
admissionCounters.oversizedDrops += 1;
replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
return;
}
if (windowsSecretAclApplies()) {
replaceWithPendingResponseSpill(id, candidate, expected, { directAdmission: true });
return;
}
try {
const ref = writeResponseSpillDurably(id, {
createdAt: candidate.createdAt,
...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}),
...(candidate.replayOwnerKey ? { replayOwnerKey: candidate.replayOwnerKey } : {}),
items: candidate.items,
...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}),
...(candidate.providers ? { providers: candidate.providers } : {}),
...(candidate.unforcedStoreFalse ? { unforcedStoreFalse: candidate.unforcedStoreFalse } : {}),
});
// Enforce the ceiling against the REAL envelope: the spill payload adds
// the {version, responseId, ...} wrapper, so a candidate within the
// wrapper's size of the cap would otherwise be retained unreadably.
if (ref.payloadBytes > responseSpillPayloadCap()) {
deleteResponseSpill(ref);
admissionCounters.oversizedDrops += 1;
replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
return;
}
const base: Omit<SpilledResponseState, "sizeBytes"> = {
kind: "spill",
createdAt: candidate.createdAt,
...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}),
...(candidate.replayOwnerKey ? { replayOwnerKey: candidate.replayOwnerKey } : {}),
...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}),
...(candidate.providers ? { providers: candidate.providers } : {}),
...(candidate.unforcedStoreFalse ? { unforcedStoreFalse: candidate.unforcedStoreFalse } : {}),
spill: ref,
};
const next: SpilledResponseState = { ...base, sizeBytes: stubSize(id, base) };
if (!replaceMapEntry(id, next, expected)) {
deleteResponseSpill(ref);
return;
}
noteSpillWriteSuccess();
admissionCounters.directSpills += 1;
noteStubSwapForTest();
if (expected?.kind === "spill") {
// Same deferred-unlink rule as replaceSpillEntryAtomically: the new stub
// is durable only after the debounced snapshot, so the old generation
// stays until a stable persist drains the queue.
deferSupersededSpill(expected.spill);
}
} catch (error) {
noteSpillWriteFailure(error);
replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
}
}
bindSpillQueueStore({
swapResidentForSpill,
replaceWithSpillFailure,
deleteEntry,
deferSupersededSpill,
replaceMapEntry,
currentEntry: (id: string) => states.get(id),
residentEntries: () => [...states],
recomputeOldestResident,
pruneResponses,
accountedResponseSpillBytes,
spillByteCap,
enforceSpilledResponseBudget,
terminalizationMaxPasses: () => RESPONSE_SPILL_SHUTDOWN_TERMINALIZATION_MAX_PASSES,
});
bindConnectionScopeStore({ deleteEntry, currentEntry: id => states.get(id) });
// Replay provenance must stay proxy-private: a WeakMap distinguishes replayed history from the
// newly appended input suffix without adding an unknown field that native passthrough could send
// upstream. The parser uses this boundary to acknowledge historical compaction markers exactly
// once. It records the boundary whether the proxy prepended the history or the client already
// carried it — the boundary is the same either way, and only its provenance differs.
const replayedInputPrefixLengths = new WeakMap<object, number>();
const clientCarriedReplays = new WeakSet<object>(); // overlap skip: the client resent the window itself
const replayFailures = new WeakMap<object, PreviousResponseReplayFailure>();
let loaded = false;
let coldSnapshotAdmission: {
path: string; fingerprint: string; bytes: number; count: number; oldestCreatedAt: number; conflictingId?: string;
} | undefined;
let persistTimer: ReturnType<typeof setTimeout> | null = null;
let pendingPersistPath: string | null = null;
/** Single-flight gate: overlapping response-state writes serialize (#612). */
let persistGate: Promise<void> = Promise.resolve();
let persistAttemptHookForTests: (() => void) | null = null;
function now(): number {
return Date.now();
}
function snapshotPath(): string {
return join(getConfigDir(), "responses-state.json");
}
function recoverSnapshotTemps(path: string): void {
// Atomic writes place their temp beside the RESOLVED target, so a symlinked
// snapshot (dotfiles-managed config dir) strands temps in the link's real
// directory where a scan of the literal config dir would never see them.
// Both locations are swept; they collapse to one when nothing is symlinked.
// resolveWriteTarget refuses a dangling link; snapshot loading stays independent.
let resolvedDir = dirname(path);
try {
resolvedDir = dirname(resolveWriteTarget(path));
} catch {
/* unresolvable link: sweep the literal dir only */
}
for (const dir of new Set([dirname(path), resolvedDir])) {
try {
recoverStaleResponseStateTemps(dir);
} catch {
/* best-effort cleanup only; snapshot loading must remain independent */
}
}
}
const snapshotLoadStore: SnapshotLoadStore = {
replaceMapEntry,
stubSize,
tombstone,
measureResidentEntry,
// Spill oversized durable rows directly so first-load prune cannot demote unrelated rows.
admitResident(id, entry) {
if (entry.sizeBytes > byteCap()) admitOversizedCandidate(id, entry);
else replaceMapEntry(id, entry);
},
};
function finishSnapshotLoad(oversized: boolean): void {
if (oversized) admissionCounters.snapshotOversizedRefusals += 1;
loaded = true;
coldSnapshotAdmission = undefined;
try { recoverOrphanedResponseSpills(ownedSpillFileNames()); } catch { /* best effort */ }
}
/** Best-effort lazy load so durable previous_response_id chains can survive a restart. */
function ensureLoaded(): boolean {
if (loaded) return true;
const path = snapshotPath();
const result = loadSnapshotEntries(path, SNAPSHOT_FILE_MAX_BYTES, snapshotLoadStore);
if (result.failed) return false;
recoverSnapshotTemps(path);
finishSnapshotLoad(result.oversized);
pruneResponses();
return true;
}
/** Every file name the process still needs: live stubs, deferred unlinks, and queued publications. */
function ownedSpillFileNames(): Set<string> {
const referenced = collectReferencedSpillFileNames(states.values(), pendingSpillUnlinks);
for (const name of spillQueueReferencedSpillFileNames()) referenced.add(name);
return referenced;
}
/** Liveness-tick counterpart of the orphan GC in ensureLoaded; a no-op until first load. */
export function sweepOrphanedResponseSpills(): number {
if (!loaded) return 0;
try {
return sweepOrphanedResponseSpillsPeriodically(ownedSpillFileNames()).removed;
} catch {
return 0;
}
}
type SnapshotWriteOutcome = "stable" | "unstable" | "failed";
async function writeBoundedSnapshot(path: string, attemptLimit: number): Promise<SnapshotWriteOutcome> {
try {
for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
const revision = durableRevision;
const payload = snapshotPayload(selectSnapshotEntries(states, snapshotTotalBytes(), SNAPSHOT_ENTRY_MAX_BYTES));
const payloadBytes = Buffer.byteLength(payload, "utf8");
const payloadDigest = Bun.hash(payload).toString(36);
// A mutation does not always change what gets persisted: entries past the
// per-entry or total byte bound are dropped from the selection, and spill
// demotion moves bytes out of it. Re-writing a byte-identical 24 MiB file
// buys nothing, so compare first — but the cached digest describes what THIS
// process last wrote, which is not the same claim as "that is what is on disk
// now". A second proxy sharing the home, or anything that rewrites the file
// in place, leaves the digest describing bytes that are gone. Before every
// release-of-a-write, the previous behaviour rewrote unconditionally and so
// repaired that silently; skipping without checking would turn a repaired
// snapshot into a lost one at the next restart.
//
// Verify against the file itself, keyed to the resolved target so a retargeted
// symlink is also a miss. Reading back a matching-size file costs far less
// than the atomic replace it avoids, and only happens when the digest already
// matched — the amplification this fixes is the repeated WRITE, not the read.
const unchanged = lastSnapshotDigest !== null
&& payloadDigest === lastSnapshotDigest
&& payloadBytes === lastSnapshotBytes
&& lastSnapshotTarget === resolveWriteTarget(path)
&& existsSync(path)
&& await snapshotOnDiskMatches(path, payload, payloadBytes);
if (!unchanged) {
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
try { chmodSync(dirname(path), 0o700); } catch { /* best-effort (e.g. Windows) */ }
await atomicWriteFileAsync(path, payload);
lastSnapshotDigest = payloadDigest;
lastSnapshotBytes = payloadBytes;
lastSnapshotTarget = resolveWriteTarget(path);
}
persistAttemptHookForTests?.();
if (revision === durableRevision) {
// Cancel only the timer whose mutation this stable revision includes,
// before an awaiting caller's microtask can record a newer revision.
if (persistTimer) clearTimeout(persistTimer);
persistTimer = null;
pendingPersistPath = null;
drainPendingSpillUnlinks();
return "stable";
}
}
return "unstable";
} catch {
return "failed";
}
}
function drainPendingSpillUnlinks(): void {
while (pendingSpillUnlinks.length > 0) {
const ref = pendingSpillUnlinks.shift()!;
deleteResponseSpill(ref);
}
}
/**
* Debounce scaled by the size of the last snapshot written.
*
* The whole snapshot is re-serialized and atomically replaced on every flush, so at
* the 24 MiB bound a fixed 2 s debounce is up to ~12 MB/s of write amplification for
* state nothing reads until the next start (#2460). Small snapshots keep the base
* cadence; the stretch is linear in size and clamped, so the write rate is roughly
* flat instead of growing with the file.
*/
function snapshotDebounceMs(): number {
if (lastSnapshotBytes <= SNAPSHOT_DEBOUNCE_SCALE_FROM_BYTES) return SNAPSHOT_DEBOUNCE_MS;
const scaled = Math.round(SNAPSHOT_DEBOUNCE_MS * (lastSnapshotBytes / SNAPSHOT_DEBOUNCE_SCALE_FROM_BYTES));
return Math.min(scaled, SNAPSHOT_DEBOUNCE_MAX_MS);
}
function schedulePersistAt(path: string, replace = false): void {
if (persistTimer && !replace) return;
if (persistTimer) clearTimeout(persistTimer);
pendingPersistPath = path;
persistTimer = setTimeout(() => { void persistNow(path); }, snapshotDebounceMs());
(persistTimer as { unref?: () => void }).unref?.();
}
async function persistNow(path: string, awaitFollowUp = false): Promise<void> {
// Include follow-up scheduling in the gate: releasing after only the disk write
// lets shutdown observe an empty timer before the background pass re-arms it.
const previous = persistGate;
let release!: () => void;
persistGate = new Promise<void>(resolve => { release = resolve; });
await previous;
try {
if (persistTimer) {
clearTimeout(persistTimer);
persistTimer = null;
}
pendingPersistPath = null;
const attemptLimit = awaitFollowUp ? MAX_SNAPSHOT_REWRITE_ATTEMPTS : 1;
let outcome = await writeBoundedSnapshot(path, attemptLimit);
if (outcome === "unstable" && awaitFollowUp) {
if (persistTimer) clearTimeout(persistTimer);
persistTimer = null;
pendingPersistPath = null;
outcome = await writeBoundedSnapshot(path, attemptLimit);
}
if (outcome === "unstable" && !awaitFollowUp) schedulePersistAt(path, true);
} finally {
release();
}
}
function noteDurableMutation(): void {
durableRevision += 1;
if (!loaded) return;
// Resolve the target path NOW: tests (and anything else) may swap OPENCODEX_HOME before the
// debounce fires, and a late write must land in the home that owned the recorded state.
schedulePersistAt(snapshotPath());
}
async function flushResponseSnapshot(): Promise<void> {
// The gate includes the background pass's timer decision, not just its write.
await persistGate;
if (persistTimer) await persistNow(pendingPersistPath ?? snapshotPath(), true);
}
/** Flush publications and snapshot state; report drain failure only after persistence completes. */
export async function flushResponseState(): Promise<void> {
const failures: unknown[] = [];
try {
await drainResponseSpillPublications();
} catch (error) {
failures.push(error);
}
try {
await flushResponseSnapshot();
} catch (error) {
failures.push(error);
}
if (failures.length === 1) throw failures[0];
if (failures.length > 1) throw new AggregateError(failures, "Response state shutdown flush incomplete");
}
function inputItems(input: unknown): unknown[] {
if (input === undefined) return [];
if (Array.isArray(input)) return input;
if (typeof input === "string") return [{ role: "user", content: input }];
return [input];
}
let replayOverlapSkips = 0;
/** Test-only: replay prepends skipped because the client already carried the history. */
export function replayOverlapSkipsForTests(): number {
return replayOverlapSkips;
}
/**
* Bring the durable spill set inside MAX_SPILLED_RESPONSE_BYTES, and report the
* bytes released.
*
* One owner, three callers: mutation pruning, the lazy load that follows a
* restart, and the periodic sweep. The periodic caller is not redundant — the
* mutation path only runs when traffic arrives, and a process can come up over
* budget from a snapshot written under a larger ceiling and then sit idle. That
* was observed in production at 1.8 GiB against a 1 GiB cap, held until the first
* request.
*
* NOT covered here: spill files orphaned by a crash. They are absent from
* `states`, so this function can neither see nor price them, and they stay with
* recoverOrphanedResponseSpills and its RESPONSE_SPILL_ORPHAN_GRACE_MS window.
* This ceiling therefore bounds what the store owns, which is every file it can
* account for, and not the directory as a whole.
*/
function enforceSpilledResponseBudget(requiredHeadroomBytes?: number): number {
// Price in-flight publications too: a file being created by
// `writeResponseSpillDurablyAsync` occupies the volume before it reaches `states`.
let spilledBytes = accountedResponseSpillBytes();
// Admission (#6747) passes the room its publication needs, 0 included. Evict only while that
// room is still reachable: re-read accounting after each attempt, since a failed unlink stays
// charged, and stop the moment the publication cannot fit so no unrelated continuation pays.
const admission = requiredHeadroomBytes !== undefined;
const headroom = admission && requiredHeadroomBytes > 0 ? requiredHeadroomBytes : 0;
const feasible = (): boolean => !admission
|| (spilledBytes = accountedResponseSpillBytes()) - spilledResponseBytes() + headroom <= spillByteCap();
if (!feasible() || spilledBytes <= spillByteCap() - headroom) return 0;
const before = spilledBytes;
// Deferred generations go first. They are already superseded, so releasing one
// costs only the crash window the queue exists to cover — the same trade
// PENDING_SPILL_UNLINKS_MAX already makes against unbounded disk. Evicting a
// live continuation to make room for a dead file would be the wrong order.
while (spilledBytes > spillByteCap() - headroom && pendingSpillUnlinks.length > 0) {
const ref = pendingSpillUnlinks.shift()!;
spilledBytes -= ref.payloadBytes;
deleteResponseSpill(ref);
if (admission) spillCounters.headroomEvictions += 1;
if (!feasible()) return before - spilledBytes;
}
// Ordered by createdAt, not by map order. `states` is not an age index:
// demotion and spill replacement delete and reinsert entries, and
// writeBoundedSnapshot serializes the map reversed, so map order can put a
// newer continuation first — and evicting that one spends a resume the older
// entry would not have cost. Sorting is O(k log k) over the spilled subset and
// runs only on a tick already over budget.
const spilled = [...states]
.filter((pair): pair is [string, SpilledResponseState] => pair[1].kind === "spill")
// createdAt is millisecond-resolution, so ties are ordinary under load. A
// stable sort would then fall back to insertion order — the very order this
// is avoiding — so break ties on the response id. Not localeCompare: the
// order must not depend on the host locale.
.sort((a, b) => a[1].createdAt - b[1].createdAt
|| (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
for (const [id, entry] of spilled) {
if (spilledBytes <= spillByteCap() - headroom) break;
spilledBytes -= entry.spill.payloadBytes;
deleteEntry(id);
if (admission) spillCounters.headroomEvictions += 1;
if (!feasible()) return before - spilledBytes;
}
return before - spilledBytes;
}
function responseExpiryCutoff(at: number): number {
return at - RESPONSE_TTL_MS;
}
function deleteExpiredEntries(at: number): number {
const cutoff = responseExpiryCutoff(at);
let removed = 0;
for (const [id, state] of states) {
if (state.createdAt >= cutoff && !connectionScopeEntryExpired(state, at)) continue;
deleteEntry(id);
removed += 1;
}
return removed;
}
function pruneResponses(at = now()): void {
deleteExpiredEntries(at);
while (states.size > MAX_STORED_RESPONSES) {
const oldest = oldestConnectionScopeResident(states, spillQueueHoldsResidentCandidate) ?? states.keys().next().value;
if (oldest === undefined) break;
deleteEntry(oldest);
}
// Unconditional RAM cap. Resident payloads demote durably; stubs/tombstones are
// deleted only when even their bounded metadata cannot fit the override.
while (storedResponseBytes > byteCap() && states.size > 0) {
// Memory-only connection residents drop before any durable entry pays a disk demotion.
const connectionOldest = oldestConnectionScopeResident(states, spillQueueHoldsResidentCandidate);
if (connectionOldest !== undefined) {
deleteEntry(connectionOldest);
continue;
}
const oldestResident = [...states].find(([id, entry]) => entry.kind === "resident"
&& !spillQueueHoldsResidentCandidate(id, entry));
const hasPendingResident = !oldestResident && [...states].some(([id, entry]) => entry.kind === "resident"
&& spillQueueHoldsResidentCandidate(id, entry));
if (hasPendingResident) break;
const oldestId = oldestResident?.[0] ?? states.keys().next().value as string | undefined;
if (oldestId === undefined) break;
const entry = states.get(oldestId)!;
if (entry.kind !== "resident") {
deleteEntry(oldestId);
continue;
}
if (windowsSecretAclApplies()) {
queuePendingResponseSpill(oldestId, entry);
continue;
}
try {
const ref = writeResponseSpillDurably(oldestId, {
createdAt: entry.createdAt,
...(entry.clientThreadId ? { clientThreadId: entry.clientThreadId } : {}),
...(entry.replayOwnerKey ? { replayOwnerKey: entry.replayOwnerKey } : {}),
items: entry.items,
...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}),
...(entry.providers ? { providers: entry.providers } : {}),
...(entry.unforcedStoreFalse ? { unforcedStoreFalse: entry.unforcedStoreFalse } : {}),
});
if (swapResidentForSpill(oldestId, entry, ref)) noteSpillWriteSuccess();
} catch (error) {
noteSpillWriteFailure(error);
replaceWithSpillFailure(oldestId, entry);
}
}
enforceSpilledResponseBudget();
}
/** Periodic TTL-only sweep; count/byte eviction remains owned by mutation paths. */
export function sweepExpiredResponseStates(at = now()): number {
const removed = deleteExpiredEntries(at);
// The disk ceiling needs a caller that does not depend on traffic. The return
// value stays the TTL count so this function's existing contract is unchanged.
enforceSpilledResponseBudget();
return removed;
}
export function responseContinuationRetainedStoreSnapshot(): RetainedStoreSnapshot {
let currentPendingBytes = 0;
for (const job of spillQueueResidentCandidates()) {
if (states.get(job.id) === job.candidate) currentPendingBytes += job.sizeBytes;
}
const detachedPendingBytes = Math.max(0, spillQueuePendingBytes() - currentPendingBytes);
const bytes = storedResponseBytes + detachedPendingBytes;
const evictableBytes = Math.max(0, residentResponseBytes - currentPendingBytes);
return {
count: states.size,
bytes,
evictableBytes,
pinnedBytes: Math.max(0, bytes - evictableBytes),
oldestAt: budgetCandidate?.[1].createdAt ?? null,
};
}
export function evictOldestResponseContinuationForBudget(): number {
if (!budgetCandidate) return 0;
const [id, entry] = budgetCandidate;
if (entry.connectionOwned === true) { deleteEntry(id); return entry.sizeBytes; }
if (windowsSecretAclApplies()) {
queuePendingResponseSpill(id, entry);
return 0;
}
try {
const ref = writeResponseSpillDurably(id, {
createdAt: entry.createdAt,
...(entry.clientThreadId ? { clientThreadId: entry.clientThreadId } : {}),
...(entry.replayOwnerKey ? { replayOwnerKey: entry.replayOwnerKey } : {}),
items: entry.items,
...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}),
...(entry.providers ? { providers: entry.providers } : {}),
...(entry.unforcedStoreFalse ? { unforcedStoreFalse: entry.unforcedStoreFalse } : {}),
});
if (swapResidentForSpill(id, entry, ref)) noteSpillWriteSuccess();
} catch (error) {
noteSpillWriteFailure(error);
replaceWithSpillFailure(id, entry);
}
const replacement = states.get(id);
return !replacement || replacement.kind === "resident"
? 0
: Math.max(0, entry.sizeBytes - replacement.sizeBytes);
}
function materializeEntry(
id: string,
entry: StoredResponseState,
): { ok: true; state: ResidentResponseState } | { ok: false; failure: PreviousResponseReplayFailure } {
if (entry.kind === "resident") return { ok: true, state: entry };
if (entry.kind === "spill-failed") {
return { ok: false, failure: { code: "previous_response_not_found", reason: "spill_failed" } };
}
const result = readResponseSpill(id, entry.spill);
if (!result.ok) {
spillCounters.readFailures += 1;
const failure: PreviousResponseReplayFailure = {
code: "previous_response_not_found",
reason: result.reason === "missing"
? "spill_missing"
: result.reason === "too_large"
? "spill_too_large"
: "spill_corrupt",
};
replaceWithSpillFailure(id, entry);
return { ok: false, failure };
}
if (entry.replayOwnerKey !== result.payload.replayOwnerKey || entry.clientThreadId !== result.payload.clientThreadId) {
return { ok: false, failure: { code: "previous_response_not_found", reason: "spill_corrupt" } };
}
// A restricted snapshot stub must agree with the validated spill's provenance.
if (entry.unforcedStoreFalse && (result.payload.unforcedStoreFalse !== true
|| entry.providerOutputStart === undefined || entry.providerOutputStart > result.payload.items.length
|| entry.providerOutputStart !== result.payload.providerOutputStart)) {
spillCounters.readFailures += 1;
replaceWithSpillFailure(id, entry);
return { ok: false, failure: { code: "previous_response_not_found", reason: "spill_corrupt" } };
}
const state = measureResidentEntry(id, {
createdAt: result.payload.createdAt,
...(result.payload.clientThreadId ? { clientThreadId: result.payload.clientThreadId } : {}),
...(result.payload.replayOwnerKey ? { replayOwnerKey: result.payload.replayOwnerKey } : {}),
items: result.payload.items,
...(result.payload.providerOutputStart !== undefined
? { providerOutputStart: result.payload.providerOutputStart }
: {}),
...(result.payload.providers ? { providers: result.payload.providers } : {}),
...(result.payload.unforcedStoreFalse ? { unforcedStoreFalse: result.payload.unforcedStoreFalse } : {}),
});
if (!state) {
spillCounters.readFailures += 1;
replaceWithSpillFailure(id, entry);
return { ok: false, failure: { code: "previous_response_not_found", reason: "spill_corrupt" } };
}
return { ok: true, state };
}
export function expandPreviousResponseInput(body: unknown, clientThreadId?: string, replayOwnerKey?: string | null): unknown {
if (!body || typeof body !== "object" || Array.isArray(body)) return body;
const request = body as Record<string, unknown>;
const previousId = typeof request.previous_response_id === "string" ? request.previous_response_id : undefined;
if (!previousId) return body;
if (!ensureLoaded()) {
if (replayOwnerKey !== undefined) replayFailures.set(request, { code: "previous_response_not_found", reason: "missing" });
return body;
}
pruneResponses();
const previous = states.get(previousId);
if (!previous) {
// Strict replay needs a local ownership proof even after a scoped entry was released.
// Legacy callers leave the owner undefined and retain their existing upstream continuation.
if (replayOwnerKey !== undefined) replayFailures.set(request, { code: "previous_response_not_found", reason: "missing" });
return body;
}
// Authorize metadata BEFORE opening a spill or exposing provider continuation state.
if (previous.kind !== "spill-failed" && !replayScopeMatches(previous, clientThreadId, replayOwnerKey)) {
replayFailures.set(request, { code: "previous_response_not_found", reason: "scope_mismatch" });
replayScopeMismatchDrops += 1;
return body;
}
const materialized = materializeEntry(previousId, previous);
if (!materialized.ok) {
replayFailures.set(request, materialized.failure);
return body;
}
const expanded = expandStoredResponseInput(request, inputItems(request.input), materialized.state);
if (expanded.overlap) {
replayOverlapSkips += 1;
clientCarriedReplays.add(expanded.body);
}
if (expanded.body !== request) replayedInputPrefixLengths.set(expanded.body, expanded.prefixLength);
return expanded.body;
}
export function previousResponseReplayFailure(body: unknown): PreviousResponseReplayFailure | undefined {
if (!body || typeof body !== "object" || Array.isArray(body)) return undefined;
return replayFailures.get(body);
}
/** Number of leading input items restored from previous_response_id state for this exact body. */
export function previousResponseReplayPrefixLength(body: unknown): number {
if (!body || typeof body !== "object" || Array.isArray(body)) return 0;
return replayedInputPrefixLengths.get(body) ?? 0;
}
/** Leading items the PROXY prepended; 0 when the client resent the stored window itself (overlap skip). */
export function previousResponseRestoredPrefixLength(body: unknown): number {
return body && typeof body === "object" && clientCarriedReplays.has(body) ? 0 : previousResponseReplayPrefixLength(body);
}
/** Copy proxy-private replay provenance to an internal clone with the same materialized input. */
export function copyPreviousResponseReplayProvenance(source: unknown, target: unknown): void {
if (!source || typeof source !== "object" || Array.isArray(source)) return;
if (!target || typeof target !== "object" || Array.isArray(target)) return;
const prefixLength = replayedInputPrefixLengths.get(source);
if (!prefixLength) return;
const input = (target as { input?: unknown }).input;
if (!Array.isArray(input) || prefixLength > input.length) return;
replayedInputPrefixLengths.set(target, prefixLength);
if (clientCarriedReplays.has(source)) clientCarriedReplays.add(target);
}
/** True when this exact request could not replay because its task scope did not match. */
export function previousResponseScopeMismatch(body: unknown): boolean {
return previousResponseReplayFailure(body)?.reason === "scope_mismatch";
}
export function previousResponseConversationId(responseId: string | undefined): string | undefined {
return previousResponseProviderState(responseId)?.cursor?.conversationId;
}
export function previousResponseProviderState(responseId: string | undefined): OcxProviderContinuationState | undefined {
if (!responseId) return undefined;
if (!ensureLoaded()) return undefined;
pruneResponses();
const state = states.get(responseId);
const providers = state?.kind === "spill-failed" ? undefined : state?.providers;
return providers ? structuredClone(providers) : undefined;
}
/**
* Observe-only snapshot of the in-RAM continuation store, surfaced via GET /api/system/memory.
* Additive and side-effect free — it does NOT lazy-load the disk snapshot, prune, or evict — so a
* diagnostics probe can sample it without perturbing request handling. `totalBytes` reads the
* running byte counter and `largestBytes` reads each entry's cached `sizeBytes`, so a probe never
* re-serializes the whole store (a large transient allocation that would fire exactly when memory
* is already under pressure). This is the seam for deciding whether RAM growth originates in this
* store (JS heap) or in the runtime allocator (native).
*/
export function responseStateMetrics(): ResponseStateMetrics {
return computeResponseStateMetrics({
states: states.values(),
stateCount: states.size,
now: now(),
totalBytes: responseContinuationRetainedStoreSnapshot().bytes,
spillWrites: spillCounters.writes,
spillWriteFailures: spillCounters.writeFailures,
spillWriteConsecutiveFailures: spillWriteHealth.consecutiveFailures,
spillLastFailureCode: spillWriteHealth.lastFailureCode,
spillLastFailureOrigin: spillWriteHealth.lastFailureOrigin,
spillLastSuccessAt: spillWriteHealth.lastSuccessAt,
spillAclRetryReturnedTimeouts: spillCounters.aclRetryReturnedTimeouts,
spillAclTimeoutMemoRefusals: spillCounters.aclTimeoutMemoRefusals,
spillCapacityRefusals: spillCounters.capacityRefusals, spillHeadroomEvictions: spillCounters.headroomEvictions,
spillLastFailureAt: spillWriteHealth.lastFailureAt,
spillReadFailures: spillCounters.readFailures,
replayScopeMismatchDrops,
});
}
/**
* Read-only spill report for `ocx doctor` and /api/system/memory. Owned = live
* stubs, deferred unlinks, queued publications, and snapshot references (what a
* restart re-owns). Shares the reclaim's orphan predicate; never unlinks.
*/
export function inspectResponseSpillStorage(): ResponseSpillDirInspection {
const referenced = ownedSpillFileNames();
for (const name of snapshotReferencedSpillFileNames(snapshotPath(), SNAPSHOT_FILE_MAX_BYTES)) {
referenced.add(name);
}
return inspectResponseSpillDir(referenced);
}
/**
* Cache completed output and max_output_tokens partial output for previous_response_id replay.
* Content-filtered incomplete and failed output are not authoritative replay history.
*/
export function rememberResponseState(
requestBody: unknown,
response: { id?: unknown; output?: unknown; status?: unknown; incomplete_details?: unknown },
providerState?: OcxProviderContinuationState | string,
opts?: { force?: boolean; clientThreadId?: string; retainForToolContinuation?: boolean; replayOwnerKey?: string | null; compiledPlaintext?: boolean; connectionScope?: ConnectionReplayScope },
): void {
if (!requestBody || typeof requestBody !== "object" || Array.isArray(requestBody)) return;
const request = requestBody as Record<string, unknown>;
if (isBodyNonPersistable(request)) return;
// `force` bypasses only the store:false skip: Codex sends `store:false` on every non-Azure
// HTTP request (and WS inherits it), yet its WS turns still chain with previous_response_id.
// The passthrough branch records with force so those chains can be expanded locally; the
// store stays in-memory under RESPONSE_TTL_MS, so this is a proxy-internal continuation cache, not
// real server-side response storage.
if (typeof response.id !== "string" || !Array.isArray(response.output)) return;
// Only an explicitly opted-in adapter may retain pending client calls despite store:false.
const unforcedStoreFalse = request.store === false && !opts?.force;
if (unforcedStoreFalse && !(opts?.retainForToolContinuation === true && hasPendingClientToolCall(response.output))) return;
if (response.status === "incomplete") {
const details = response.incomplete_details;
if (!details || typeof details !== "object" || Array.isArray(details)
|| (details as { reason?: unknown }).reason !== "max_output_tokens") return;
} else if (response.status !== undefined && response.status !== "completed") return;
if ((opts?.compiledPlaintext && !validReplayOwnerKey(opts.replayOwnerKey)) || opts?.replayOwnerKey === null || (opts?.replayOwnerKey !== undefined && !validReplayOwnerKey(opts.replayOwnerKey))) return;
const connectionScope = opts?.connectionScope;
if (connectionScope && !connectionReplayScopeAcceptsEntry(connectionScope, response.id, opts?.replayOwnerKey, opts?.clientThreadId)) return;
if (!connectionScope && !ensureLoaded()) return;
const existing = states.get(response.id);
// Never let a colliding upstream id replace a different caller's retained response.
if (existing && existing.kind !== "spill-failed" && !replayScopeMatches(existing, opts?.clientThreadId, opts?.replayOwnerKey)
&& !(!connectionScope && validReplayOwnerKey(opts?.replayOwnerKey) && isConnectionOwnedResponseState(existing))) return;
const normalizedProviderState: OcxProviderContinuationState = typeof providerState === "string"
? { cursor: { conversationId: providerState } }
: structuredClone(providerState ?? {});
if (normalizedProviderState.cursor?.conversationId) {
normalizedProviderState.cursor.checkpointUsable = !response.output.some(item => {
return !!item && typeof item === "object" && (item as { type?: unknown }).type === "function_call";
});
}
const clientThreadId = normalizedClientThreadId(opts?.clientThreadId);
// Compute the normalized array once and reuse it for both fields, so the recorded
// boundary can never disagree with the items it indexes.
const requestItems = inputItems(request.input);
setResidentEntry(response.id, {
createdAt: now(),
...(clientThreadId ? { clientThreadId } : {}),
...(opts?.replayOwnerKey ? { replayOwnerKey: opts.replayOwnerKey } : {}),
items: [...requestItems, ...response.output],
...(connectionScope ? { connectionOwned: true as const, connectionScope } : {}),
// Where response.output begins. A replay skip requires a matched item at or past this
// index that also carries a provider-issued id — position alone proves only that an item
// sits on the provider side, not that the provider authored it.
providerOutputStart: requestItems.length,
// Always preserve the Cursor conversation id so the next tool-result turn can continue the SAME
// Cursor conversation (multi-turn continuation). Separately track whether Cursor's own
// checkpoint/cache is safe to reuse: a turn that ended with a pending client tool call produced an
// incomplete agent turn on the Cursor side (we suspended without a real mcpResult), so its
// checkpoint must not be reused — but the conversation id string itself is still valid.
...(Object.keys(normalizedProviderState).length > 0 ? { providers: normalizedProviderState } : {}),
...(unforcedStoreFalse ? { unforcedStoreFalse: true } : {}),
});
if (!connectionScope) enforceAppOwnedMemoryBudget();
}
/** Test-only persistence churn hook; invoked after each atomic snapshot rewrite. */
export function setResponseStatePersistAttemptHookForTests(hook: (() => void) | null): void {
persistAttemptHookForTests = hook;
}
/** Test-only: deterministically run the pending background debounce pass. */
export async function runPendingResponseStatePersistForTests(): Promise<void> {
if (!persistTimer) return;
await persistNow(pendingPersistPath ?? snapshotPath());
}
/** Test-only: observe whether a debounce/follow-up pass is pending. */
export function responseStatePersistPendingForTests(): boolean {
return persistTimer !== null;
}
/** Memory-only reset (simulates a process restart: the snapshot file survives). */
export function clearResponseStateMemoryForTests(): void {
if (persistTimer) {
clearTimeout(persistTimer);
persistTimer = null;
}
pendingPersistPath = null;
resetSpillQueueForTests();
resetPeriodicSpillSweepCursorForTests();
for (const [id, entry] of states) forgetConnectionScopeEntry(id, entry);
states.clear();
storedResponseBytes = 0;
residentResponseBytes = 0;
budgetCandidate = undefined;
durableRevision = 0;
pendingSpillUnlinks.length = 0;
resetSpillCountersForTests();
spillWriteHealth.consecutiveFailures = 0;
spillWriteHealth.lastFailureCode = null;
spillWriteHealth.lastFailureOrigin = null;
spillWriteHealth.lastFailureAt = null;
spillWriteHealth.lastSuccessAt = null;
replayScopeMismatchDrops = 0;
replayOverlapSkips = 0;
persistAttemptHookForTests = null;
lastSnapshotBytes = 0;
lastSnapshotDigest = null;
lastSnapshotTarget = null;
loaded = false;
coldSnapshotAdmission = undefined;
}
export function clearResponseStateForTests(): void {
for (const entry of states.values()) deleteOwnedSpills(entry);
clearResponseStateMemoryForTests();
try {
unlinkSync(snapshotPath());
} catch {
/* no snapshot on disk */
}
try { rmSync(responseSpillDirectory(), { recursive: true, force: true }); } catch { /* no spill directory */ }
}