@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
1,370 lines • 60.5 kB
text/typescript
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, unlinkSync } from "node:fs";
import { dirname, join } from "node:path";
import { atomicWriteFileAsync, getConfigDir, resolveWriteTarget } from "../config";
import { enforceAppOwnedMemoryBudget, type RetainedStoreSnapshot } from "../lib/app-owned-memory";
import { windowsSecretAclApplies } from "../lib/windows-secret-acl";
import type { OcxProviderContinuationState } from "../types";
import {
deleteResponseSpill,
inspectResponseSpillDir,
noteStubSwapForTest,
readResponseSpill,
recoverOrphanedResponseSpills,
resetPeriodicSpillSweepCursorForTests,
responseSpillDirectory,
responseSpillPayloadCap,
sweepOrphanedResponseSpillsPeriodically,
type ResponseSpillDirInspection,
type ResponseSpillRef,
writeResponseSpillDurably,
} from "./spill-store";
import { collectReferencedSpillFileNames, snapshotReferencedSpillFileNames } from "./state/spill-inspect";
import { selectSnapshotEntries } from "./state/snapshot-select";
import { clientCarriedPrefixLength, providerIssuedIdentity } from "./state/replay-fingerprint";
export type { ResponseStateTempRecoveryResult, ResponseStateTempRecoveryOptions } from "./state/temp-recovery";
export type { ResponseSpillDirInspection } from "./spill-store";
export { recoverStaleResponseStateTemps, reclaimAbandonedResponseStateTemps, inspectAbandonedResponseStateTemps, sweepAbandonedResponseStateTemps } from "./state/temp-recovery";
import { recoverStaleResponseStateTemps } from "./state/temp-recovery";
export type { ResponseSpillWriteFailureCode, ResponseSpillWriteStatus, ResponseSpillWriteFailureOrigin } from "./state/spill-failure";
import type { ResponseSpillWriteFailureCode, ResponseSpillWriteStatus, ResponseSpillWriteFailureOrigin } from "./state/spill-failure";
export { responseAdmissionCountersForTests } from "./state/spill-failure";
import { admissionCounters, noteSpillWriteFailure, noteSpillWriteSuccess, spillCounters, spillWriteHealth } from "./state/spill-failure";
import { loadSnapshotEntry } from "./state/snapshot-codec";
import { isBodyNonPersistable } from "./state/body-policy";
export { isBodyNonPersistable, markBodyNonPersistable } from "./state/body-policy";
export { flushPendingResponseSpillsForTests, awaitResponseSpillPublicationTailForTests, pendingResponseSpillMetricsForTests, setResponseSpillShutdownBudgetForTests, setResponseSpillAsyncAclAttemptBudgetForTests, setResponseSpillShutdownTerminalizationPassLimitForTests } from "./state/spill-queue";
import {
bindSpillQueueStore,
cancelPendingResponseSpill,
drainResponseSpillPublications,
queuePendingResponseSpill,
replaceWithPendingResponseSpill,
resetSpillQueueForTests,
spillQueueAccounting,
spillQueueHoldsResidentCandidate,
spillQueuePendingBytes,
spillQueueReferencedSpillFileNames,
spillQueueResidentCandidates,
spillQueueSupersededSpillFor,
} from "./state/spill-queue";
const MAX_STORED_RESPONSES = 1_000;
/**
* Retention for locally replayed continuation state.
*
* A Codex client chained by `previous_response_id` sends ONLY the new turn and expects this
* process to hold everything before it, so this constant is the practical memory span of every
* conversation that does not go to the canonical ChatGPT backend. At the original one hour, a
* session resumed after lunch expanded to nothing and the delta — one user line — was all the
* provider ever saw, which reads to the operator as the model losing the conversation.
*
* A day is safe to hold because retention is no longer what bounds this store: the resident cap
* (MAX_STORED_RESPONSE_BYTES), the spill ceiling (MAX_SPILLED_RESPONSE_BYTES) and the entry count
* all evict oldest-first, and every turn re-stores the whole chain under a fresh id, so the live
* conversation is the last thing any of those three caps would drop. Raising the TTL therefore
* moves eviction from the clock to those budgets rather than growing the ceiling.
*/
export const RESPONSE_TTL_MS = 24 * 60 * 60 * 1_000;
const SNAPSHOT_DEBOUNCE_MS = 2_000;
/** Snapshot size below which the debounce stays at its base value. */
const SNAPSHOT_DEBOUNCE_SCALE_FROM_BYTES = 1 * 1024 * 1024;
/** Ceiling for the stretched debounce. Continuation state is only read after a
* restart, and a graceful shutdown flushes, so the exposure a longer debounce adds
* is bounded by a hard kill — paid against rewriting the whole snapshot every 2 s. */
const SNAPSHOT_DEBOUNCE_MAX_MS = 30_000;
/** In-memory high-water byte cap across all entries. Forced store:false retention (kiro/cursor
* continuation chains) stores the full expanded input each turn — ~quadratic bytes per chain —
* so a count cap alone cannot bound memory. Oldest-first eviction applies past this mark. */
export const MAX_STORED_RESPONSE_BYTES = 64 * 1024 * 1024;
/**
* Aggregate ceiling for the durable spill directory: the disk-side counterpart to
* the RAM ceiling above. Without it the spilled set is bounded only per-file
* (MAX_RESPONSE_SPILL_PAYLOAD_BYTES, 256 MiB) and per-entry (MAX_STORED_RESPONSES,
* 1000), whose product is 250 GiB — larger than the disk of any host this runs on.
* The only effective bound was therefore RESPONSE_TTL_MS, which makes disk use a
* function of client request rate rather than of anything this process controls.
*
* Measured on one macOS host, 2026-08-30: a client spilling ~150 MB payloads at
* ~1.4/min held 6.8 GB after 44 minutes, still climbing toward the ~12 GB an
* hour-long window implies, and filled the volume. Retention itself was correct
* throughout — the TTL evicted that whole cohort an hour later — so what was
* missing is a budget, not a sweep.
*
* 1 GiB comes from the same sample (n=31), whose spilled sizes are strongly
* bimodal: median 1.1 MiB against a p90 of 198.7 MiB, near the per-file ceiling.
* At that median the count cap and this ceiling bind within 8% of each other
* (1000 x 1.1 MiB = 1.07 GiB), so ordinary traffic sees no eviction it would not
* already have seen and only the large tail is cut. Erring small is the safe
* direction: too low costs a replay miss, an already-handled path surfaced as
* previous_response_not_found, while too high costs the host's disk and every
* unrelated process on it.
*/
export const MAX_SPILLED_RESPONSE_BYTES = 1024 * 1024 * 1024;
/** Legacy snapshot selection only. Spill demotion is governed solely by the RAM cap above. */
const SNAPSHOT_ENTRY_MAX_BYTES = 2 * 1024 * 1024;
const SNAPSHOT_TOTAL_MAX_BYTES = 24 * 1024 * 1024;
/** Refuse-to-parse ceiling for an existing snapshot file (above the 24 MiB write
* bound, so anything we wrote ourselves always loads; guards against externally
* planted or pre-cap unbounded files being parsed whole). */
const SNAPSHOT_FILE_MAX_BYTES = 32 * 1024 * 1024;
const MAX_SNAPSHOT_REWRITE_ATTEMPTS = 4;
const RESPONSE_SPILL_SHUTDOWN_TERMINALIZATION_MAX_PASSES = MAX_STORED_RESPONSES + 1;
export interface ResidentResponseState {
kind: "resident";
createdAt: number;
clientThreadId?: string;
items: unknown[];
/** Index in `items` where provider output begins; see clientCarriedPrefixLength. */
providerOutputStart?: number;
providers?: OcxProviderContinuationState;
sizeBytes: number;
}
export interface SpilledResponseState {
kind: "spill";
createdAt: number;
clientThreadId?: string;
/** Mirrors the spilled payload boundary so a spilled entry keeps its anchor. */
providerOutputStart?: number;
providers?: OcxProviderContinuationState;
spill: ResponseSpillRef;
sizeBytes: number;
}
export interface SpillFailedResponseState {
kind: "spill-failed";
createdAt: number;
sizeBytes: number;
}
export type StoredResponseState = ResidentResponseState | SpilledResponseState | SpillFailedResponseState;
export type ResidentInput = Omit<ResidentResponseState, "kind" | "sizeBytes">;
export type PreviousResponseReplayFailure = {
code: "previous_response_not_found";
reason: "spill_missing" | "spill_corrupt" | "spill_failed" | "spill_too_large" | "scope_mismatch";
};
const states = new Map<string, StoredResponseState>();
let storedResponseBytes = 0;
let residentResponseBytes = 0;
let oldestResidentId: string | undefined;
let oldestResidentAt: number | null = null;
let byteCapOverride: number | null = null;
let snapshotTotalCapOverride: number | null = null;
let stateRevision = 0;
/** Byte length and digest of the last snapshot actually written, for the
* identical-payload skip and the size-scaled debounce. The payload itself is not
* retained: at the 24 MiB bound that would double the snapshot's memory cost. */
let lastSnapshotBytes = 0;
let lastSnapshotDigest: string | null = null;
// The resolved file the digest above describes. Keeping it means a config-dir
// change or a retargeted symlink is a miss rather than a false "unchanged".
let lastSnapshotTarget: string | null = null;
/**
* Is the snapshot on disk still byte-for-byte what we last wrote?
*
* The cached digest proves what this process wrote, not what is there now. Size is
* checked first so the common mismatch costs a `stat`, and the content comparison
* only runs when the size already agrees. Any read failure answers "no" and the
* caller rewrites — the safe direction.
*/
async function snapshotOnDiskMatches(path: string, payload: string, payloadBytes: number): Promise<boolean> {
try {
const file = Bun.file(path);
if (file.size !== payloadBytes) return false;
if (await file.text() !== payload) return false;
// Content matching is not the whole invariant. This file holds persisted request
// and response bodies, and `atomicWriteFileAsync` writes it owner-only; the
// unconditional rewrite used to restore that on every mutation. Skipping without
// checking would let a broadened mode persist indefinitely, so treat a widened
// file as "does not match" and let the caller rewrite it through the hardening
// path. POSIX only — Windows ACLs are re-applied by that same write path.
if (process.platform !== "win32") {
const mode = statSync(path).mode & 0o777;
if (mode !== 0o600) return false;
}
return true;
} catch {
return false;
}
}
let replayScopeMismatchDrops = 0;
// Superseded spill generations awaiting a durable snapshot before unlink
// (review C1-1: unlinking at swap time races a crash against the debounced
// snapshot — the reloaded OLD stub would point at a deleted file).
const pendingSpillUnlinks: ResponseSpillRef[] = [];
// The queue itself must stay bounded (review C2-2: repeated replacements with
// a persistently failing snapshot write would otherwise grow it without
// limit). Beyond the cap the OLDEST superseded generation is unlinked
// immediately: the accepted worst case is that a crash inside that window
// reloads a stub whose file is gone, which fails replay with the explicit
// structured 400 — bounded-loss, never silent corruption or unbounded disk.
const PENDING_SPILL_UNLINKS_MAX = 128;
function deferSupersededSpill(ref: ResponseSpillRef | undefined): void {
if (!ref) return;
pendingSpillUnlinks.push(ref);
while (pendingSpillUnlinks.length > PENDING_SPILL_UNLINKS_MAX) {
deleteResponseSpill(pendingSpillUnlinks.shift()!);
}
}
function byteCap(): number {
return byteCapOverride ?? MAX_STORED_RESPONSE_BYTES;
}
/** Test-only: lower/restore the in-memory byte cap (null restores the default). */
export function setResponseStateByteCapForTests(bytes: number | null): void {
byteCapOverride = bytes;
}
function snapshotTotalBytes(): number {
return snapshotTotalCapOverride ?? SNAPSHOT_TOTAL_MAX_BYTES;
}
/** Test-only: lower/restore the durable snapshot byte budget (null restores the default). */
export function setResponseStateSnapshotByteCapForTests(bytes: number | null): void {
snapshotTotalCapOverride = bytes;
}
/** Test-only: current in-memory byte accounting (proves evictions release their bytes). */
export function getStoredResponseBytesForTests(): number {
return storedResponseBytes;
}
let spillByteCapOverride: number | null = null;
function spillByteCap(): number {
return spillByteCapOverride ?? MAX_SPILLED_RESPONSE_BYTES;
}
/**
* Live total of durable spill payloads. Recomputed per call rather than carried as
* a running counter: spilled entries reach `states` through several insertion paths
* (demotion swap, direct oversized admission, snapshot reload), and one missed
* increment there would silently disable the cap, where an O(MAX_STORED_RESPONSES)
* walk cannot drift.
*/
function spilledResponseBytes(): number {
let total = 0;
for (const entry of states.values()) {
if (entry.kind === "spill") total += entry.spill.payloadBytes;
}
// Superseded generations awaiting a durable snapshot are still files on disk.
// Counting only `states` would let PENDING_SPILL_UNLINKS_MAX of them sit outside
// the budget while it reports itself satisfied.
for (const ref of pendingSpillUnlinks) total += ref.payloadBytes;
return total;
}
/**
* Accounted on-disk bytes: files that exist, plus the peak footprint of publications
* already in flight.
*
* The cap is enforced against this rather than against `spilledResponseBytes()` alone,
* because a publication that has not finished is still consuming the volume. On Windows
* the gap between "queued" and "installed" is however long `icacls` takes, and the
* measured incident this cap answers accumulated 6.8 GiB in 44 minutes.
*/
function accountedResponseSpillBytes(): number {
// Superseded generations a pending job still owns are files on disk too. A same-id
// replacement removes the old spill from `states` and hands its ref to the job, so
// counting only `states` plus `pendingSpillUnlinks` loses it for the whole publication
// — during a copy fallback that is old generation + new temp + new destination, three
// envelopes priced as two.
const accounting = spillQueueAccounting();
return spilledResponseBytes() + accounting.reservedBytes + accounting.jobOwnedBytes
+ accounting.unreclaimableBytes;
}
/** Test-only: lower/restore the durable spill cap (null restores the default). */
export function setSpilledResponseByteCapForTests(bytes: number | null): void {
spillByteCapOverride = bytes;
}
/** Test-only: current durable spill accounting (proves evictions unlink their files). */
export function getSpilledResponseBytesForTests(): number {
return spilledResponseBytes();
}
/** Test-only: on-disk bytes plus in-flight publication reservations. */
export function getAccountedResponseSpillBytesForTests(): number {
return accountedResponseSpillBytes();
}
function serializedBytes(value: unknown): number | null {
try {
const serialized = JSON.stringify(value);
return serialized === undefined ? null : Buffer.byteLength(serialized, "utf8");
} catch {
return null;
}
}
function measureResidentEntry(id: string, entry: ResidentInput): ResidentResponseState | null {
const sizeBytes = serializedBytes({
responseId: id,
createdAt: entry.createdAt,
...(entry.clientThreadId ? { clientThreadId: entry.clientThreadId } : {}),
items: entry.items,
...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}),
...(entry.providers ? { providers: entry.providers } : {}),
});
return sizeBytes === null ? null : { kind: "resident", ...entry, sizeBytes };
}
function recomputeOldestResident(): void {
oldestResidentId = undefined;
oldestResidentAt = null;
for (const [id, state] of states) {
if (state.kind !== "resident") continue;
if (spillQueueHoldsResidentCandidate(id, state)) continue;
if (oldestResidentAt !== null && state.createdAt >= oldestResidentAt) continue;
oldestResidentId = id;
oldestResidentAt = state.createdAt;
}
}
function replaceMapEntry(id: string, next: StoredResponseState, expected?: StoredResponseState): boolean {
const existing = states.get(id);
if (expected && existing !== expected) return false;
storedResponseBytes -= existing?.sizeBytes ?? 0;
storedResponseBytes += next.sizeBytes;
if (existing?.kind === "resident") {
residentResponseBytes -= existing.sizeBytes;
}
if (next.kind === "resident") {
residentResponseBytes += next.sizeBytes;
}
if (storedResponseBytes < 0) storedResponseBytes = 0;
if (residentResponseBytes < 0) residentResponseBytes = 0;
if (existing) states.delete(id);
states.set(id, next);
if (oldestResidentId === id) {
recomputeOldestResident();
} else if (next.kind === "resident" && (oldestResidentAt === null || next.createdAt < oldestResidentAt)) {
oldestResidentId = id;
oldestResidentAt = next.createdAt;
}
stateRevision += 1;
return true;
}
function stubSize(id: string, entry: Omit<SpilledResponseState, "sizeBytes">): number {
return serializedBytes({ responseId: id, ...entry }) ?? 0;
}
function tombstone(id: string, createdAt: number): SpillFailedResponseState {
const base = { kind: "spill-failed" as const, createdAt };
return { ...base, sizeBytes: serializedBytes({ responseId: id, ...base }) ?? 0 };
}
function deleteOwnedSpills(entry: StoredResponseState): void {
if (entry.kind === "spill") deleteResponseSpill(entry.spill);
}
/** The ONLY deletion point: TTL, count, byte, and explicit deletes all route here. */
function deleteEntry(id: string, options: { deleteSpill?: boolean } = {}): void {
const existing = states.get(id);
if (!existing) return;
const supersededSpill = cancelPendingResponseSpill(id);
storedResponseBytes -= existing.sizeBytes;
if (existing.kind === "resident") {
residentResponseBytes -= existing.sizeBytes;
}
if (storedResponseBytes < 0) storedResponseBytes = 0;
if (residentResponseBytes < 0) residentResponseBytes = 0;
states.delete(id);
if (oldestResidentId === id) recomputeOldestResident();
stateRevision += 1;
if (options.deleteSpill !== false) deleteOwnedSpills(existing);
if (options.deleteSpill !== false && supersededSpill) deleteResponseSpill(supersededSpill);
}
function replaceWithSpillFailure(
id: string,
expected?: StoredResponseState,
options: { deferSpillUnlink?: boolean } = {},
): void {
const existing = states.get(id);
if (expected && existing !== expected) return;
const failed = tombstone(id, expected?.createdAt ?? existing?.createdAt ?? now());
if (replaceMapEntry(id, failed, expected)) {
if (existing) {
if (options.deferSpillUnlink && existing.kind === "spill") {
// Crash consistency (same rule as replaceSpillEntryAtomically): the old
// durable snapshot still references this generation until the tombstone
// itself is durable — queue the unlink for the next stable persist.
pendingSpillUnlinks.push(existing.spill);
while (pendingSpillUnlinks.length > PENDING_SPILL_UNLINKS_MAX) {
deleteResponseSpill(pendingSpillUnlinks.shift()!);
}
} else {
deleteOwnedSpills(existing);
}
}
}
}
function swapResidentForSpill(id: string, expected: ResidentResponseState, ref: ResponseSpillRef): boolean {
const base: Omit<SpilledResponseState, "sizeBytes"> = {
kind: "spill",
createdAt: expected.createdAt,
...(expected.clientThreadId ? { clientThreadId: expected.clientThreadId } : {}),
...(expected.providers ? { providers: expected.providers } : {}),
spill: ref,
};
const next: SpilledResponseState = { ...base, sizeBytes: stubSize(id, base) };
if (!replaceMapEntry(id, next, expected)) {
deleteResponseSpill(ref);
return false;
}
noteStubSwapForTest();
return true;
}
function replaceSpillEntryAtomically(
id: string,
expected: SpilledResponseState,
candidate: ResidentResponseState,
): void {
try {
const ref = writeResponseSpillDurably(id, {
createdAt: candidate.createdAt,
...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}),
items: candidate.items,
...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}),
...(candidate.providers ? { providers: candidate.providers } : {}),
});
const base: Omit<SpilledResponseState, "sizeBytes"> = {
kind: "spill",
createdAt: candidate.createdAt,
...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}),
...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}),
...(candidate.providers ? { providers: candidate.providers } : {}),
spill: ref,
};
const next: SpilledResponseState = { ...base, sizeBytes: stubSize(id, base) };
if (!replaceMapEntry(id, next, expected)) {
deleteResponseSpill(ref);
return;
}
noteSpillWriteSuccess();
noteStubSwapForTest();
// The old generation is NOT unlinked here (review C1-1): the new stub is
// only durable once the debounced snapshot flushes — a crash before that
// reloads the OLD stub, which must still find its file. Queue the unlink;
// persistNow() drains the queue only after the snapshot write succeeds.
pendingSpillUnlinks.push(expected.spill);
while (pendingSpillUnlinks.length > PENDING_SPILL_UNLINKS_MAX) {
deleteResponseSpill(pendingSpillUnlinks.shift()!);
}
} catch (error) {
noteSpillWriteFailure(error);
// deferSpillUnlink: the durable snapshot may still reference the old
// generation; deleting it now would strand the old stub after a crash.
replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
}
}
function setResidentEntry(id: string, entry: ResidentInput): void {
const expected = states.get(id);
const candidate = measureResidentEntry(id, entry);
if (!candidate) {
replaceWithSpillFailure(id, expected);
// A tombstone is tiny but still resident state: the hard-cap invariant
// must hold on EVERY mutation path (review C2-1 — with a test cap below
// tombstone size, skipping the prune leaves the store over cap).
pruneResponses();
return;
}
if (candidate.sizeBytes > byteCap()) {
admitOversizedCandidate(id, candidate, expected);
pruneResponses();
return;
}
if (windowsSecretAclApplies() && (expected?.kind === "spill" || spillQueueSupersededSpillFor(id))) {
replaceWithPendingResponseSpill(id, candidate, expected);
pruneResponses();
return;
}
if (expected?.kind === "spill") {
replaceSpillEntryAtomically(id, expected, candidate);
pruneResponses();
return;
}
if (windowsSecretAclApplies()) cancelPendingResponseSpill(id);
if (!replaceMapEntry(id, candidate, expected)) return;
pruneResponses();
}
/**
* Admission boundary for candidates that can never fit as resident (larger
* than the whole resident-map cap). Writes them DIRECTLY to durable spill and
* installs only the stub — the oversized candidate never becomes resident and
* no unrelated resident is demoted to make room for it. Candidates above the
* single-spill payload ceiling are tombstoned instead: retaining a spill the
* replay ceiling would refuse to read is write-only waste.
*/
function admitOversizedCandidate(
id: string,
candidate: ResidentResponseState,
expected?: StoredResponseState,
): void {
if (candidate.sizeBytes > responseSpillPayloadCap()) {
admissionCounters.oversizedDrops += 1;
replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
return;
}
if (windowsSecretAclApplies()) {
replaceWithPendingResponseSpill(id, candidate, expected, { directAdmission: true });
return;
}
try {
const ref = writeResponseSpillDurably(id, {
createdAt: candidate.createdAt,
...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}),
items: candidate.items,
...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}),
...(candidate.providers ? { providers: candidate.providers } : {}),
});
// Enforce the ceiling against the REAL envelope: the spill payload adds
// the {version, responseId, ...} wrapper, so a candidate within the
// wrapper's size of the cap would otherwise be retained unreadably.
if (ref.payloadBytes > responseSpillPayloadCap()) {
deleteResponseSpill(ref);
admissionCounters.oversizedDrops += 1;
replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
return;
}
const base: Omit<SpilledResponseState, "sizeBytes"> = {
kind: "spill",
createdAt: candidate.createdAt,
...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}),
...(candidate.providers ? { providers: candidate.providers } : {}),
spill: ref,
};
const next: SpilledResponseState = { ...base, sizeBytes: stubSize(id, base) };
if (!replaceMapEntry(id, next, expected)) {
deleteResponseSpill(ref);
return;
}
noteSpillWriteSuccess();
admissionCounters.directSpills += 1;
noteStubSwapForTest();
if (expected?.kind === "spill") {
// Same deferred-unlink rule as replaceSpillEntryAtomically: the new stub
// is durable only after the debounced snapshot, so the old generation
// stays until a stable persist drains the queue.
pendingSpillUnlinks.push(expected.spill);
while (pendingSpillUnlinks.length > PENDING_SPILL_UNLINKS_MAX) {
deleteResponseSpill(pendingSpillUnlinks.shift()!);
}
}
} catch (error) {
noteSpillWriteFailure(error);
replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
}
}
bindSpillQueueStore({
swapResidentForSpill,
replaceWithSpillFailure,
deleteEntry,
deferSupersededSpill,
replaceMapEntry,
currentEntry: (id: string) => states.get(id),
residentEntries: () => [...states],
recomputeOldestResident,
schedulePersist,
pruneResponses,
accountedResponseSpillBytes,
spillByteCap,
enforceSpilledResponseBudget,
terminalizationMaxPasses: () => RESPONSE_SPILL_SHUTDOWN_TERMINALIZATION_MAX_PASSES,
});
// Replay provenance must stay proxy-private: a WeakMap distinguishes replayed history from the
// newly appended input suffix without adding an unknown field that native passthrough could send
// upstream. The parser uses this boundary to acknowledge historical compaction markers exactly
// once. It records the boundary whether the proxy prepended the history or the client already
// carried it — the boundary is the same either way, and only its provenance differs.
const replayedInputPrefixLengths = new WeakMap<object, number>();
const replayFailures = new WeakMap<object, PreviousResponseReplayFailure>();
let loaded = false;
let persistTimer: ReturnType<typeof setTimeout> | null = null;
let pendingPersistPath: string | null = null;
/** Single-flight gate: overlapping response-state writes serialize (#612). */
let persistGate: Promise<void> = Promise.resolve();
let persistAttemptHookForTests: (() => void) | null = null;
function now(): number {
return Date.now();
}
function snapshotPath(): string {
return join(getConfigDir(), "responses-state.json");
}
/**
* Best-effort disk snapshot so previous_response_id chains survive a proxy restart (the
* dominant expansion-miss cause: an in-memory-only store dies with the process, and the next
* chained turn then reaches the upstream as a naked delta). Load is lazy on first store access;
* persistence is debounced + unref'd so the hot path never blocks and the process can exit.
* Every disk failure is swallowed — the snapshot is a cache, not a source of truth.
*/
function ensureLoaded(): void {
if (loaded) return;
loaded = true;
const path = snapshotPath();
// Atomic writes place their temp beside the RESOLVED target, so a symlinked
// snapshot (dotfiles-managed config dir) strands temps in the link's real
// directory where a scan of the literal config dir would never see them.
// Both locations are swept; they collapse to one when nothing is symlinked.
// resolveWriteTarget refuses a dangling link; snapshot loading stays independent.
let resolvedDir = dirname(path);
try {
resolvedDir = dirname(resolveWriteTarget(path));
} catch {
/* unresolvable link: sweep the literal dir only */
}
for (const dir of new Set([dirname(path), resolvedDir])) {
try {
recoverStaleResponseStateTemps(dir);
} catch {
/* best-effort cleanup only; snapshot loading must remain independent */
}
}
try {
if (existsSync(path)) {
// Bound the read BEFORE parse: the 24 MiB write cap constrains snapshots
// this process wrote, not a pre-existing oversized file. statSync follows
// symlinks deliberately — readFileSync below follows them too, so the
// size gate must measure the same target the read would.
const stat = statSync(path);
if (!stat.isFile()) {
// Symlink to a FIFO/device (e.g. /dev/zero): reading would block or
// return unbounded input. Only regular files are ever parsed.
} else if (stat.size > SNAPSHOT_FILE_MAX_BYTES) {
admissionCounters.snapshotOversizedRefusals += 1;
} else {
const raw = JSON.parse(readFileSync(path, "utf-8")) as { version?: unknown; states?: unknown };
if ((raw.version === 1 || raw.version === 2) && Array.isArray(raw.states)) {
for (const entry of raw.states) {
if (!Array.isArray(entry) || entry.length !== 2 || typeof entry[0] !== "string") continue;
loadSnapshotEntry(entry[0], entry[1], {
replaceMapEntry,
stubSize,
tombstone,
measureResidentEntry,
admitOversizedCandidate,
byteCap,
});
}
}
}
}
} catch {
/* missing/corrupt snapshot: start empty */
}
try { recoverOrphanedResponseSpills(ownedSpillFileNames()); } catch { /* best effort */ }
pruneResponses();
}
/** Every file name the process still needs: live stubs, deferred unlinks, and queued publications. */
function ownedSpillFileNames(): Set<string> {
const referenced = collectReferencedSpillFileNames(states.values(), pendingSpillUnlinks);
for (const name of spillQueueReferencedSpillFileNames()) referenced.add(name);
return referenced;
}
/** Liveness-tick counterpart of the orphan GC in ensureLoaded; a no-op until first load. */
export function sweepOrphanedResponseSpills(): number {
if (!loaded) return 0;
try {
return sweepOrphanedResponseSpillsPeriodically(ownedSpillFileNames()).removed;
} catch {
return 0;
}
}
type SnapshotWriteOutcome = "stable" | "unstable" | "failed";
async function writeBoundedSnapshot(path: string, attemptLimit: number): Promise<SnapshotWriteOutcome> {
// Serialize writers so concurrent flush + debounce cannot race on temps / ACL (#612).
const previous = persistGate;
let release!: () => void;
persistGate = new Promise<void>(resolve => { release = resolve; });
await previous;
try {
for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
const revision = stateRevision;
const entries = selectSnapshotEntries(states, snapshotTotalBytes(), SNAPSHOT_ENTRY_MAX_BYTES);
const payload = JSON.stringify({ version: 2, states: entries });
const payloadBytes = Buffer.byteLength(payload, "utf8");
const payloadDigest = Bun.hash(payload).toString(36);
// A mutation does not always change what gets persisted: entries past the
// per-entry or total byte bound are dropped from the selection, and spill
// demotion moves bytes out of it. Re-writing a byte-identical 24 MiB file
// buys nothing, so compare first — but the cached digest describes what THIS
// process last wrote, which is not the same claim as "that is what is on disk
// now". A second proxy sharing the home, or anything that rewrites the file
// in place, leaves the digest describing bytes that are gone. Before every
// release-of-a-write, the previous behaviour rewrote unconditionally and so
// repaired that silently; skipping without checking would turn a repaired
// snapshot into a lost one at the next restart.
//
// Verify against the file itself, keyed to the resolved target so a retargeted
// symlink is also a miss. Reading back a matching-size file costs far less
// than the atomic replace it avoids, and only happens when the digest already
// matched — the amplification this fixes is the repeated WRITE, not the read.
const unchanged = lastSnapshotDigest !== null
&& payloadDigest === lastSnapshotDigest
&& payloadBytes === lastSnapshotBytes
&& lastSnapshotTarget === resolveWriteTarget(path)
&& existsSync(path)
&& await snapshotOnDiskMatches(path, payload, payloadBytes);
if (!unchanged) {
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
try { chmodSync(dirname(path), 0o700); } catch { /* best-effort (e.g. Windows) */ }
await atomicWriteFileAsync(path, payload);
lastSnapshotDigest = payloadDigest;
lastSnapshotBytes = payloadBytes;
lastSnapshotTarget = resolveWriteTarget(path);
}
persistAttemptHookForTests?.();
if (revision === stateRevision) return "stable";
}
return "unstable";
} catch {
return "failed";
} finally {
release();
}
}
function drainPendingSpillUnlinks(): void {
while (pendingSpillUnlinks.length > 0) {
const ref = pendingSpillUnlinks.shift()!;
deleteResponseSpill(ref);
}
}
/**
* Debounce scaled by the size of the last snapshot written.
*
* The whole snapshot is re-serialized and atomically replaced on every flush, so at
* the 24 MiB bound a fixed 2 s debounce is up to ~12 MB/s of write amplification for
* state nothing reads until the next start (#2460). Small snapshots keep the base
* cadence; the stretch is linear in size and clamped, so the write rate is roughly
* flat instead of growing with the file.
*/
function snapshotDebounceMs(): number {
if (lastSnapshotBytes <= SNAPSHOT_DEBOUNCE_SCALE_FROM_BYTES) return SNAPSHOT_DEBOUNCE_MS;
const scaled = Math.round(SNAPSHOT_DEBOUNCE_MS * (lastSnapshotBytes / SNAPSHOT_DEBOUNCE_SCALE_FROM_BYTES));
return Math.min(scaled, SNAPSHOT_DEBOUNCE_MAX_MS);
}
function schedulePersistAt(path: string, replace = false): void {
if (persistTimer && !replace) return;
if (persistTimer) clearTimeout(persistTimer);
pendingPersistPath = path;
persistTimer = setTimeout(() => { void persistNow(path); }, snapshotDebounceMs());
(persistTimer as { unref?: () => void }).unref?.();
}
async function persistNow(path: string, awaitFollowUp = false): Promise<void> {
if (persistTimer) {
clearTimeout(persistTimer);
persistTimer = null;
}
pendingPersistPath = null;
const attemptLimit = awaitFollowUp ? MAX_SNAPSHOT_REWRITE_ATTEMPTS : 1;
let outcome = await writeBoundedSnapshot(path, attemptLimit);
if (outcome === "unstable" && awaitFollowUp) {
if (persistTimer) clearTimeout(persistTimer);
persistTimer = null;
pendingPersistPath = null;
outcome = await writeBoundedSnapshot(path, attemptLimit);
}
if (outcome === "stable") drainPendingSpillUnlinks();
else if (outcome === "unstable" && !awaitFollowUp) schedulePersistAt(path, true);
}
function schedulePersist(): void {
// Resolve the target path NOW: tests (and anything else) may swap OPENCODEX_HOME before the
// debounce fires, and a late write must land in the home that owned the recorded state.
schedulePersistAt(snapshotPath());
}
async function flushResponseSnapshot(): Promise<void> {
if (persistTimer) {
await persistNow(pendingPersistPath ?? snapshotPath(), true);
return;
}
// No pending timer: still await any in-flight write so shutdown does not race (#612).
await persistGate;
// A bounded background pass may have scheduled its same-path follow-up while
// this flush was waiting on the single-flight gate. Shutdown owns one awaited
// bounded follow-up rather than returning behind that unref'd timer.
if (persistTimer) await persistNow(pendingPersistPath ?? snapshotPath(), true);
}
/** Flush publications and snapshot state; report drain failure only after persistence completes. */
export async function flushResponseState(): Promise<void> {
const failures: unknown[] = [];
try {
await drainResponseSpillPublications();
} catch (error) {
failures.push(error);
}
try {
await flushResponseSnapshot();
} catch (error) {
failures.push(error);
}
if (failures.length === 1) throw failures[0];
if (failures.length > 1) throw new AggregateError(failures, "Response state shutdown flush incomplete");
}
function inputItems(input: unknown): unknown[] {
if (input === undefined) return [];
if (Array.isArray(input)) return input;
if (typeof input === "string") return [{ role: "user", content: input }];
return [input];
}
let replayOverlapSkips = 0;
/** Test-only: replay prepends skipped because the client already carried the history. */
export function replayOverlapSkipsForTests(): number {
return replayOverlapSkips;
}
/**
* Bring the durable spill set inside MAX_SPILLED_RESPONSE_BYTES, and report the
* bytes released.
*
* One owner, three callers: mutation pruning, the lazy load that follows a
* restart, and the periodic sweep. The periodic caller is not redundant — the
* mutation path only runs when traffic arrives, and a process can come up over
* budget from a snapshot written under a larger ceiling and then sit idle. That
* was observed in production at 1.8 GiB against a 1 GiB cap, held until the first
* request.
*
* NOT covered here: spill files orphaned by a crash. They are absent from
* `states`, so this function can neither see nor price them, and they stay with
* recoverOrphanedResponseSpills and its RESPONSE_SPILL_ORPHAN_GRACE_MS window.
* This ceiling therefore bounds what the store owns, which is every file it can
* account for, and not the directory as a whole.
*/
function enforceSpilledResponseBudget(): number {
// Price in-flight publications too: a file being created by
// `writeResponseSpillDurablyAsync` occupies the volume before it reaches `states`.
let spilledBytes = accountedResponseSpillBytes();
if (spilledBytes <= spillByteCap()) return 0;
const before = spilledBytes;
// Deferred generations go first. They are already superseded, so releasing one
// costs only the crash window the queue exists to cover — the same trade
// PENDING_SPILL_UNLINKS_MAX already makes against unbounded disk. Evicting a
// live continuation to make room for a dead file would be the wrong order.
while (spilledBytes > spillByteCap() && pendingSpillUnlinks.length > 0) {
const ref = pendingSpillUnlinks.shift()!;
spilledBytes -= ref.payloadBytes;
deleteResponseSpill(ref);
}
// Ordered by createdAt, not by map order. `states` is not an age index:
// demotion and spill replacement delete and reinsert entries, and
// writeBoundedSnapshot serializes the map reversed, so map order can put a
// newer continuation first — and evicting that one spends a resume the older
// entry would not have cost. Sorting is O(k log k) over the spilled subset and
// runs only on a tick already over budget.
const spilled = [...states]
.filter((pair): pair is [string, SpilledResponseState] => pair[1].kind === "spill")
// createdAt is millisecond-resolution, so ties are ordinary under load. A
// stable sort would then fall back to insertion order — the very order this
// is avoiding — so break ties on the response id. Not localeCompare: the
// order must not depend on the host locale.
.sort((a, b) => a[1].createdAt - b[1].createdAt
|| (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
for (const [id, entry] of spilled) {
if (spilledBytes <= spillByteCap()) break;
spilledBytes -= entry.spill.payloadBytes;
deleteEntry(id);
}
return before - spilledBytes;
}
function pruneResponses(at = now()): void {
for (const [id, state] of states) {
if (at - state.createdAt > RESPONSE_TTL_MS) deleteEntry(id);
}
while (states.size > MAX_STORED_RESPONSES) {
const oldest = states.keys().next().value;
if (!oldest) break;
deleteEntry(oldest);
}
// Unconditional RAM cap. Resident payloads demote durably; stubs/tombstones are
// deleted only when even their bounded metadata cannot fit the override.
while (storedResponseBytes > byteCap() && states.size > 0) {
const oldestResident = [...states].find(([id, entry]) => entry.kind === "resident"
&& !spillQueueHoldsResidentCandidate(id, entry));
const hasPendingResident = !oldestResident && [...states].some(([id, entry]) => entry.kind === "resident"
&& spillQueueHoldsResidentCandidate(id, entry));
if (hasPendingResident) break;
const oldestId = oldestResident?.[0] ?? states.keys().next().value as string | undefined;
if (!oldestId) break;
const entry = states.get(oldestId)!;
if (entry.kind !== "resident") {
deleteEntry(oldestId);
continue;
}
if (windowsSecretAclApplies()) {
queuePendingResponseSpill(oldestId, entry);
continue;
}
try {
const ref = writeResponseSpillDurably(oldestId, {
createdAt: entry.createdAt,
...(entry.clientThreadId ? { clientThreadId: entry.clientThreadId } : {}),
items: entry.items,
...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}),
...(entry.providers ? { providers: entry.providers } : {}),
});
if (swapResidentForSpill(oldestId, entry, ref)) noteSpillWriteSuccess();
} catch (error) {
noteSpillWriteFailure(error);
replaceWithSpillFailure(oldestId, entry);
}
}
enforceSpilledResponseBudget();
}
/** Periodic TTL-only sweep; count/byte eviction remains owned by mutation paths. */
export function sweepExpiredResponseStates(at = now()): number {
let removed = 0;
for (const [id, state] of states) {
if (at - state.createdAt <= RESPONSE_TTL_MS) continue;
deleteEntry(id);
removed += 1;
}
// The disk ceiling needs a caller that does not depend on traffic. The return
// value stays the TTL count so this function's existing contract is unchanged.
const reclaimed = enforceSpilledResponseBudget();
if (removed > 0 || reclaimed > 0) schedulePersist();
return removed;
}
export function responseContinuationRetainedStoreSnapshot(): RetainedStoreSnapshot {
let currentPendingBytes = 0;
for (const job of spillQueueResidentCandidates()) {
if (states.get(job.id) === job.candidate) currentPendingBytes += job.sizeBytes;
}
const detachedPendingBytes = Math.max(0, spillQueuePendingBytes() - currentPendingBytes);
const bytes = storedResponseBytes + detachedPendingBytes;
const evictableBytes = Math.max(0, residentResponseBytes - currentPendingBytes);
return {
count: states.size,
bytes,
evictableBytes,
pinnedBytes: Math.max(0, bytes - evictableBytes),
oldestAt: oldestResidentAt,
};
}
export function evictOldestResponseContinuationForBudget(): number {
if (oldestResidentId === undefined) return 0;
const id = oldestResidentId;
const entry = states.get(id);
if (!entry || entry.kind !== "resident") return 0;
if (windowsSecretAclApplies()) {
queuePendingResponseSpill(id, entry);
schedulePersist();
return 0;
}
try {
const ref = writeResponseSpillDurably(id, {
createdAt: entry.createdAt,
...(entry.clientThreadId ? { clientThreadId: entry.clientThreadId } : {}),
items: entry.items,
...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}),
...(entry.providers ? { providers: entry.providers } : {}),
});
if (swapResidentForSpill(id, entry, ref)) noteSpillWriteSuccess();
} catch (error) {
noteSpillWriteFailure(error);
replaceWithSpillFailure(id, entry);
}
schedulePersist();
const replacement = states.get(id);
return !replacement || replacement.kind === "resident"
? 0
: Math.max(0, entry.sizeBytes - replacement.sizeBytes);
}
function materializeEntry(
id: string,
entry: StoredResponseState,
): { ok: true; state: ResidentResponseState } | { ok: false; failure: PreviousResponseReplayFailure } {
if (entry.kind === "resident") return { ok: true, state: entry };
if (entry.kind === "spill-failed") {
return { ok: false, failure: { code: "previous_response_not_found", reason: "spill_failed" } };
}
const result = readResponseSpill(id, entry.spill);
if (!result.ok) {
spillCounters.readFailures += 1;
const failure: PreviousResponseReplayFailure = {
code: "previous_response_not_found",
reason: result.reason === "missing"
? "spill_missing"
: result.reason === "too_large"
? "spill_too_large"
: "spill_corrupt",
};
replaceWithSpillFailure(id, entry);
schedulePersist();
return { ok: false, failure };
}
const state = measureResidentEntry(id, {
createdAt: result.payload.createdAt,
...(result.payload.clientThreadId ? { clientThreadId: result.payload.clientThreadId } : {}),
items: result.payload.items,
...(result.payload.providerOutputStart !== undefined
? { providerOutputStart: result.payload.providerOutputStart }
: {}),
...(result.payload.providers ? { providers: result.payload.providers } : {}),
});
if (!state) {
spillCounters.readFailures += 1;
replaceWithSpillFailure(id, entry);
schedulePersist();
return { ok: false, failure: { code: "previous_response_not_found", reason: "spill_corrupt" } };
}
return { ok: true, state };
}
function normalizedClientThreadId(value: unknown): string | undefined {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
}
export function expandPreviousResponseInput(body: unknown, clientThreadId?: string): unknown {
if (!body || typeof body !== "object" || Array.isArray(body)) return body;
const request = body as Record<string, unknown>;
const previousId = typeof request.previous_response_id === "string" ? request.previous_response_id : undefined;
if (!previousId) return body;
ensureLoaded();
pruneResponses();
const previous = states.get(previousId);
if (!previous) return body;
const materialized = materializeEntry(previousId, previous);
if (!materialized.ok) {
replayFailures.set(request, materialized.failure);
return body;
}
const requestThreadId = normalizedClientThreadId(clientThreadId);
const storedThreadId = normalizedClientThreadId(materialized.state.clientThreadId);
// A Codex task must never inherit another task's continuation, nor a legacy unscoped entry.
// Unscoped callers retain backward-compatible replay only with other unscoped entries.
if (requestThreadId !== storedThreadId) {
replayFailures.set(request, { code: "previous_response_not_found", reason: "scope_mismatch" });
replayScopeMismatchDrops += 1;
return body;
}
// The client already replayed this history verbatim. Prepending the stored copy would
// double it, and the doubled turn is stored again, so the next turn triples (#1412 saw
// 127k of real context reach 1.3M tokens this way).
//
// Three conditions, all required. The run must cover the whole stored entry; it must reach
// the provider-output region; and some matched item in that region must carry a
// provider-issued id. The last one is the load-bearing part: content equality alone proves
// two items look alike, not that they are the same occurrence, so a client that merely
// repeats its own message would otherwise authorize a skip that deletes real history.
// There is no invariant that provider output always carries ids, so an entry whose output
// has none simply never skips.
{
const clientInput = inputItems(request.input);
const stored = materialized.state.items;
const anchor = materialized.state.providerOutputStart;
const carried = clientCarriedPrefixLength(stored, clientInput);
if (
carried === stored.length
&& anchor !== undefined
&& carried > anchor
&& stored.slice(anchor, carried).some(item => providerIssuedIdentity(item) !== null)
) {
replayOverlapSkips += 1;
// Keep previous_response_id: Kiro and Cursor recover their conversation ids from it
// (kiro-wire.ts, cursor/request-builder.ts). Only the concatenation is skipped.
const unchanged = { ...request };
// Same provenance boundary a real expansion would record, so the replayed prefix does
// not re-acknowledge historical compaction markers (parser.ts) and stays visible to
// guidance de-duplication (collaboration.ts).
replayedInputPrefixLengths.set(unchanged, carried);
return unchanged;
}
}
const expanded = {
...request,
input: [...materialized.state.items, ...inputItems(request.input)],
};
replayedInputPrefixLengths.set(expanded, materialized.state.items.length);
return expanded;
}
export function previousResponseReplayFailure(body: unknown): PreviousResponseReplayFailure | undefined {
if (!body || typeof body !== "object" || Array.isArray(body)) return undefined;
return replayFailures.get(body);
}
/** Number of leading input items restored from previous_response_id state for this exact body. */
export function previousResponseReplayPrefixLength(body: unknown): number {
if (!body || typeof body !== "object" || Array.isArray(body)) return 0;
return replayedInputPrefixLengths.get(body) ?? 0;
}
/** Copy proxy-private replay provenance to an internal clone with the same materialized input. */
export function copyPreviousResponseReplayProvenance(source: unknown, target: unknown): void {
if (!source || typeof source !== "object" || Array.isArray(source)) return;
if (!target || typeof target !== "object" || Array.isArray(target)) return;
const prefixLength = replayedInputPrefixLengths.get(source);
if (!prefixLength) return;
const input = (target as { input?: unknown }).input;
if (!Array.isArray(input) || prefixLength > input.length) return;
replayedInputPrefixLengths.set(target, prefixLength);
}
/** True when this exact request could not replay because its task scope did not match. */
export function previousResponseScopeMismatch(body: unknown): boolean {
return previousResponseReplayFailure(body)?.reason === "scope_mismatch";
}
export function previousResponseConversationId(responseId: string | undefined): string | undefined {
return previousResponseProviderState(responseId)?.cursor?.conversationId;
}
export function previousResponseProviderState(responseId: string | undefined): OcxProviderContinuationState | undefined {
if (!responseId) return undefined;
ensureLoaded();
pruneResponses();
const state = states.get(responseId);
const providers = state?.kind === "spill-failed" ? undefined : state?.providers;
return providers ? structuredClone(providers) : undefined;
}
export interface ResponseStateMetrics {
count: number;
residentCount: number;
spillStubCount: number;
tombstoneCount: number;
totalBytes: number;
spillPayloadBytes: number;
largestBytes: number;
oldestAgeMs: number;
spillWrites: number;
spillWriteFailures: number;
spillWriteStatus: ResponseSpillWriteStatus;
spillWriteConsecutiveFailures: number;
spillLastWriteFailureCode: ResponseSpillWriteFailureCode | null;
spillLastWriteFailureOrigin: ResponseSpillWriteFailureOrigin | null;
spillAclRetryReturnedTimeouts: number;
spillAclTimeoutMemoRefusals: number;
spillLastWriteFailureAt: number | null;
spillLastWriteSuccessAt: number | null;
spillReadFailures: number;
replayScopeMismatchDrops: number;
}
/**
* Observe-only snapshot of the in-RAM continuation store, surfaced via GET /api/system/memory.
* Additive and side-effect free — it does NOT lazy-load the disk snapshot, prune, or evict — so a
* diagnostics probe can sample it without perturbing request handling. `totalBytes` reads the
* running byte counter and `largestBytes` reads each entry's cached `sizeBytes`, so a probe never
* re-serializes the whole store (a large transient allocation that would fire exactly when memory
* is already under pressure). This is the seam for deciding whether RAM growth originates in this
* store (JS heap) or in the runtime allocator (native).
*/
export function responseStateMetrics(): ResponseStateMetrics {
const at = now();
let largestBytes = 0;
let oldestCreatedAt = at;
let residentCount = 0;
let spillStubCount = 0;
let tombstoneCount = 0;
let spillPayloadBytes = 0;
for (const state of states.values()) {
const bytes = state.sizeBytes;
if (bytes > largestBytes) largestBytes = bytes;
if (state.createdAt < oldestCreatedAt) oldestCreatedAt = state.createdAt;
if (state.kind === "resident") {
residentCount += 1;
} else if (state.kind === "spill") {
spillStubCount += 1;
spillPayloadBytes += state.spill.payloadBytes;
} else tombstoneCount += 1;
}
return {
count: states.size,
residentCount,
spillStubCount,
tombstoneCount,
totalBytes: responseContinuationRetainedStoreSnapshot().bytes,
spillPayloadBytes,
largestBytes,
oldestAgeMs: states.size > 0 ? at - oldestCreatedAt : 0,
spillWrites: spillCounters.writes,
spillWriteFailures: spillCounters.writeFailures,
spillWriteStatus: spillWriteHealth.consecutiveFailures > 0
? "degraded"
: spillWriteHealth.lastSuccessAt !== null
? "healthy"
: "initial",
spillWriteConsecutiveFailures: spillWriteHealth.consecutiveFailures,
spillLastWriteFailureCode: spillWriteHealth.lastFailureCode,
spillLastWriteFailureOrigin: spillWriteHealth.lastFailureOrigin,
spillAclRetryReturnedTimeouts: spillCounters.aclRetryReturnedTimeouts,
spillAclTimeoutMemoRefusals: spillCounters.aclTimeoutMemoRefusals,
spillLastWriteFailureAt: spillWriteHealth.lastFailureAt,
spillLastWriteSuccessAt: spillWriteHealth.lastSuccessAt,
spillReadFailures: spillCounters.readFailures,
replayScopeMismatchDrops,
};
}
/**
* Read-only spill report for `ocx doctor` and /api/system/memory. Owned = live
* stubs, deferred unlinks, queued publications, and snapshot references (what a
* restart re-owns). Shares the reclaim's orphan predicate; never unlinks.
*/
export function inspectResponseSpillStorage(): ResponseSpillDirInspection {
const referenced = ownedSpillFileNames();
for (const name of snapshotReferencedSpillFileNames(snapshotPath(), SNAPSHOT_FILE_MAX_BYTES)) {
referenced.add(name);
}
return inspectResponseSpillDir(referenced);
}
/**
* Cache completed output and max_output_tokens partial output for previous_response_id replay.
* Content-filtered incomplete and failed output are not authoritative replay history.
*/
export function rememberResponseState(
requestBody: unknown,
response: { id?: unknown; output?: unknown; status?: unknown; incomplete_details?: unknown },
providerState?: OcxProviderContinuationState | string,
opts?: { force?: boolean; clientThreadId?: string },
): void {
if (!requestBody || typeof requestBody !== "object" || Array.isArray(requestBody)) return;
const request = requestBody as Record<string, unknown>;
if (isBodyNonPersistable(request)) return;
// `force` bypasses only the store:false skip: Codex sends `store:false` on every non-Azure
// HTTP request (and WS inherits it), yet its WS turns still chain with previous_response_id.
// The passthrough branch records with force so those chains can be expanded locally; the
// store stays in-memory under RESPONSE_TTL_MS, so this is a proxy-internal continuation cache, not
// real server-side response storage.
if (request.store === false && !opts?.force) return;
if (typeof response.id !== "string" || !Array.isArray(response.output)) return;
if (response.status === "incomplete") {
const details = response.incomplete_details;
if (!details || typeof details !== "object" || Array.isArray(details)
|| (details as { reason?: unknown }).reason !== "max_output_tokens") return;
} else if (response.status !== undefined && response.status !== "completed") return;
ensureLoaded();
const normalizedProviderState: OcxProviderContinuationState = typeof providerState === "string"
? { cursor: { conversationId: providerState } }
: structuredClone(providerState ?? {});
if (normalizedProviderState.cursor?.conversationId) {
normalizedProviderState.cursor.checkpointUsable = !response.output.some(item => {
return !!item && typeof item === "object" && (item as { type?: unknown }).type === "function_call";
});
}
const clientThreadId = normalizedClientThreadId(opts?.clientThreadId);
// Compute the normalized array once and reuse it for both fields, so the recorded
// boundary can never disagree with the items it indexes.
const requestItems = inputItems(request.input);
setResidentEntry(response.id, {
createdAt: now(),
...(clientThreadId ? { clientThreadId } : {}),
items: [...requestItems, ...response.output],
// Where response.output begins. A replay skip requires a matched item at or past this
// index that also carries a provider-issued id — position alone proves only that an item
// sits on the provider side, not that the provider authored it.
providerOutputStart: requestItems.length,
// Always preserve the Cursor conversation id so the next tool-result turn can continue the SAME
// Cursor conversation (multi-turn continuation). Separately track whether Cursor's own
// checkpoint/cache is safe to reuse: a turn that ended with a pending client tool call produced an
// incomplete agent turn on the Cursor side (we suspended without a real mcpResult), so its
// checkpoint must not be reused — but the conversation id string itself is still valid.
...(Object.keys(normalizedProviderState).length > 0 ? { providers: normalizedProviderState } : {}),
});
enforceAppOwnedMemoryBudget();
schedulePersist();
}
/** Test-only persistence churn hook; invoked after each atomic snapshot rewrite. */
export function setResponseStatePersistAttemptHookForTests(hook: (() => void) | null): void {
persistAttemptHookForTests = hook;
}
/** Test-only: deterministically run the pending background debounce pass. */
export async function runPendingResponseStatePersistForTests(): Promise<void> {
if (!persistTimer) return;
await persistNow(pendingPersistPath ?? snapshotPath());
}
/** Test-only: observe whether a debounce/follow-up pass is pending. */
export function responseStatePersistPendingForTests(): boolean {
return persistTimer !== null;
}
/** Memory-only reset (simulates a process restart: the snapshot file survives). */
export function clearResponseStateMemoryForTests(): void {
if (persistTimer) {
clearTimeout(persistTimer);
persistTimer = null;
}
pendingPersistPath = null;
resetSpillQueueForTests();
resetPeriodicSpillSweepCursorForTests();
states.clear();
storedResponseBytes = 0;
residentResponseBytes = 0;
oldestResidentId = undefined;
oldestResidentAt = null;
stateRevision = 0;
pendingSpillUnlinks.length = 0;
spillCounters.writes = 0;
spillCounters.writeFailures = 0;
spillCounters.readFailures = 0;
spillCounters.aclRetryReturnedTimeouts = 0;
spillCounters.aclTimeoutMemoRefusals = 0;
spillWriteHealth.consecutiveFailures = 0;
spillWriteHealth.lastFailureCode = null;
spillWriteHealth.lastFailureOrigin = null;
spillWriteHealth.lastFailureAt = null;
spillWriteHealth.lastSuccessAt = null;
replayScopeMismatchDrops = 0;
replayOverlapSkips = 0;
persistAttemptHookForTests = null;
lastSnapshotBytes = 0;
lastSnapshotDigest = null;
lastSnapshotTarget = null;
loaded = false;
}
export function clearResponseStateForTests(): void {
for (const entry of states.values()) deleteOwnedSpills(entry);
clearResponseStateMemoryForTests();
try {
unlinkSync(snapshotPath());
} catch {
/* no snapshot on disk */
}
try { rmSync(responseSpillDirectory(), { recursive: true, force: true }); } catch { /* no spill directory */ }
}