UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

702 lines (656 loc) • 31.5 kB
import { existsSync } from "node:fs"; import { basename } from "node:path"; import { cleanupSupersededResponseSpillPublication, createResponseSpillPublicationControl, deleteResponseSpill, markResponseSpillPublicationSuperseded, MAX_RESPONSE_SPILL_PAYLOAD_BYTES, prospectiveResponseSpillBytes, responseSpillPayloadCap, responseSpillNow, type ResponseSpillPublicationControl, type ResponseSpillRef, setResponseSpillUnlinkFailureObserver, writeResponseSpillDurably, writeResponseSpillDurablyAsync, } from "../spill-store"; import { enforceAppOwnedMemoryBudget } from "../../lib/app-owned-memory"; import { admissionCounters, noteSpillCapacityRefusal, noteSpillWriteFailure, noteSpillWriteSuccess, spillAclMemoRefusalOrigin, type ResponseSpillWriteFailureCode, type ResponseSpillWriteFailureOrigin, } from "./spill-failure"; import type { ResidentResponseState, StoredResponseState } from "../state"; const RESPONSE_SPILL_SHUTDOWN_BUDGET_MS = 5_000; const RESPONSE_SPILL_SHUTDOWN_FALLBACK_RESERVE_MS = 4_000; const RESPONSE_SPILL_ASYNC_ACL_ATTEMPT_BUDGET_MS = 30_000; export interface SpillQueueStore { swapResidentForSpill(id: string, expected: ResidentResponseState, ref: ResponseSpillRef): boolean; replaceWithSpillFailure(id: string, expected?: StoredResponseState, options?: { deferSpillUnlink?: boolean }): void; deleteEntry(id: string, options?: { deleteSpill?: boolean }): void; deferSupersededSpill(ref: ResponseSpillRef | undefined): void; replaceMapEntry(id: string, next: StoredResponseState, expected?: StoredResponseState): boolean; currentEntry(id: string): StoredResponseState | undefined; residentEntries(): Array<[string, StoredResponseState]>; recomputeOldestResident(): void; pruneResponses(): void; accountedResponseSpillBytes(): number; spillByteCap(): number; /** No argument: plain cap pruning. A number (0 included): admission reclaim of that headroom. */ enforceSpilledResponseBudget(requiredHeadroomBytes?: number): number; terminalizationMaxPasses(): number; } let store: SpillQueueStore | null = null; export function bindSpillQueueStore(next: SpillQueueStore): void { store = next; } function requireStore(): SpillQueueStore { if (!store) throw new Error("spill-queue store is not bound"); return store; } /** * Windows keeps the candidate replayable while required ACL hardening runs off the event loop. * Pending bytes are pinned, not evictable; cap them below the process-owned 512 MiB ceiling so an * icacls outage cannot turn the serialized queue into an unbounded resident backlog. */ const MAX_PENDING_RESPONSE_SPILL_BYTES = MAX_RESPONSE_SPILL_PAYLOAD_BYTES; interface PendingResponseSpill { id: string; candidate: ResidentResponseState | null; supersededSpill?: ResponseSpillRef; directAdmission: boolean; running: boolean; cancelled: boolean; released: boolean; sizeBytes: number; /** Peak on-disk bytes reserved for this publication; released exactly once on settle. */ reservedBytes: number; publicationControl: ResponseSpillPublicationControl; } const pendingResponseSpills = new Set<PendingResponseSpill>(); const pendingResponseSpillById = new Map<string, PendingResponseSpill>(); let pendingResponseSpillBytes = 0; /** * On-disk bytes a queued publication is about to occupy but has not yet installed into * `states`. * * `spilledResponseBytes()` walks installed spills and deferred unlinks — files that * already exist. It cannot see one that `writeResponseSpillDurablyAsync` is in the * middle of creating, and on Windows that middle can last as long as `icacls` takes. * Without a reservation the cap holds only when writes are fast, which is not a cap. * * The reserved figure is the PEAK footprint, not the payload: publication can fall back * from hard-linking to an exclusive copy, and during that fallback the destination copy * and the temp file exist simultaneously. Reserving one envelope would leave the overshoot * intact at half its magnitude. * * Ownership is single: a job holds its reservation from queue until * `releasePendingResponseSpill`, which every exit from the publication path reaches * through the `finally` in `runPendingResponseSpill` and through cancellation of a * not-yet-running job. A leaked reservation is monotonic — it would ratchet the usable * cap toward zero — so the release must stay on the settlement path rather than in a * parallel bookkeeping pass. */ let reservedResponseSpillBytes = 0; /** * Paths a failed cleanup left on the volume, with the bytes each one occupies. * * A failed unlink leaves a real file behind, so the cap has to keep seeing it. But a * never-decremented total would be phantom debt: a Windows lock that clears a moment * later, or the async writer's own retry, can remove the file while the charge stays * forever — and with 256 MiB payloads two conservative charges consume the whole default * cap, after which nothing can spill for the life of the process. * * So the debt is per PATH, priced at what that path actually holds, and settled the * moment the path is gone. `reconcileUnreclaimableSpillPaths` re-checks on every read of * the accounted total, which is the same tick that would otherwise refuse an admission. */ const unreclaimableSpillPaths = new Map<string, number>(); function chargeUnreclaimableSpillPath(path: string | null | undefined, bytes: number): void { if (!path || bytes <= 0) return; unreclaimableSpillPaths.set(path, bytes); } // A failed unlink of an installed or deferred spill leaves a real file too; price it the same way. setResponseSpillUnlinkFailureObserver(chargeUnreclaimableSpillPath); /** Drop charges for paths that have since disappeared; returns the surviving total. */ function reconcileUnreclaimableSpillPaths(): number { let total = 0; for (const [path, bytes] of [...unreclaimableSpillPaths]) { if (existsSync(path)) total += bytes; else unreclaimableSpillPaths.delete(path); } return total; } /** * Peak on-disk footprint of publishing this candidate: temp plus destination copy. * * Measured from the production serializer rather than from `candidate.sizeBytes`. The * resident measurement omits the `version` field the published envelope carries, so * pricing an admission by it undercounts and lets a request sitting exactly at the cap * still exceed it. Falls back to the resident figure only when serialization fails, which * is the same condition that will fail the publication itself. */ function publicationFootprintBytes(id: string, candidate: ResidentResponseState): number { const exact = prospectiveResponseSpillBytes(id, spillPayloadForResident(candidate)); return (exact ?? candidate.sizeBytes) * 2; } let responseSpillPublicationTail: Promise<void> = Promise.resolve(); let responseSpillShutdownBudgetOverride: { totalMs: number; fallbackReserveMs: number } | null = null; let responseSpillShutdownTerminalizationPassLimitOverride: number | null = null; let responseSpillAsyncAclAttemptBudgetOverride: number | null = null; function releasePendingResponseSpill(job: PendingResponseSpill): void { if (job.released) return; job.released = true; pendingResponseSpillBytes = Math.max(0, pendingResponseSpillBytes - job.sizeBytes); reservedResponseSpillBytes = Math.max(0, reservedResponseSpillBytes - job.reservedBytes); pendingResponseSpills.delete(job); if (pendingResponseSpillById.get(job.id) === job) pendingResponseSpillById.delete(job.id); job.candidate = null; } export function cancelPendingResponseSpill(id: string): ResponseSpillRef | undefined { const job = pendingResponseSpillById.get(id); if (!job) return undefined; pendingResponseSpillById.delete(id); job.cancelled = true; markResponseSpillPublicationSuperseded(job.publicationControl); const superseded = job.supersededSpill; // Ownership TRANSFERS to the caller. Leaving the ref on the cancelled job would let the // accounting walk count the same physical file twice — once here and once on the // replacement — and an overcount evicts live continuations to make room for bytes that // are not there. delete job.supersededSpill; // A queued job has not captured the candidate in an async frame yet, so release it now. // A running job retains its accounting until settlement and will discard its stale file. if (!job.running) releasePendingResponseSpill(job); return superseded; } function isAclTimeout(error: unknown): boolean { return !!error && typeof error === "object" && "code" in error && String((error as { code?: unknown }).code) === "ETIMEDOUT"; } function spillPayloadForResident(candidate: ResidentResponseState): Parameters<typeof writeResponseSpillDurably>[1] { return { createdAt: candidate.createdAt, ...(candidate.clientThreadId ? { clientThreadId: candidate.clientThreadId } : {}), ...(candidate.replayOwnerKey ? { replayOwnerKey: candidate.replayOwnerKey } : {}), items: candidate.items, ...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}), ...(candidate.providers ? { providers: candidate.providers } : {}), ...(candidate.unforcedStoreFalse ? { unforcedStoreFalse: candidate.unforcedStoreFalse } : {}), }; } async function runPendingResponseSpill(job: PendingResponseSpill): Promise<void> { if (job.cancelled || !job.candidate) return; job.running = true; const candidate = job.candidate; let ref: ResponseSpillRef | null = null; let exhaustedAclRetry = false; let aclRetryFailureOrigin: ResponseSpillWriteFailureOrigin | null = null; try { const state = spillPayloadForResident(candidate); try { ref = await writeResponseSpillDurablyAsync(job.id, state, { aclBudgetMs: responseSpillAsyncAclAttemptBudgetMs(), publicationControl: job.publicationControl, }); } catch (error) { if (!isAclTimeout(error)) throw error; // The ACL helper permits exactly one caller-owned recovery budget. The resident generation // remains replayable during both attempts, so a transient timeout never becomes a tombstone. try { ref = await writeResponseSpillDurablyAsync(job.id, state, { aclBudgetMs: responseSpillAsyncAclAttemptBudgetMs(), retryTimedOutOnce: true, publicationControl: job.publicationControl, }); } catch (retryError) { exhaustedAclRetry = isAclTimeout(retryError); // A returned timeout can also mean an exhausted budget before the next OS command. aclRetryFailureOrigin = spillAclMemoRefusalOrigin(retryError) ?? (exhaustedAclRetry ? "retry_returned_timeout" : null); throw retryError; } } if (ref.payloadBytes > responseSpillPayloadCap()) { deleteResponseSpill(ref); ref = null; if (job.directAdmission) admissionCounters.oversizedDrops += 1; throw Object.assign(new Error("Response spill payload exceeds replay ceiling"), { code: "EFBIG" }); } if (requireStore().currentEntry(job.id) !== candidate || job.cancelled) { deleteResponseSpill(ref); ref = null; return; } if (requireStore().swapResidentForSpill(job.id, candidate, ref)) { ref = null; noteSpillWriteSuccess(); if (job.directAdmission) admissionCounters.directSpills += 1; requireStore().deferSupersededSpill(job.supersededSpill); } } catch (error) { if (ref) deleteResponseSpill(ref); if (requireStore().currentEntry(job.id) === candidate && !job.cancelled) { noteSpillWriteFailure(error, exhaustedAclRetry ? "EACLRETRYEXHAUSTED" : undefined, aclRetryFailureOrigin); requireStore().replaceWithSpillFailure(job.id, candidate); requireStore().deferSupersededSpill(job.supersededSpill); } } finally { const cancelled = job.cancelled; releasePendingResponseSpill(job); requireStore().recomputeOldestResident(); if (!cancelled) { requireStore().pruneResponses(); enforceAppOwnedMemoryBudget(); } } } export function queuePendingResponseSpill( id: string, candidate: ResidentResponseState, options: { supersededSpill?: ResponseSpillRef; directAdmission?: boolean } = {}, ): void { if (candidate.connectionOwned === true) { // Refuse without changing the map or a foreign pending publication for this id. // The state owner's memory-pressure and close paths perform scoped eviction. return; } const inheritedSpill = cancelPendingResponseSpill(id) ?? options.supersededSpill; if (pendingResponseSpillBytes + candidate.sizeBytes > MAX_PENDING_RESPONSE_SPILL_BYTES) { noteSpillWriteFailure(null, "ECAPACITY"); requireStore().replaceWithSpillFailure(id, candidate); requireStore().deferSupersededSpill(inheritedSpill); return; } // Enforce the disk cap BEFORE the temp or destination file is created. Deleting the // overflow afterwards is not equivalent: on Windows the file can outlive the decision // by as long as ACL hardening takes, which is the window the measured 6.8 GiB // accumulated in. Reclaim first, and only refuse if the peak footprint still does not // fit — an eviction pass can free a live continuation's worth of room. const footprint = publicationFootprintBytes(id, candidate); // The superseded generation this job is about to own is already off `states` and not // yet on the job, so it is invisible to the walk. Price it here or admission decides // against a total that is short by a whole envelope. const inheritedBytes = inheritedSpill?.payloadBytes ?? 0; if (requireStore().accountedResponseSpillBytes() + footprint + inheritedBytes > requireStore().spillByteCap()) { // An envelope over the replay ceiling ends in EFBIG anyway; never evict a continuation for it. if (footprint / 2 <= responseSpillPayloadCap()) requireStore().enforceSpilledResponseBudget(footprint + inheritedBytes); if (requireStore().accountedResponseSpillBytes() + footprint + inheritedBytes > requireStore().spillByteCap()) { noteSpillCapacityRefusal(); requireStore().replaceWithSpillFailure(id, candidate); requireStore().deferSupersededSpill(inheritedSpill); return; } } const job: PendingResponseSpill = { id, candidate, ...(inheritedSpill ? { supersededSpill: inheritedSpill } : {}), directAdmission: options.directAdmission === true, running: false, cancelled: false, released: false, sizeBytes: candidate.sizeBytes, reservedBytes: footprint, publicationControl: createResponseSpillPublicationControl(), }; pendingResponseSpills.add(job); pendingResponseSpillById.set(id, job); pendingResponseSpillBytes += job.sizeBytes; reservedResponseSpillBytes += job.reservedBytes; requireStore().recomputeOldestResident(); responseSpillPublicationTail = responseSpillPublicationTail .then(() => runPendingResponseSpill(job), () => runPendingResponseSpill(job)); } export function replaceWithPendingResponseSpill( id: string, candidate: ResidentResponseState, expected: StoredResponseState | undefined, options: { directAdmission?: boolean } = {}, ): boolean { if (candidate.connectionOwned === true) { // Only the state owner's transactional admission validates liveness, applies limits, // and indexes these residents for socket-close cleanup. No spill seam may install one. return false; } const inheritedSpill = pendingResponseSpillById.get(id)?.supersededSpill ?? (expected?.kind === "spill" ? expected.spill : undefined); if (!requireStore().replaceMapEntry(id, candidate, expected)) return false; queuePendingResponseSpill(id, candidate, { ...(inheritedSpill ? { supersededSpill: inheritedSpill } : {}), directAdmission: options.directAdmission === true, }); return true; } /** Test-only: settle every serialized Windows spill publication. */ export async function flushPendingResponseSpillsForTests(): Promise<void> { await drainResponseSpillPublications(); } /** Test-only: observe ordinary queue settlement without invoking shutdown fallback. */ export async function awaitResponseSpillPublicationTailForTests(): Promise<void> { await responseSpillPublicationTail; } /** Test-only: observe the bounded queue without exposing payloads. */ export function pendingResponseSpillMetricsForTests(): { count: number; bytes: number } { return { count: pendingResponseSpills.size, bytes: pendingResponseSpillBytes }; } /** Test-only: shorten the shutdown drain/fallback budget (null restores production values). */ export function setResponseSpillShutdownBudgetForTests( budget: { totalMs: number; fallbackReserveMs: number } | null, ): void { responseSpillShutdownBudgetOverride = budget; } /** Test-only: shorten the ordinary async whole-attempt ACL budget. */ export function setResponseSpillAsyncAclAttemptBudgetForTests(budgetMs: number | null): void { responseSpillAsyncAclAttemptBudgetOverride = budgetMs; } function responseSpillAsyncAclAttemptBudgetMs(): number { return responseSpillAsyncAclAttemptBudgetOverride ?? RESPONSE_SPILL_ASYNC_ACL_ATTEMPT_BUDGET_MS; } /** Test-only: lower the hard terminalization pass guard (null restores production). */ export function setResponseSpillShutdownTerminalizationPassLimitForTests(limit: number | null): void { responseSpillShutdownTerminalizationPassLimitOverride = limit; } function responseSpillShutdownTerminalizationPassLimit(): number { return responseSpillShutdownTerminalizationPassLimitOverride ?? requireStore().terminalizationMaxPasses(); } function responseSpillShutdownBudget(): { totalMs: number; fallbackReserveMs: number } { return responseSpillShutdownBudgetOverride ?? { totalMs: RESPONSE_SPILL_SHUTDOWN_BUDGET_MS, fallbackReserveMs: RESPONSE_SPILL_SHUTDOWN_FALLBACK_RESERVE_MS, }; } function awaitResponseSpillTailUntil(observed: Promise<void>, deadline: number): Promise<boolean> { const remaining = deadline - responseSpillNow(); if (remaining <= 0) return Promise.resolve(false); return new Promise(resolve => { let finished = false; const finish = (settled: boolean): void => { if (finished) return; finished = true; clearTimeout(timer); resolve(settled); }; const timer = setTimeout(() => finish(false), remaining); observed.then(() => finish(true), () => finish(true)); }); } function installShutdownFallbackSpill( job: PendingResponseSpill, candidate: ResidentResponseState, aclBudgetMs: number, ): void { let ref: ResponseSpillRef | null = null; // Supersession released this job's reservation, but the synchronous write below is the // largest publication of the shutdown path and has its own link-then-copy fallback // holding a temp and a destination at once. Re-reserve for its duration so the cap is // not blind exactly where the drain does its heaviest work, and settle in `finally` so // every return, throw and mismatch releases it. const footprint = publicationFootprintBytes(job.id, candidate); reservedResponseSpillBytes += footprint; try { // Supersession released this job, so its superseded generation is no longer visible // to the accounting walk — but the file is still on the volume until // `deferSupersededSpill` or a delete takes it. Price it here or the fallback decides // against a total short by that whole envelope, which is exactly the gap that lets // `debt + footprint <= cap < old + debt + footprint` publish over budget. const supersededBytes = job.supersededSpill?.payloadBytes ?? 0; // The drain must not publish over the cap either. Reclaim first; if the footprint // still does not fit — which is what unreclaimable cleanup debt looks like — the // honest close-out is a tombstone, not another file on a volume that is already // over budget. `replaceWithSpillFailure` is the same fail-closed ending the budget // exhaustion path uses, so replay reports `spill_failed` and the client resends. if (requireStore().accountedResponseSpillBytes() + supersededBytes > requireStore().spillByteCap()) { // The footprint is already reserved above, so only the superseded generation is extra room. if (footprint / 2 <= responseSpillPayloadCap()) requireStore().enforceSpilledResponseBudget(supersededBytes); if (requireStore().accountedResponseSpillBytes() + supersededBytes > requireStore().spillByteCap()) { if (requireStore().currentEntry(job.id) === candidate) { noteSpillCapacityRefusal(); requireStore().replaceWithSpillFailure(job.id, candidate); requireStore().deferSupersededSpill(job.supersededSpill); } throw Object.assign(new Error("Response spill shutdown fallback exceeds the durable disk cap"), { code: "ENOSPC" }); } } ref = writeResponseSpillDurably(job.id, spillPayloadForResident(candidate), { aclBudgetMs }); if (ref.payloadBytes > responseSpillPayloadCap()) { deleteResponseSpill(ref); ref = null; if (job.directAdmission) admissionCounters.oversizedDrops += 1; throw Object.assign(new Error("Response spill payload exceeds replay ceiling"), { code: "EFBIG" }); } if (requireStore().currentEntry(job.id) !== candidate) { deleteResponseSpill(ref); ref = null; return; } if (requireStore().swapResidentForSpill(job.id, candidate, ref)) { ref = null; noteSpillWriteSuccess(); if (job.directAdmission) admissionCounters.directSpills += 1; requireStore().deferSupersededSpill(job.supersededSpill); } } catch (error) { if (ref) deleteResponseSpill(ref); if (requireStore().currentEntry(job.id) === candidate) { noteSpillWriteFailure(error); requireStore().replaceWithSpillFailure(job.id, candidate); requireStore().deferSupersededSpill(job.supersededSpill); } throw error; } finally { reservedResponseSpillBytes = Math.max(0, reservedResponseSpillBytes - footprint); } } function terminalizeShutdownFallbackCandidate( job: PendingResponseSpill, candidate: ResidentResponseState, failureCode: ResponseSpillWriteFailureCode = "ETIMEDOUT", ): void { if (requireStore().currentEntry(job.id) !== candidate) return; noteSpillWriteFailure(null, failureCode); requireStore().replaceWithSpillFailure(job.id, candidate); requireStore().deferSupersededSpill(job.supersededSpill); } function pendingShutdownFallbackCandidates(): Array<{ job: PendingResponseSpill; candidate: ResidentResponseState; }> { return [...pendingResponseSpills] .map(job => ({ job, candidate: job.candidate })) .filter((entry): entry is { job: PendingResponseSpill; candidate: ResidentResponseState } => !!entry.candidate); } function supersedeShutdownFallbackBatch( pending: Array<{ job: PendingResponseSpill; candidate: ResidentResponseState }>, failures: Error[], ): void { for (const { job } of pending) { job.cancelled = true; markResponseSpillPublicationSuperseded(job.publicationControl); } for (const { job } of pending) { const cleanupFailure = cleanupSupersededResponseSpillPublication(job.publicationControl); if (cleanupFailure) { failures.push(cleanupFailure); // Cleanup failed, so an async temp or destination is STILL on the volume. Releasing // the reservation would un-account a file that exists, and the fallback write that // follows reserves only its own footprint — three envelopes on disk priced as two. // // Charge the surviving PATHS rather than a flat two envelopes: `clearOwnedPath` // nulls whichever it managed to remove, so one failure is one file, not two. The // charge is settled automatically once the path disappears, which a retried unlink // or a released Windows lock can still do. const perPath = Math.max(1, Math.floor(job.reservedBytes / 2)); chargeUnreclaimableSpillPath(job.publicationControl.tempPath, perPath); chargeUnreclaimableSpillPath(job.publicationControl.destinationPath, perPath); } releasePendingResponseSpill(job); } } function stopAtShutdownTerminalizationPassLimit( pending: Array<{ job: PendingResponseSpill; candidate: ResidentResponseState }>, failures: Error[], ): void { failures.push(Object.assign(new Error("Response spill shutdown terminalization pass limit exceeded"), { code: "ELOOP" })); supersedeShutdownFallbackBatch(pending, failures); for (const { job, candidate } of pending) { terminalizeShutdownFallbackCandidate(job, candidate, "ELOOP"); } for (const [id, state] of requireStore().residentEntries()) { if (state.kind !== "resident") continue; if (state.connectionOwned === true) { requireStore().deleteEntry(id); continue; } noteSpillWriteFailure(null, "ELOOP"); requireStore().replaceWithSpillFailure(id, state); } requireStore().recomputeOldestResident(); requireStore().pruneResponses(); enforceAppOwnedMemoryBudget(); } function terminalizeExhaustedShutdownFallback( initial: Array<{ job: PendingResponseSpill; candidate: ResidentResponseState }>, failures: Error[], ): void { let pending = initial; let passes = 0; const passLimit = responseSpillShutdownTerminalizationPassLimit(); // Every pass replaces each captured resident with a tombstone. Pruning may expose // another finite batch, but resident count strictly decreases until none can requeue. while (pending.length > 0) { if (passes >= passLimit) { stopAtShutdownTerminalizationPassLimit(pending, failures); return; } passes += 1; supersedeShutdownFallbackBatch(pending, failures); for (const { job, candidate } of pending) { failures.push(Object.assign(new Error("Response spill shutdown fallback budget exhausted"), { code: "ETIMEDOUT" })); terminalizeShutdownFallbackCandidate(job, candidate); } requireStore().recomputeOldestResident(); requireStore().pruneResponses(); enforceAppOwnedMemoryBudget(); pending = pendingShutdownFallbackCandidates(); } } function fallbackPendingResponseSpills(reserveMs: number): Error[] { // Same clock as the harden work this reserve is budgeting — see `responseSpillNow`. const deadline = responseSpillNow() + reserveMs; const failures: Error[] = []; for (;;) { const pending = pendingShutdownFallbackCandidates(); if (pending.length === 0) return failures; if (responseSpillNow() >= deadline) { terminalizeExhaustedShutdownFallback(pending, failures); return failures; } supersedeShutdownFallbackBatch(pending, failures); let reserveExhausted = false; for (let index = 0; index < pending.length; index += 1) { const { job, candidate } = pending[index]!; if (requireStore().currentEntry(job.id) !== candidate) continue; const remaining = deadline - responseSpillNow(); if (remaining <= 0) { reserveExhausted = true; for (const exhausted of pending.slice(index)) { failures.push(Object.assign(new Error("Response spill shutdown fallback budget exhausted"), { code: "ETIMEDOUT" })); terminalizeShutdownFallbackCandidate(exhausted.job, exhausted.candidate); } break; } try { installShutdownFallbackSpill(job, candidate, remaining); } catch (error) { failures.push(error instanceof Error ? error : new Error("Response spill shutdown fallback failed")); } } requireStore().recomputeOldestResident(); requireStore().pruneResponses(); enforceAppOwnedMemoryBudget(); if (reserveExhausted || responseSpillNow() >= deadline) { terminalizeExhaustedShutdownFallback(pendingShutdownFallbackCandidates(), failures); return failures; } } } export async function drainResponseSpillPublications(): Promise<void> { const budget = responseSpillShutdownBudget(); const fallbackReserveMs = Math.min(budget.totalMs, Math.max(1, budget.fallbackReserveMs)); const drainDeadline = responseSpillNow() + Math.max(0, budget.totalMs - fallbackReserveMs); for (;;) { if (pendingResponseSpills.size === 0) return; const observed = responseSpillPublicationTail; const settled = await awaitResponseSpillTailUntil(observed, drainDeadline); if (!settled) { const failures = fallbackPendingResponseSpills(fallbackReserveMs); if (failures.length > 0) { throw new AggregateError(failures, "Response spill shutdown fallback incomplete"); } return; } if (observed === responseSpillPublicationTail) return; } } /** * Byte accounting the facade's `accountedResponseSpillBytes` adds on top of the * installed-spill walk: reserved publication footprint, files a pending job still * owns through its superseded generation, and per-path cleanup debt that still * exists on the volume. */ export function spillQueueAccounting(): { reservedBytes: number; jobOwnedBytes: number; unreclaimableBytes: number } { let jobOwnedBytes = 0; for (const job of pendingResponseSpills) { if (job.supersededSpill) jobOwnedBytes += job.supersededSpill.payloadBytes; } return { reservedBytes: reservedResponseSpillBytes, jobOwnedBytes, unreclaimableBytes: reconcileUnreclaimableSpillPaths(), }; } /** Resident candidates still owned by queued publications, for facade-side accounting. */ export function spillQueueResidentCandidates(): Array<{ id: string; candidate: ResidentResponseState; sizeBytes: number }> { const candidates: Array<{ id: string; candidate: ResidentResponseState; sizeBytes: number }> = []; for (const job of pendingResponseSpills) { if (job.candidate) candidates.push({ id: job.id, candidate: job.candidate, sizeBytes: job.sizeBytes }); } return candidates; } /** Bytes pinned by queued jobs themselves (not their superseded generations). */ export function spillQueuePendingBytes(): number { return pendingResponseSpillBytes; } /** True when this resident entry is the candidate a queued publication will install. */ export function spillQueueHoldsResidentCandidate(id: string, state: ResidentResponseState): boolean { return pendingResponseSpillById.get(id)?.candidate === state; } /** Superseded generation a queued job will replace, if one is already parked on it. */ export function spillQueueSupersededSpillFor(id: string): ResponseSpillRef | undefined { return pendingResponseSpillById.get(id)?.supersededSpill; } /** * File names an orphan sweep must keep: each queued job's superseded generation, * its in-flight temp, and its destination — all still owned but not yet named by * `states` or `pendingSpillUnlinks`. */ export function spillQueueReferencedSpillFileNames(): Set<string> { const names = new Set<string>(); for (const job of pendingResponseSpills) { if (job.supersededSpill) names.add(job.supersededSpill.fileName); if (job.publicationControl.tempPath) names.add(basename(job.publicationControl.tempPath)); if (job.publicationControl.destinationPath) names.add(basename(job.publicationControl.destinationPath)); } return names; } /** Test-only: release queued jobs and zero the queue-owned byte accounting. */ export function resetSpillQueueForTests(): void { for (const id of [...pendingResponseSpillById.keys()]) cancelPendingResponseSpill(id); pendingResponseSpillById.clear(); reservedResponseSpillBytes = 0; unreclaimableSpillPaths.clear(); }