UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

536 lines (511 loc) • 23.7 kB
/** * Dashboard memory-card drain-and-restart (#563). * * Waits up to 60s for active turns unless a manual API caller explicitly opts * into a shorter grace, with the existing 60s cleanup watchdog, then respawns. Never runs restoreNativeCodex / stripGrokConfig — this is a * recycle to reclaim RSS, not a teardown. * * Respawn policy (matches real supervisor configs in src/service.ts): * - Supervised child (`OCX_SERVICE=1` + viable service): exit(1) so * failure-only supervisors (systemd Restart=on-failure, WinSW onfailure, * Task Scheduler ERRORLEVEL loop) bring the proxy back. * - Otherwise: detached `ocx start --port <live>` (bypasses ensure's * codexAutoStart gate), mark recycle so exit cleanup keeps injection, exit(0). * Installed-but-stale/missing service assets are NOT treated as supervised — * exit(1) would leave the proxy dead with `Service: installed, stale or missing * service assets` and a /healthz timeout. * - The replacement is spawned by `src/server/restart-replacement.ts`: it carries * `OCX_RESTART_PARENT_PID` and its output goes to the bounded `restart-handoff.log`. * Only after a completed drain (the handoff waits for health) is a replacement that * exits before it answers retried, twice, inside the readiness budget. A deadline, * failed-drain or listener-stop-fallback handoff resolves on spawn and exits: the * replacement's own parent wait and port reclaim cover it, and any other early exit * there is not retried. * - If detached spawn fails (sync throw, pre-start `error`, or every attempt exited * early): exit(1) without markRecycling — after drain the listen socket is already * closed, so a latch reset cannot recover serving. Clear inherited `OCX_SERVICE` * so exit cleanup can restore Codex/Grok fences when a stale service marker has no * viable supervisor. Log only a stable errno code — never the raw message * (paths in ENOENT often include the OS username). * - Except after a committed client connection (a join into a Child): Codex already * routes to the client runtime the next start serves on this port, so the failed * handoff still marks recycling and exit cleanup keeps that routing instead of * silently falling back to native Codex. * - Desktop-supervised child (the desktop app spawned it with `OCX_DESKTOP_SUPERVISED=1` * and is still its parent): no spawn. Mark recycle and exit 75; the app sees the exit * and starts the replacement itself, so it keeps owning, stopping and quitting it. * Checked before the service rule: that app, not a service manager, is the parent. */ import { abortAndReleaseAllTurns, acquireTemporaryDrain, beginShutdownDrain, drainAndShutdown, getActiveTurnCount, getServerListenPort, isDraining, isShutdownDraining, markRecyclingForExit, stopServerListener, } from "../lifecycle"; import { isServiceViable } from "../../service"; import { readClientConnectionState } from "../../client/state"; import { withSiblingMarker } from "../../codex/sibling-start"; import { issueSiblingHandoff } from "../../codex/sibling-handoff"; import { readRuntimePort } from "../../config/process-state"; import { spendLedgerRestartEnvironment } from "../../lib/spend-ledger-owner"; import { SUPERVISOR_MARKER_ENVS } from "../../lib/supervisor-markers"; import { DESKTOP_RESTART_EXIT_CODE, DESKTOP_SUPERVISED_ENV, MEMORY_DRAIN_RESTART_MS, isDesktopSupervised, } from "../../lib/system-restart-contract"; import { spawnReplacementStart } from "../restart-replacement"; import { retireRestartParentOwnership } from "../restart-parent-ownership"; export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../../lib/system-restart-contract"; export { waitForReplacementReady, type ReplacementReadinessIo } from "../restart-replacement"; export const DEADLINE_LISTENER_STOP_TIMEOUT_MS = 5_000; const RESTART_RESPONSE_FLUSH_DELAY_MS = 200; export interface SystemRestartIo { /** Cancel and release admitted turns without stopping the listener or process. */ abortActiveTurns?: () => void; acquireTemporaryDrain?: () => { release(): void } | null; drainAndShutdown?: typeof drainAndShutdown; /** True when a background service can actually respawn this process after exit(1). */ isServiceViable?: () => boolean; isSupervisedServiceChild?: () => boolean; /** True when the desktop app that spawned this process starts its replacement after exit 75. */ isDesktopSupervised?: () => boolean; /** Ordinary start; deadline handoff may defer health until parent exit releases OS locks. */ spawnStart?: (port?: number, waitForHealthBeforeParentExit?: boolean) => void | Promise<void>; /** Idempotent listener close; must settle before an ordinary start is spawned. */ stopListener?: () => void | Promise<void>; markRecycling?: () => void; /** True once a client connection has committed (a join into a Child); read only after a failed handoff. */ isClientConnected?: () => boolean; exitProcess?: (code: number) => void; schedule?: (fn: () => void | Promise<void>, ms: number) => void; scheduleDeadline?: (fn: () => void, ms: number) => () => void; isDraining?: () => boolean; isShutdownDraining?: () => boolean; beginShutdownDrain?: () => boolean; setDraining?: (value: boolean) => void; getActiveTurnCount?: () => number; listenPort?: () => number | undefined; now?: () => number; } export interface SystemRestartOptions { /** Explicit API opt-in; omission preserves the existing 60s drain. */ drainGraceMs?: number; } export interface SystemRestartAdmission { /** Only the caller that created this pending restart receives its veto. */ onAccepted?: (veto: () => void) => void; /** Recheck external authority before draining or handing off. */ beforeScheduledDrain?: () => boolean; } let restartIo: SystemRestartIo = {}; /** Prevents double-scheduling during the 200ms response-flush window. */ let restartAccepted = false; let acceptedDrainGraceMs = MEMORY_DRAIN_RESTART_MS; type RestartDrainOutcome = "completed" | "failed" | "rejected" | "deadline"; type BoundedSettlementOutcome = "completed" | "rejected" | "deadline"; function waitForRestartDrain( drainPromise: Promise<boolean | void>, deadlineMs: number, now: () => number, scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>, ): Promise<RestartDrainOutcome> { const remainingMs = Math.max(0, deadlineMs - now()); if (remainingMs === 0) { // Observe any late rejection even though orchestration is already terminal. void drainPromise.catch(() => {}); return Promise.resolve("deadline"); } return new Promise<RestartDrainOutcome>((resolve) => { let settled = false; let cancelDeadline: (() => void) | undefined; const finish = (outcome: RestartDrainOutcome) => { if (settled) return; settled = true; cancelDeadline?.(); resolve(outcome); }; cancelDeadline = scheduleDeadline(() => finish("deadline"), remainingMs); if (settled) cancelDeadline(); void drainPromise.then( succeeded => finish(succeeded === false ? "failed" : "completed"), () => finish("rejected"), ); }); } function waitForBoundedSettlement( promise: Promise<void>, timeoutMs: number, scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>, ): Promise<BoundedSettlementOutcome> { return new Promise<BoundedSettlementOutcome>((resolve) => { let settled = false; let cancelDeadline: (() => void) | undefined; const finish = (outcome: BoundedSettlementOutcome) => { if (settled) return; settled = true; cancelDeadline?.(); resolve(outcome); }; cancelDeadline = scheduleDeadline(() => finish("deadline"), timeoutMs); if (settled) cancelDeadline(); void promise.then( () => finish("completed"), () => finish("rejected"), ); }); } /** * Accepted stops and signals permanently veto automatic handoff. Management requests carry * separate temporary intents while teardown awaits, so a refusal can release its own intent * without clearing another pending stop, an accepted stop, or a signal. */ let explicitShutdownRequested = false; const pendingShutdownIntents = new Set<object>(); const shutdownIntentWaiters = new Set<() => void>(); function shutdownIntentChanged(): void { const waiters = [...shutdownIntentWaiters]; shutdownIntentWaiters.clear(); for (const resolve of waiters) resolve(); } function nextShutdownIntentChange(): Promise<void> { return new Promise(resolve => { shutdownIntentWaiters.add(resolve); }); } export function noteExplicitShutdownRequested(): void { explicitShutdownRequested = true; shutdownIntentChanged(); } /** Veto automatic handoff while a stop is pending; release only this caller's intent. */ export function acquireExplicitShutdownIntent(): () => void { const intent = {}; pendingShutdownIntents.add(intent); return () => { if (pendingShutdownIntents.delete(intent)) shutdownIntentChanged(); }; } /** Test seam — reset between tests. */ export function setSystemRestartIoForTests(io: SystemRestartIo = {}): void { restartIo = io; restartAccepted = false; acceptedDrainGraceMs = MEMORY_DRAIN_RESTART_MS; explicitShutdownRequested = false; pendingShutdownIntents.clear(); shutdownIntentChanged(); } /** The port this process is listening on, or undefined when that cannot be established. */ export function resolveListenPort(): number | undefined { const live = getServerListenPort(); if (live) return live; const runtime = readRuntimePort(process.pid); if (runtime && runtime.port > 0) return runtime.port; return undefined; } function isSupervisedServiceChild(io: SystemRestartIo = {}): boolean { if (process.env.OCX_SERVICE !== "1") return false; // Presence is not enough: stale/missing service assets report installed but will not // respawn after exit(1). Dashboard status/recovery must fall through to detached start. return (io.isServiceViable ?? isServiceViable)(); } /** Stable, path-free spawn failure label for logs (never interpolate err.message). */ function spawnFailureCode(err: unknown): string { if (err && typeof err === "object" && "code" in err) { const code = (err as NodeJS.ErrnoException).code; if (typeof code === "string" && code.length > 0 && code.length <= 64) return code; } return "spawn_failed"; } /** * After a failed handoff, keep Codex routing when a client connection has committed. * * `connectClient` already pointed Codex at the client runtime that the next start (a service * relaunch, the desktop app, or `ocx start`) serves on this same port. Restoring native Codex here * would be a silent local fallback while client state says connected. An unreadable state is not * proof of a join, so it keeps today's restore. */ function keepRoutingForCommittedClient(io: SystemRestartIo): void { let connected = false; try { connected = (io.isClientConnected ?? (() => readClientConnectionState().kind === "connected"))(); } catch { connected = false; } if (connected) (io.markRecycling ?? markRecyclingForExit)(); } /** * Hand the restart to the desktop app that spawned this process: keep Codex routing for the * replacement it starts on this port, and exit with the code it restarts on. False when nothing * supervises this process that way, and then nothing happened. */ function handOffToDesktop(io: SystemRestartIo, exitProcess: (code: number) => void): boolean { if (!(io.isDesktopSupervised ?? isDesktopSupervised)()) return false; (io.markRecycling ?? markRecyclingForExit)(); exitProcess(DESKTOP_RESTART_EXIT_CODE); return true; } /** * The replacement's environment before `spawnReplacementStart` adds the restart-parent marker and * the runtime provenance: never under a service marker, a sibling's replacement stays a sibling, * and only a parent-exit handoff marks the bounded spend-ledger lease wait. */ export function replacementStartEnvironment( waitForHealthBeforeParentExit: boolean, parentPid: number = process.pid, ): NodeJS.ProcessEnv { // A sibling's replacement stays a sibling even if the owner is down while it probes. const sourceEnv: NodeJS.ProcessEnv = withSiblingMarker(process.env, issueSiblingHandoff); delete sourceEnv.OCX_SERVICE; // A detached replacement is not the desktop app's child, nor a service manager's or the Windows // wrapper's; it must never exit to a supervisor that is not waiting on it. for (const name of SUPERVISOR_MARKER_ENVS) delete sourceEnv[name]; return spendLedgerRestartEnvironment( sourceEnv, waitForHealthBeforeParentExit ? undefined : parentPid, ); } function spawnDetachedStart( port?: number, waitForHealthBeforeParentExit = true, ): Promise<void> { let env: NodeJS.ProcessEnv; try { env = replacementStartEnvironment(waitForHealthBeforeParentExit); // Prepare the sibling handoff while its runtime record still exists. Only completed // drain + listener stop may release parent ownership before waiting for child health. if (waitForHealthBeforeParentExit) retireRestartParentOwnership(port); } catch (err) { return Promise.reject(err); } return spawnReplacementStart({ port, waitForHealth: waitForHealthBeforeParentExit, env }); } async function completeDeferredParentExitHandoff( io: SystemRestartIo, exitProcess: (code: number) => void, port: number | undefined, phase: "deadline" | "listener-stop fallback", canHandoff: () => boolean = () => true, automatic = false, ): Promise<void> { while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested) await nextShutdownIntentChange(); if (!canHandoff()) return; try { await (io.spawnStart ?? spawnDetachedStart)(port, false); } catch (err) { console.warn( `Drain-and-restart ${phase} spawn failed (${spawnFailureCode(err)}); exiting without replacement`, ); delete process.env.OCX_SERVICE; keepRoutingForCommittedClient(io); exitProcess(1); return; } (io.markRecycling ?? markRecyclingForExit)(); exitProcess(0); } async function completeDeadlineRestartHandoff( io: SystemRestartIo, exitProcess: (code: number) => void, port: number | undefined, scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>, canHandoff: () => boolean = () => true, automatic = false, ): Promise<void> { while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested) await nextShutdownIntentChange(); if (!canHandoff()) return; if (handOffToDesktop(io, exitProcess)) return; const supervised = (io.isSupervisedServiceChild ?? (() => isSupervisedServiceChild(io)))(); if (supervised) { // Failure-only supervisors ignore exit(0); intentional non-zero triggers respawn. if (!canHandoff()) return; exitProcess(1); return; } const stopPromise = Promise.resolve().then( () => (io.stopListener ?? (() => stopServerListener()))(), ); const stopOutcome = await waitForBoundedSettlement( stopPromise, DEADLINE_LISTENER_STOP_TIMEOUT_MS, scheduleDeadline, ); if (stopOutcome === "rejected") { console.warn("Drain-and-restart deadline listener stop failed; continuing parent-exit handoff"); } else if (stopOutcome === "deadline") { console.warn("Drain-and-restart deadline listener stop timed out; continuing parent-exit handoff"); } // The ordinary child must survive parent exit: a failed/pending socket close // or overdue cleanup is completed by process teardown, without a hidden mode. await completeDeferredParentExitHandoff(io, exitProcess, port, "deadline", canHandoff, automatic); } /** * Accept a drain-and-restart request. Returns immediately; the drain + * respawn waits 200ms for the HTTP response to flush; shorter grace cuts turns separately. * Idempotent while already draining: returns the accepted shape again. */ export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: SystemRestartAdmission = {}, options: SystemRestartOptions = {}): { accepted: true; alreadyDraining: boolean; activeTurnCount: number; drainTimeoutMs: number; } { const shutdownActive = io.isShutdownDraining ?? io.isDraining ?? isShutdownDraining; const alreadyDraining = restartAccepted || shutdownActive(); const drainGraceMs = alreadyDraining ? (restartAccepted ? acceptedDrainGraceMs : MEMORY_DRAIN_RESTART_MS) : (options.drainGraceMs ?? MEMORY_DRAIN_RESTART_MS); const activeTurnCount = (io.getActiveTurnCount ?? getActiveTurnCount)(); const schedule = io.schedule ?? ((fn, ms) => { setTimeout(() => { void fn(); }, ms); }); if (!alreadyDraining) { restartAccepted = true; acceptedDrainGraceMs = drainGraceMs; const automatic = Boolean(admission.onAccepted); const temporaryDrain = automatic ? (io.acquireTemporaryDrain ?? (() => acquireTemporaryDrain("automatic-restart")))() : null; let cancelEarlyCut: (() => void) | undefined; const releasePending = () => { cancelEarlyCut?.(); temporaryDrain?.release(); restartAccepted = false; }; if (automatic && !temporaryDrain) { restartAccepted = false; return { accepted: true, alreadyDraining: true, activeTurnCount, drainTimeoutMs: MEMORY_DRAIN_RESTART_MS }; } let pending = true; let vetoed = false; admission.onAccepted?.(() => { if (!pending) return; pending = false; vetoed = true; releasePending(); shutdownIntentChanged(); }); const now = io.now ?? Date.now; const acceptedAtMs = now(); const restartDeadlineMs = acceptedAtMs + MEMORY_DRAIN_RESTART_MS; const drainDeadlineMs = acceptedAtMs + drainGraceMs; const scheduleDeadline = io.scheduleDeadline ?? ((fn, ms) => { const timer = setTimeout(fn, ms); return () => clearTimeout(timer); }); // Reject new data-plane traffic immediately (503), before the 200ms response-flush delay. if (!automatic) { if (io.beginShutdownDrain) io.beginShutdownDrain(); else if (io.setDraining) io.setDraining(true); else beginShutdownDrain(); } // A shorter grace cuts admitted turns at acceptance + grace, but listener stop // and process exit must still wait for the full response-flush window. if (!vetoed && drainGraceMs < RESTART_RESPONSE_FLUSH_DELAY_MS) { cancelEarlyCut = scheduleDeadline(() => { if (vetoed) return; (io.abortActiveTurns ?? (() => abortAndReleaseAllTurns(new Error("server shutdown"))))(); }, drainGraceMs); } schedule(async () => { if (vetoed) return; // Keep the one admitted restart queued until every temporary stop intent settles. while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested && !vetoed) await nextShutdownIntentChange(); if (vetoed) return; pending = false; const canHandoff = () => { // Only admission-bound (automatic) restarts yield to an explicit stop; manual restart // requests keep their existing semantics. if (admission.onAccepted && (explicitShutdownRequested || pendingShutdownIntents.size > 0)) return false; try { return admission.beforeScheduledDrain?.() ?? true; } catch { return false; } // Unknown ownership is not authority to restart. }; if (!canHandoff()) { releasePending(); return; } if (automatic) { if (io.beginShutdownDrain) io.beginShutdownDrain(); else if (!io.setDraining) beginShutdownDrain(); temporaryDrain?.release(); } // Preserve the live binding before drainAndShutdown (or its deadline race) // closes the listener and makes both the server ref and runtime metadata stale. const restartPort = (io.listenPort ?? resolveListenPort)(); const drain = io.drainAndShutdown ?? drainAndShutdown; // Bound active/scoped-drain waiting independently of cleanup and readiness. // A cut turn may already have executed upstream; never replay it automatically. const drainPromise = Promise.resolve().then( () => drain(undefined, Math.max(0, drainDeadlineMs - now())), ); let drainOutcome = await waitForRestartDrain(drainPromise, restartDeadlineMs, now, scheduleDeadline); while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested) await nextShutdownIntentChange(); if (!canHandoff()) return; if (automatic && now() >= restartDeadlineMs) drainOutcome = "deadline"; const exitProcess = io.exitProcess ?? ((code: number) => { process.exit(code); }); if (drainOutcome === "deadline") { console.warn("Drain-and-restart deadline expired; forcing terminal restart handoff"); await completeDeadlineRestartHandoff(io, exitProcess, restartPort, scheduleDeadline, canHandoff, automatic); return; } if (drainOutcome === "failed" || drainOutcome === "rejected") { // drainAndShutdown stops the listener in finally. Even if ancillary cleanup // rejects, an accepted restart must still reach replacement or terminal exit. console.warn("Drain-and-restart cleanup failed; continuing terminal restart handoff"); } if (handOffToDesktop(io, exitProcess)) return; const supervised = (io.isSupervisedServiceChild ?? (() => isSupervisedServiceChild(io)))(); if (supervised) { // Failure-only supervisors ignore exit(0); intentional non-zero triggers respawn. if (!canHandoff()) return; (io.exitProcess ?? ((code: number) => { process.exit(code); }))(1); return; } try { await (io.stopListener ?? (() => stopServerListener()))(); } catch { console.warn("Drain-and-restart listener stop failed; continuing parent-exit handoff"); await completeDeferredParentExitHandoff( io, exitProcess, restartPort, "listener-stop fallback", canHandoff, automatic, ); return; } try { while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested) await nextShutdownIntentChange(); if (!canHandoff()) return; if (automatic && now() >= restartDeadlineMs) { await completeDeferredParentExitHandoff(io, exitProcess, restartPort, "deadline", canHandoff, automatic); return; } // A rejected drain has uncertain cleanup ownership, so it uses the same // parent-exit handoff as a deadline. Only a fully completed drain waits // for replacement health in the old process. await (io.spawnStart ?? spawnDetachedStart)(restartPort, drainOutcome === "completed"); } catch (err) { console.warn( `⚠️ Drain-and-restart spawn failed (${spawnFailureCode(err)}); exiting without replacement`, ); // Listen socket is already stopped; do not markRecycling — no child to inherit fences. // No replacement inherited the routing. Clear a stale service marker so // this unsupervised parent restores clients after the failed handoff — unless a // committed client connection already owns that routing. delete process.env.OCX_SERVICE; keepRoutingForCommittedClient(io); exitProcess(1); return; } (io.markRecycling ?? markRecyclingForExit)(); exitProcess(drainOutcome === "failed" || drainOutcome === "rejected" ? 1 : 0); }, RESTART_RESPONSE_FLUSH_DELAY_MS); } return { accepted: true, alreadyDraining, activeTurnCount, drainTimeoutMs: drainGraceMs, }; }