@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
536 lines (511 loc) • 23.7 kB
text/typescript
/**
* Dashboard memory-card drain-and-restart (#563).
*
* Waits up to 60s for active turns unless a manual API caller explicitly opts
* into a shorter grace, with the existing 60s cleanup watchdog, then respawns. Never runs restoreNativeCodex / stripGrokConfig — this is a
* recycle to reclaim RSS, not a teardown.
*
* Respawn policy (matches real supervisor configs in src/service.ts):
* - Supervised child (`OCX_SERVICE=1` + viable service): exit(1) so
* failure-only supervisors (systemd Restart=on-failure, WinSW onfailure,
* Task Scheduler ERRORLEVEL loop) bring the proxy back.
* - Otherwise: detached `ocx start --port <live>` (bypasses ensure's
* codexAutoStart gate), mark recycle so exit cleanup keeps injection, exit(0).
* Installed-but-stale/missing service assets are NOT treated as supervised —
* exit(1) would leave the proxy dead with `Service: installed, stale or missing
* service assets` and a /healthz timeout.
* - The replacement is spawned by `src/server/restart-replacement.ts`: it carries
* `OCX_RESTART_PARENT_PID` and its output goes to the bounded `restart-handoff.log`.
* Only after a completed drain (the handoff waits for health) is a replacement that
* exits before it answers retried, twice, inside the readiness budget. A deadline,
* failed-drain or listener-stop-fallback handoff resolves on spawn and exits: the
* replacement's own parent wait and port reclaim cover it, and any other early exit
* there is not retried.
* - If detached spawn fails (sync throw, pre-start `error`, or every attempt exited
* early): exit(1) without markRecycling — after drain the listen socket is already
* closed, so a latch reset cannot recover serving. Clear inherited `OCX_SERVICE`
* so exit cleanup can restore Codex/Grok fences when a stale service marker has no
* viable supervisor. Log only a stable errno code — never the raw message
* (paths in ENOENT often include the OS username).
* - Except after a committed client connection (a join into a Child): Codex already
* routes to the client runtime the next start serves on this port, so the failed
* handoff still marks recycling and exit cleanup keeps that routing instead of
* silently falling back to native Codex.
* - Desktop-supervised child (the desktop app spawned it with `OCX_DESKTOP_SUPERVISED=1`
* and is still its parent): no spawn. Mark recycle and exit 75; the app sees the exit
* and starts the replacement itself, so it keeps owning, stopping and quitting it.
* Checked before the service rule: that app, not a service manager, is the parent.
*/
import {
abortAndReleaseAllTurns,
acquireTemporaryDrain,
beginShutdownDrain,
drainAndShutdown,
getActiveTurnCount,
getServerListenPort,
isDraining,
isShutdownDraining,
markRecyclingForExit,
stopServerListener,
} from "../lifecycle";
import { isServiceViable } from "../../service";
import { readClientConnectionState } from "../../client/state";
import { withSiblingMarker } from "../../codex/sibling-start";
import { issueSiblingHandoff } from "../../codex/sibling-handoff";
import { readRuntimePort } from "../../config/process-state";
import { spendLedgerRestartEnvironment } from "../../lib/spend-ledger-owner";
import { SUPERVISOR_MARKER_ENVS } from "../../lib/supervisor-markers";
import {
DESKTOP_RESTART_EXIT_CODE,
DESKTOP_SUPERVISED_ENV,
MEMORY_DRAIN_RESTART_MS,
isDesktopSupervised,
} from "../../lib/system-restart-contract";
import { spawnReplacementStart } from "../restart-replacement";
import { retireRestartParentOwnership } from "../restart-parent-ownership";
export { MEMORY_DRAIN_RESTART_MS, REPLACEMENT_READY_TIMEOUT_MS } from "../../lib/system-restart-contract";
export { waitForReplacementReady, type ReplacementReadinessIo } from "../restart-replacement";
export const DEADLINE_LISTENER_STOP_TIMEOUT_MS = 5_000;
const RESTART_RESPONSE_FLUSH_DELAY_MS = 200;
export interface SystemRestartIo {
/** Cancel and release admitted turns without stopping the listener or process. */
abortActiveTurns?: () => void;
acquireTemporaryDrain?: () => { release(): void } | null;
drainAndShutdown?: typeof drainAndShutdown;
/** True when a background service can actually respawn this process after exit(1). */
isServiceViable?: () => boolean;
isSupervisedServiceChild?: () => boolean;
/** True when the desktop app that spawned this process starts its replacement after exit 75. */
isDesktopSupervised?: () => boolean;
/** Ordinary start; deadline handoff may defer health until parent exit releases OS locks. */
spawnStart?: (port?: number, waitForHealthBeforeParentExit?: boolean) => void | Promise<void>;
/** Idempotent listener close; must settle before an ordinary start is spawned. */
stopListener?: () => void | Promise<void>;
markRecycling?: () => void;
/** True once a client connection has committed (a join into a Child); read only after a failed handoff. */
isClientConnected?: () => boolean;
exitProcess?: (code: number) => void;
schedule?: (fn: () => void | Promise<void>, ms: number) => void;
scheduleDeadline?: (fn: () => void, ms: number) => () => void;
isDraining?: () => boolean;
isShutdownDraining?: () => boolean;
beginShutdownDrain?: () => boolean;
setDraining?: (value: boolean) => void;
getActiveTurnCount?: () => number;
listenPort?: () => number | undefined;
now?: () => number;
}
export interface SystemRestartOptions {
/** Explicit API opt-in; omission preserves the existing 60s drain. */
drainGraceMs?: number;
}
export interface SystemRestartAdmission {
/** Only the caller that created this pending restart receives its veto. */
onAccepted?: (veto: () => void) => void;
/** Recheck external authority before draining or handing off. */
beforeScheduledDrain?: () => boolean;
}
let restartIo: SystemRestartIo = {};
/** Prevents double-scheduling during the 200ms response-flush window. */
let restartAccepted = false;
let acceptedDrainGraceMs = MEMORY_DRAIN_RESTART_MS;
type RestartDrainOutcome = "completed" | "failed" | "rejected" | "deadline";
type BoundedSettlementOutcome = "completed" | "rejected" | "deadline";
function waitForRestartDrain(
drainPromise: Promise<boolean | void>,
deadlineMs: number,
now: () => number,
scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>,
): Promise<RestartDrainOutcome> {
const remainingMs = Math.max(0, deadlineMs - now());
if (remainingMs === 0) {
// Observe any late rejection even though orchestration is already terminal.
void drainPromise.catch(() => {});
return Promise.resolve("deadline");
}
return new Promise<RestartDrainOutcome>((resolve) => {
let settled = false;
let cancelDeadline: (() => void) | undefined;
const finish = (outcome: RestartDrainOutcome) => {
if (settled) return;
settled = true;
cancelDeadline?.();
resolve(outcome);
};
cancelDeadline = scheduleDeadline(() => finish("deadline"), remainingMs);
if (settled) cancelDeadline();
void drainPromise.then(
succeeded => finish(succeeded === false ? "failed" : "completed"),
() => finish("rejected"),
);
});
}
function waitForBoundedSettlement(
promise: Promise<void>,
timeoutMs: number,
scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>,
): Promise<BoundedSettlementOutcome> {
return new Promise<BoundedSettlementOutcome>((resolve) => {
let settled = false;
let cancelDeadline: (() => void) | undefined;
const finish = (outcome: BoundedSettlementOutcome) => {
if (settled) return;
settled = true;
cancelDeadline?.();
resolve(outcome);
};
cancelDeadline = scheduleDeadline(() => finish("deadline"), timeoutMs);
if (settled) cancelDeadline();
void promise.then(
() => finish("completed"),
() => finish("rejected"),
);
});
}
/**
* Accepted stops and signals permanently veto automatic handoff. Management requests carry
* separate temporary intents while teardown awaits, so a refusal can release its own intent
* without clearing another pending stop, an accepted stop, or a signal.
*/
let explicitShutdownRequested = false;
const pendingShutdownIntents = new Set<object>();
const shutdownIntentWaiters = new Set<() => void>();
function shutdownIntentChanged(): void {
const waiters = [...shutdownIntentWaiters];
shutdownIntentWaiters.clear();
for (const resolve of waiters) resolve();
}
function nextShutdownIntentChange(): Promise<void> {
return new Promise(resolve => { shutdownIntentWaiters.add(resolve); });
}
export function noteExplicitShutdownRequested(): void {
explicitShutdownRequested = true;
shutdownIntentChanged();
}
/** Veto automatic handoff while a stop is pending; release only this caller's intent. */
export function acquireExplicitShutdownIntent(): () => void {
const intent = {};
pendingShutdownIntents.add(intent);
return () => { if (pendingShutdownIntents.delete(intent)) shutdownIntentChanged(); };
}
/** Test seam — reset between tests. */
export function setSystemRestartIoForTests(io: SystemRestartIo = {}): void {
restartIo = io;
restartAccepted = false;
acceptedDrainGraceMs = MEMORY_DRAIN_RESTART_MS;
explicitShutdownRequested = false;
pendingShutdownIntents.clear();
shutdownIntentChanged();
}
/** The port this process is listening on, or undefined when that cannot be established. */
export function resolveListenPort(): number | undefined {
const live = getServerListenPort();
if (live) return live;
const runtime = readRuntimePort(process.pid);
if (runtime && runtime.port > 0) return runtime.port;
return undefined;
}
function isSupervisedServiceChild(io: SystemRestartIo = {}): boolean {
if (process.env.OCX_SERVICE !== "1") return false;
// Presence is not enough: stale/missing service assets report installed but will not
// respawn after exit(1). Dashboard status/recovery must fall through to detached start.
return (io.isServiceViable ?? isServiceViable)();
}
/** Stable, path-free spawn failure label for logs (never interpolate err.message). */
function spawnFailureCode(err: unknown): string {
if (err && typeof err === "object" && "code" in err) {
const code = (err as NodeJS.ErrnoException).code;
if (typeof code === "string" && code.length > 0 && code.length <= 64) return code;
}
return "spawn_failed";
}
/**
* After a failed handoff, keep Codex routing when a client connection has committed.
*
* `connectClient` already pointed Codex at the client runtime that the next start (a service
* relaunch, the desktop app, or `ocx start`) serves on this same port. Restoring native Codex here
* would be a silent local fallback while client state says connected. An unreadable state is not
* proof of a join, so it keeps today's restore.
*/
function keepRoutingForCommittedClient(io: SystemRestartIo): void {
let connected = false;
try {
connected = (io.isClientConnected ?? (() => readClientConnectionState().kind === "connected"))();
} catch {
connected = false;
}
if (connected) (io.markRecycling ?? markRecyclingForExit)();
}
/**
* Hand the restart to the desktop app that spawned this process: keep Codex routing for the
* replacement it starts on this port, and exit with the code it restarts on. False when nothing
* supervises this process that way, and then nothing happened.
*/
function handOffToDesktop(io: SystemRestartIo, exitProcess: (code: number) => void): boolean {
if (!(io.isDesktopSupervised ?? isDesktopSupervised)()) return false;
(io.markRecycling ?? markRecyclingForExit)();
exitProcess(DESKTOP_RESTART_EXIT_CODE);
return true;
}
/**
* The replacement's environment before `spawnReplacementStart` adds the restart-parent marker and
* the runtime provenance: never under a service marker, a sibling's replacement stays a sibling,
* and only a parent-exit handoff marks the bounded spend-ledger lease wait.
*/
export function replacementStartEnvironment(
waitForHealthBeforeParentExit: boolean,
parentPid: number = process.pid,
): NodeJS.ProcessEnv {
// A sibling's replacement stays a sibling even if the owner is down while it probes.
const sourceEnv: NodeJS.ProcessEnv = withSiblingMarker(process.env, issueSiblingHandoff);
delete sourceEnv.OCX_SERVICE;
// A detached replacement is not the desktop app's child, nor a service manager's or the Windows
// wrapper's; it must never exit to a supervisor that is not waiting on it.
for (const name of SUPERVISOR_MARKER_ENVS) delete sourceEnv[name];
return spendLedgerRestartEnvironment(
sourceEnv,
waitForHealthBeforeParentExit ? undefined : parentPid,
);
}
function spawnDetachedStart(
port?: number,
waitForHealthBeforeParentExit = true,
): Promise<void> {
let env: NodeJS.ProcessEnv;
try {
env = replacementStartEnvironment(waitForHealthBeforeParentExit);
// Prepare the sibling handoff while its runtime record still exists. Only completed
// drain + listener stop may release parent ownership before waiting for child health.
if (waitForHealthBeforeParentExit) retireRestartParentOwnership(port);
} catch (err) {
return Promise.reject(err);
}
return spawnReplacementStart({ port, waitForHealth: waitForHealthBeforeParentExit, env });
}
async function completeDeferredParentExitHandoff(
io: SystemRestartIo,
exitProcess: (code: number) => void,
port: number | undefined,
phase: "deadline" | "listener-stop fallback",
canHandoff: () => boolean = () => true,
automatic = false,
): Promise<void> {
while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested) await nextShutdownIntentChange();
if (!canHandoff()) return;
try {
await (io.spawnStart ?? spawnDetachedStart)(port, false);
} catch (err) {
console.warn(
`Drain-and-restart ${phase} spawn failed (${spawnFailureCode(err)}); exiting without replacement`,
);
delete process.env.OCX_SERVICE;
keepRoutingForCommittedClient(io);
exitProcess(1);
return;
}
(io.markRecycling ?? markRecyclingForExit)();
exitProcess(0);
}
async function completeDeadlineRestartHandoff(
io: SystemRestartIo,
exitProcess: (code: number) => void,
port: number | undefined,
scheduleDeadline: NonNullable<SystemRestartIo["scheduleDeadline"]>,
canHandoff: () => boolean = () => true,
automatic = false,
): Promise<void> {
while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested) await nextShutdownIntentChange();
if (!canHandoff()) return;
if (handOffToDesktop(io, exitProcess)) return;
const supervised = (io.isSupervisedServiceChild ?? (() => isSupervisedServiceChild(io)))();
if (supervised) {
// Failure-only supervisors ignore exit(0); intentional non-zero triggers respawn.
if (!canHandoff()) return;
exitProcess(1);
return;
}
const stopPromise = Promise.resolve().then(
() => (io.stopListener ?? (() => stopServerListener()))(),
);
const stopOutcome = await waitForBoundedSettlement(
stopPromise,
DEADLINE_LISTENER_STOP_TIMEOUT_MS,
scheduleDeadline,
);
if (stopOutcome === "rejected") {
console.warn("Drain-and-restart deadline listener stop failed; continuing parent-exit handoff");
} else if (stopOutcome === "deadline") {
console.warn("Drain-and-restart deadline listener stop timed out; continuing parent-exit handoff");
}
// The ordinary child must survive parent exit: a failed/pending socket close
// or overdue cleanup is completed by process teardown, without a hidden mode.
await completeDeferredParentExitHandoff(io, exitProcess, port, "deadline", canHandoff, automatic);
}
/**
* Accept a drain-and-restart request. Returns immediately; the drain +
* respawn waits 200ms for the HTTP response to flush; shorter grace cuts turns separately.
* Idempotent while already draining: returns the accepted shape again.
*/
export function acceptSystemRestart(io: SystemRestartIo = restartIo, admission: SystemRestartAdmission = {}, options: SystemRestartOptions = {}): {
accepted: true;
alreadyDraining: boolean;
activeTurnCount: number;
drainTimeoutMs: number;
} {
const shutdownActive = io.isShutdownDraining
?? io.isDraining
?? isShutdownDraining;
const alreadyDraining = restartAccepted || shutdownActive();
const drainGraceMs = alreadyDraining
? (restartAccepted ? acceptedDrainGraceMs : MEMORY_DRAIN_RESTART_MS)
: (options.drainGraceMs ?? MEMORY_DRAIN_RESTART_MS);
const activeTurnCount = (io.getActiveTurnCount ?? getActiveTurnCount)();
const schedule = io.schedule ?? ((fn, ms) => { setTimeout(() => { void fn(); }, ms); });
if (!alreadyDraining) {
restartAccepted = true;
acceptedDrainGraceMs = drainGraceMs;
const automatic = Boolean(admission.onAccepted);
const temporaryDrain = automatic
? (io.acquireTemporaryDrain ?? (() => acquireTemporaryDrain("automatic-restart")))()
: null;
let cancelEarlyCut: (() => void) | undefined;
const releasePending = () => {
cancelEarlyCut?.();
temporaryDrain?.release();
restartAccepted = false;
};
if (automatic && !temporaryDrain) {
restartAccepted = false;
return { accepted: true, alreadyDraining: true, activeTurnCount, drainTimeoutMs: MEMORY_DRAIN_RESTART_MS };
}
let pending = true;
let vetoed = false;
admission.onAccepted?.(() => {
if (!pending) return;
pending = false;
vetoed = true;
releasePending();
shutdownIntentChanged();
});
const now = io.now ?? Date.now;
const acceptedAtMs = now();
const restartDeadlineMs = acceptedAtMs + MEMORY_DRAIN_RESTART_MS;
const drainDeadlineMs = acceptedAtMs + drainGraceMs;
const scheduleDeadline = io.scheduleDeadline ?? ((fn, ms) => {
const timer = setTimeout(fn, ms);
return () => clearTimeout(timer);
});
// Reject new data-plane traffic immediately (503), before the 200ms response-flush delay.
if (!automatic) {
if (io.beginShutdownDrain) io.beginShutdownDrain();
else if (io.setDraining) io.setDraining(true);
else beginShutdownDrain();
}
// A shorter grace cuts admitted turns at acceptance + grace, but listener stop
// and process exit must still wait for the full response-flush window.
if (!vetoed && drainGraceMs < RESTART_RESPONSE_FLUSH_DELAY_MS) {
cancelEarlyCut = scheduleDeadline(() => {
if (vetoed) return;
(io.abortActiveTurns ?? (() => abortAndReleaseAllTurns(new Error("server shutdown"))))();
}, drainGraceMs);
}
schedule(async () => {
if (vetoed) return;
// Keep the one admitted restart queued until every temporary stop intent settles.
while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested && !vetoed) await nextShutdownIntentChange();
if (vetoed) return;
pending = false;
const canHandoff = () => {
// Only admission-bound (automatic) restarts yield to an explicit stop; manual restart
// requests keep their existing semantics.
if (admission.onAccepted && (explicitShutdownRequested || pendingShutdownIntents.size > 0)) return false;
try { return admission.beforeScheduledDrain?.() ?? true; }
catch { return false; } // Unknown ownership is not authority to restart.
};
if (!canHandoff()) { releasePending(); return; }
if (automatic) {
if (io.beginShutdownDrain) io.beginShutdownDrain();
else if (!io.setDraining) beginShutdownDrain();
temporaryDrain?.release();
}
// Preserve the live binding before drainAndShutdown (or its deadline race)
// closes the listener and makes both the server ref and runtime metadata stale.
const restartPort = (io.listenPort ?? resolveListenPort)();
const drain = io.drainAndShutdown ?? drainAndShutdown;
// Bound active/scoped-drain waiting independently of cleanup and readiness.
// A cut turn may already have executed upstream; never replay it automatically.
const drainPromise = Promise.resolve().then(
() => drain(undefined, Math.max(0, drainDeadlineMs - now())),
);
let drainOutcome = await waitForRestartDrain(drainPromise, restartDeadlineMs, now, scheduleDeadline);
while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested) await nextShutdownIntentChange();
if (!canHandoff()) return;
if (automatic && now() >= restartDeadlineMs) drainOutcome = "deadline";
const exitProcess = io.exitProcess ?? ((code: number) => { process.exit(code); });
if (drainOutcome === "deadline") {
console.warn("Drain-and-restart deadline expired; forcing terminal restart handoff");
await completeDeadlineRestartHandoff(io, exitProcess, restartPort, scheduleDeadline, canHandoff, automatic);
return;
}
if (drainOutcome === "failed" || drainOutcome === "rejected") {
// drainAndShutdown stops the listener in finally. Even if ancillary cleanup
// rejects, an accepted restart must still reach replacement or terminal exit.
console.warn("Drain-and-restart cleanup failed; continuing terminal restart handoff");
}
if (handOffToDesktop(io, exitProcess)) return;
const supervised = (io.isSupervisedServiceChild ?? (() => isSupervisedServiceChild(io)))();
if (supervised) {
// Failure-only supervisors ignore exit(0); intentional non-zero triggers respawn.
if (!canHandoff()) return;
(io.exitProcess ?? ((code: number) => { process.exit(code); }))(1);
return;
}
try {
await (io.stopListener ?? (() => stopServerListener()))();
} catch {
console.warn("Drain-and-restart listener stop failed; continuing parent-exit handoff");
await completeDeferredParentExitHandoff(
io,
exitProcess,
restartPort,
"listener-stop fallback",
canHandoff,
automatic,
);
return;
}
try {
while (automatic && pendingShutdownIntents.size > 0 && !explicitShutdownRequested) await nextShutdownIntentChange();
if (!canHandoff()) return;
if (automatic && now() >= restartDeadlineMs) {
await completeDeferredParentExitHandoff(io, exitProcess, restartPort, "deadline", canHandoff, automatic);
return;
}
// A rejected drain has uncertain cleanup ownership, so it uses the same
// parent-exit handoff as a deadline. Only a fully completed drain waits
// for replacement health in the old process.
await (io.spawnStart ?? spawnDetachedStart)(restartPort, drainOutcome === "completed");
} catch (err) {
console.warn(
`⚠️ Drain-and-restart spawn failed (${spawnFailureCode(err)}); exiting without replacement`,
);
// Listen socket is already stopped; do not markRecycling — no child to inherit fences.
// No replacement inherited the routing. Clear a stale service marker so
// this unsupervised parent restores clients after the failed handoff — unless a
// committed client connection already owns that routing.
delete process.env.OCX_SERVICE;
keepRoutingForCommittedClient(io);
exitProcess(1);
return;
}
(io.markRecycling ?? markRecyclingForExit)();
exitProcess(drainOutcome === "failed" || drainOutcome === "rejected" ? 1 : 0);
}, RESTART_RESPONSE_FLUSH_DELAY_MS);
}
return {
accepted: true,
alreadyDraining,
activeTurnCount,
drainTimeoutMs: drainGraceMs,
};
}