@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
486 lines (458 loc) • 23.6 kB
text/typescript
/**
* The opt-in service health sentinel process (#6671).
*
* The proxy spawns this with its stdin as a pipe whose write end only the proxy holds. When the
* proxy exits for any reason the operating system closes that pipe and this process sees EOF and
* leaves. It asks `/healthz` on the cadence in health-sentinel-policy.ts and, when the policy says
* the proxy has stopped answering, records the recovery and terminates the proxy. The supervisor
* that started the proxy — the Task Scheduler batch loop, launchd, systemd, WinSW — restarts it as
* it would after any crash. Nothing is replayed.
*
* The kill is bound to the proxy, not to a number:
* - Windows reuses ids quickly, so the sentinel opens a handle on its parent at startup, accepts it
* only if that process was created before the sentinel, and terminates through the handle
* (health-sentinel-windows.ts). While the handle is open the id cannot be reused.
* - Linux holds a pidfd after rechecking the live parent relationship, and signals that fd.
* Only unsupported kernels/architectures retain the live-ppid fallback used by macOS.
* - macOS checks the live parent id immediately before SIGKILL; PID reuse between those two
* syscalls remains a documented residual (structure/ops/service-health-recovery.md).
*
* SIGKILL rather than SIGTERM: a proxy whose event loop is stuck cannot run its own SIGTERM
* handler, and systemd counts a SIGTERM death as a clean exit that `Restart=on-failure` does not
* restart.
*
* The final checks and the `intent` write run under the service ownership mutation lease, so the
* decision cannot interleave with a service install, repair, takeover or startup ownership
* decision; a busy lease defers. The lease is released before the kill itself: on Windows the
* sentinel can be terminated together with the proxy, and a lease it still held would block the
* replacement's startup until reclaimed. Every history update takes the same lease. No synchronous
* subprocess runs here: a sentinel that can hang protects nothing.
*/
import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
import { dirname } from "node:path";
import {
DEFAULT_HEALTH_SENTINEL_TIMING,
createHealthSentinelPolicy,
type HealthProbeOutcome,
type HealthSentinelTiming,
} from "./health-sentinel-policy";
import { probeHealth, probeUrlHost, type HealthProbeTarget } from "./health-sentinel-probe";
export const HEALTH_SENTINEL_ENV = {
targetPid: "OCX_HEALTH_SENTINEL_TARGET_PID",
runtimePath: "OCX_HEALTH_SENTINEL_RUNTIME_PATH",
historyPath: "OCX_HEALTH_SENTINEL_HISTORY_PATH",
logPath: "OCX_HEALTH_SENTINEL_LOG_PATH",
/** JSON array of the service-state paths whose ownership mutation lease guards the kill. */
statePaths: "OCX_HEALTH_SENTINEL_STATE_PATHS",
/** Test-only timing override (JSON, partial); production uses the policy defaults. */
timing: "OCX_HEALTH_SENTINEL_TIMING",
} as const;
const HISTORY_LIMIT = 10;
const LOG_CAP_BYTES = 256 * 1024;
const EXIT_WAIT_MS = 10_000;
const LEASE_WAIT_MS = 500;
export type SentinelRuntimeRecord = { pid: number; port: number; hostname?: string; sibling: boolean };
/**
* One recovery. `intent` is written before the kill; the entry is then settled as `terminated`
* (exit observed), `unconfirmed` (no exit observed in time), `failed` (the kill was refused) or
* `skipped` (the target had already gone). Every state but `skipped` counts toward the cap.
*/
export type RecoveryEntry = { at: number; pid: number; state: RecoveryState };
export type RecoveryState = "intent" | "terminated" | "unconfirmed" | "failed" | "skipped";
const RECOVERY_STATES: readonly RecoveryState[] = ["intent", "terminated", "unconfirmed", "failed", "skipped"];
export type TerminateResult = "terminated" | "not_target" | "failed";
export type HealthSentinelDeps = {
targetPid: number;
now(): number;
sleep(ms: number): Promise<void>;
probe(target: HealthProbeTarget, timeoutMs: number): Promise<HealthProbeOutcome>;
readRuntime(): SentinelRuntimeRecord | null;
/** Resolves when the target's pipe closes: the target has exited. */
targetExited: Promise<void>;
isTargetGone(): boolean;
/** Bind to the target once at startup; a failure stands the sentinel down. */
bindTarget(): { ok: true } | { ok: false; reason: string };
/** Release the process owner on every exit, refusal, cancellation or exception. */
closeTarget(): void;
/** Terminate the bound target, re-proving it is still the target at the last moment. */
terminateTarget(): TerminateResult;
/** Run under the ownership mutation lease, released on return; "busy" when it could not be taken promptly. */
withOwnershipLease<T>(run: () => T): T | "busy";
/** Earlier recoveries that count toward the cap, or null when the history is unreadable. */
readHistory(): number[] | null;
/** Durably write or settle one entry; false when it could not be written. */
writeHistory(entry: RecoveryEntry): boolean;
log(line: string): void;
timing?: Partial<HealthSentinelTiming>;
};
export type HealthSentinelResult =
| { outcome: "target_exited" }
| { outcome: "stood_down"; reason: string }
| { outcome: "recovered"; state: RecoveryState };
const GONE = Symbol("gone");
/**
* How late this process's own event loop has run, measured on the wall clock so that a machine
* sleep counts too. The policy discards a timeout observed while the sentinel itself was late.
*/
export function startLagMonitor(tickMs = 250, now: () => number = Date.now) {
let last = now();
let max = 0;
const timer = setInterval(() => {
const at = now();
max = Math.max(max, at - last - tickMs);
last = at;
}, tickMs);
(timer as { unref?: () => void }).unref?.();
return {
reset(): void { max = 0; last = now(); },
read(): number { return Math.max(max, now() - last - tickMs); },
stop(): void { clearInterval(timer); },
};
}
/**
* Race `work` against the target's exit. The exit is one AbortSignal subscribed once per run, and
* each race detaches its listener when it settles: a sentinel that probes a healthy proxy for
* months must not accumulate a handler per probe on one never-settling promise.
*/
async function raceGone<T>(work: Promise<T>, exited: AbortSignal): Promise<T | typeof GONE> {
if (exited.aborted) return GONE;
let onExit: () => void = () => {};
const gone = new Promise<typeof GONE>(resolve => {
onExit = () => resolve(GONE);
exited.addEventListener("abort", onExit, { once: true });
});
try {
return await Promise.race([work, gone]);
} finally {
exited.removeEventListener("abort", onExit);
}
}
type Attempt =
| { kind: "stand_down"; reason: string }
| { kind: "gone" }
| { kind: "intent"; entry: RecoveryEntry };
function matchesProbeTarget(runtime: SentinelRuntimeRecord | null, pid: number, target: HealthProbeTarget): boolean {
return runtime !== null && runtime.pid === pid && !runtime.sibling
&& runtime.port === target.port && probeUrlHost(runtime.hostname) === target.hostname;
}
export async function runHealthSentinel(deps: HealthSentinelDeps): Promise<HealthSentinelResult> {
try { return await observeHealthSentinel(deps); }
finally { deps.closeTarget(); }
}
async function observeHealthSentinel(deps: HealthSentinelDeps): Promise<HealthSentinelResult> {
const timing: HealthSentinelTiming = { ...DEFAULT_HEALTH_SENTINEL_TIMING, ...deps.timing };
const standDown = (reason: string): HealthSentinelResult => {
deps.log(`standing down: ${reason} (pid ${deps.targetPid})`);
return { outcome: "stood_down", reason };
};
const exit = new AbortController();
void deps.targetExited.then(() => exit.abort());
const exitSignal = exit.signal;
const bound = deps.bindTarget();
if (!bound.ok) return standDown(bound.reason);
const history = deps.readHistory();
if (history === null) return standDown("the recovery history cannot be read");
const policy = createHealthSentinelPolicy({
targetPid: deps.targetPid,
startedAt: deps.now(),
recoveryHistory: history,
timing,
});
let pinnedTarget: HealthProbeTarget | null = null;
for (;;) {
if (deps.isTargetGone()) return { outcome: "target_exited" };
const runtime = deps.readRuntime();
if (!runtime || runtime.pid !== deps.targetPid) return standDown("the runtime record no longer names the watched process");
if (runtime.sibling) return standDown("the watched process is a sibling proxy");
const hostname = probeUrlHost(runtime.hostname);
if (hostname === null) return standDown("the recorded bind address is not numeric; DNS names cannot authorize recovery");
const target: HealthProbeTarget = pinnedTarget ?? { hostname, port: runtime.port };
if (!matchesProbeTarget(runtime, deps.targetPid, target)) return standDown("the recorded probe endpoint changed");
pinnedTarget = target;
const outcome = await raceGone(deps.probe(target, timing.probeTimeoutMs), exitSignal);
if (outcome === GONE) return { outcome: "target_exited" };
const wasArmed = policy.state().armed;
const decision = policy.observe(outcome);
if (!wasArmed && policy.state().armed) deps.log(`watching pid ${deps.targetPid} on port ${runtime.port}`);
if (decision.action === "stand_down") return standDown(decision.reason);
if (decision.action === "recover") {
const attempt = deps.withOwnershipLease((): Attempt => {
// Everything that could have changed during the probe is re-read under the lease.
if (deps.isTargetGone()) return { kind: "gone" };
const current = deps.readRuntime();
if (!matchesProbeTarget(current, deps.targetPid, target)) {
return { kind: "stand_down", reason: "the runtime record changed while the target was unresponsive" };
}
// Persist first: an attempted recovery must count toward the cap, and on Windows this
// process may be terminated together with the target.
const entry: RecoveryEntry = { at: deps.now(), pid: deps.targetPid, state: "intent" };
if (!deps.writeHistory(entry)) return { kind: "stand_down", reason: "the recovery history cannot be written" };
deps.log(`recovering pid ${deps.targetPid} on port ${runtime.port}: ${decision.reason}; terminating it so its supervisor restarts it (in-flight requests are not replayed)`);
return { kind: "intent", entry };
});
const settle = (entry: RecoveryEntry) => {
// Best-effort: an unsettled entry stays `intent`, which still counts toward the cap.
if (deps.withOwnershipLease(() => deps.writeHistory(entry)) === "busy") deps.log("could not settle the recovery entry: the service ownership lease is busy");
};
if (attempt === "busy") {
deps.log("recovery deferred: the service ownership lease is busy");
} else if (attempt.kind === "gone") {
return { outcome: "target_exited" };
} else if (attempt.kind === "stand_down") {
return standDown(attempt.reason);
} else {
const { entry } = attempt;
// The lease, history and log work above takes time. Re-check that the evidence is still
// fresh right before the irreversible step: a proxy that recovered meanwhile must not be
// killed on old observations. A clock that ran backwards makes the evidence stale too. The
// final runtime-record read is included before the freshness check so time spent on it
// cannot authorize termination on stale observations.
const finalRuntime = deps.readRuntime();
const evidenceAge = deps.now() - outcome.endedAt;
if (evidenceAge < 0 || evidenceAge > timing.maxEvidenceAgeMs || deps.isTargetGone()) {
settle({ ...entry, state: "skipped" });
if (deps.isTargetGone()) return { outcome: "target_exited" };
deps.log(`discarding stale evidence (${Math.round(evidenceAge / 1000)}s old) for pid ${deps.targetPid}; observing again`);
if (await raceGone(deps.sleep(policy.nextDelayMs()), exitSignal) === GONE) return { outcome: "target_exited" };
continue;
}
if (!matchesProbeTarget(finalRuntime, deps.targetPid, target)) {
settle({ ...entry, state: "skipped" });
return standDown("the recorded probe endpoint changed before termination");
}
const result = deps.terminateTarget();
if (result === "not_target") {
settle({ ...entry, state: "skipped" });
return { outcome: "target_exited" };
}
if (result === "failed") {
settle({ ...entry, state: "failed" });
return standDown("termination was refused");
}
const exited = (await raceGone(deps.sleep(EXIT_WAIT_MS), exitSignal)) === GONE || deps.isTargetGone();
const state: RecoveryState = exited ? "terminated" : "unconfirmed";
settle({ ...entry, state });
deps.log(exited ? `pid ${deps.targetPid} exited` : `pid ${deps.targetPid} did not exit within ${EXIT_WAIT_MS / 1000}s`);
return { outcome: "recovered", state };
}
}
if (await raceGone(deps.sleep(policy.nextDelayMs()), exitSignal) === GONE) return { outcome: "target_exited" };
}
}
function isRecoveryEntry(value: unknown): value is RecoveryEntry {
const entry = value as RecoveryEntry | null;
return typeof entry === "object" && entry !== null
&& typeof entry.at === "number" && Number.isFinite(entry.at)
&& typeof entry.pid === "number" && Number.isSafeInteger(entry.pid)
&& RECOVERY_STATES.includes(entry.state);
}
/** Recovery entries, `[]` when no history exists, null when one exists but cannot be read. */
export function readRecoveryEntries(path: string): RecoveryEntry[] | null {
let raw: string;
try {
raw = readFileSync(path, "utf8");
} catch (error) {
return (error as NodeJS.ErrnoException).code === "ENOENT" ? [] : null;
}
try {
const parsed = JSON.parse(raw) as { recoveries?: unknown };
if (!Array.isArray(parsed.recoveries) || !parsed.recoveries.every(isRecoveryEntry)) return null;
return parsed.recoveries.slice(-HISTORY_LIMIT);
} catch {
return null;
}
}
/** Times of recoveries that count toward the cap. */
export function countedRecoveryTimes(entries: readonly RecoveryEntry[]): number[] {
return entries.filter(entry => entry.state !== "skipped").map(entry => entry.at);
}
/**
* Durably write `entry`, replacing the entry with the same time and pid. The temp file is synced
* before the rename, so a crash leaves either the old history or the new one. Writers are
* serialized by the ownership lease the runner holds around the intent write.
*/
export function writeRecoveryEntry(path: string, entry: RecoveryEntry): boolean {
const existing = readRecoveryEntries(path);
if (existing === null) return false;
const others = existing.filter(item => item.at !== entry.at || item.pid !== entry.pid);
const recoveries = [...others, entry].sort((a, b) => a.at - b.at).slice(-HISTORY_LIMIT);
const temporary = `${path}.${process.pid}.tmp`;
let descriptor: number | null = null;
try {
mkdirSync(dirname(path), { recursive: true });
descriptor = openSync(temporary, "w", 0o600);
writeSync(descriptor, `${JSON.stringify({ recoveries })}\n`);
fsyncSync(descriptor);
closeSync(descriptor);
descriptor = null;
renameSync(temporary, path);
// The rename is the publication; a storage error here means it may not survive a crash.
return syncDirectory(dirname(path));
} catch {
if (descriptor !== null) { try { closeSync(descriptor); } catch { /* already closed */ } }
try { unlinkSync(temporary); } catch { /* absent */ }
return false;
}
}
/** Codes that mean "this filesystem does not sync directories", not "the write may be lost". */
const DIRECTORY_SYNC_UNSUPPORTED = new Set(["EINVAL", "ENOTSUP", "EOPNOTSUPP", "EISDIR", "EBADF"]);
/**
* Publish a rename durably. Windows cannot open a directory for fsync, and some filesystems
* refuse it; both are treated as supported-as-far-as-possible. Any other failure (EIO, ENOSPC...)
* is a storage error and is reported, so no termination follows an intent that may be lost.
*/
export function syncDirectory(path: string, io: { open?: typeof openSync; fsync?: typeof fsyncSync } = {}): boolean {
if (process.platform === "win32") return true;
let descriptor: number | null = null;
try {
descriptor = (io.open ?? openSync)(path, "r");
(io.fsync ?? fsyncSync)(descriptor);
return true;
} catch (error) {
return DIRECTORY_SYNC_UNSUPPORTED.has((error as NodeJS.ErrnoException).code ?? "");
} finally {
if (descriptor !== null) { try { closeSync(descriptor); } catch { /* already closed */ } }
}
}
export function appendBoundedLog(path: string, line: string, now = new Date()): void {
try {
mkdirSync(dirname(path), { recursive: true });
let size = 0;
try { size = statSync(path).size; } catch { /* absent */ }
const entry = `[${now.toISOString()}] ${line}\n`;
if (size + entry.length > LOG_CAP_BYTES) writeFileSync(path, entry, "utf8");
else writeFileSync(path, entry, { encoding: "utf8", flag: "a" });
} catch {
// Logging must never stop the sentinel.
}
}
export function readSentinelRuntime(path: string): SentinelRuntimeRecord | null {
try {
const parsed = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>;
const { pid, port, hostname } = parsed;
if (hostname !== undefined && typeof hostname !== "string") return null;
if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 0) return null;
if (typeof port !== "number" || !Number.isSafeInteger(port) || port <= 0 || port > 65_535) return null;
return { pid, port, ...(typeof hostname === "string" ? { hostname } : {}), sibling: parsed.siblingOfPort !== undefined };
} catch {
return null;
}
}
/** Partial timing from the test override; anything malformed is ignored. */
export function parseTimingOverride(raw: string | undefined): Partial<HealthSentinelTiming> {
if (!raw) return {};
try {
const parsed = JSON.parse(raw) as Record<string, unknown>;
const timing: Partial<HealthSentinelTiming> = {};
for (const key of Object.keys(DEFAULT_HEALTH_SENTINEL_TIMING) as (keyof HealthSentinelTiming)[]) {
const value = parsed[key];
if (typeof value === "number" && Number.isFinite(value) && value >= 0) timing[key] = value;
}
return timing;
} catch {
return {};
}
}
function parseStatePaths(raw: string | undefined): string[] | null {
try {
const parsed = JSON.parse(raw ?? "") as unknown;
return Array.isArray(parsed) && parsed.length > 0 && parsed.every(item => typeof item === "string") ? parsed : null;
} catch {
return null;
}
}
/** Entry used by `bun health-sentinel-main.ts` and by the compiled binary's private entrypoint. */
export async function runHealthSentinelProcess(env: NodeJS.ProcessEnv = process.env): Promise<number> {
const targetPid = Number(env[HEALTH_SENTINEL_ENV.targetPid]);
const runtimePath = env[HEALTH_SENTINEL_ENV.runtimePath];
const historyPath = env[HEALTH_SENTINEL_ENV.historyPath];
const logPath = env[HEALTH_SENTINEL_ENV.logPath];
const statePaths = parseStatePaths(env[HEALTH_SENTINEL_ENV.statePaths]);
if (!Number.isSafeInteger(targetPid) || targetPid <= 0 || !runtimePath || !historyPath || !logPath || !statePaths) return 2;
let gone = false;
const targetExited = new Promise<void>(resolve => {
const done = () => { gone = true; resolve(); };
process.stdin.on("end", done);
process.stdin.on("close", done);
process.stdin.on("error", done);
process.stdin.on("data", () => { /* the proxy never writes; drain anything */ });
process.stdin.resume();
});
const { acquireOwnershipMutationLease } = await import("./ownership-mutation-lease.mjs");
let windowsHandle: import("./health-sentinel-windows").ParentProcessHandle | null = null;
let windowsBindFailure: string | null = null;
if (process.platform === "win32") {
const { openParentProcessHandle } = await import("./health-sentinel-windows");
const opened = openParentProcessHandle(targetPid);
if (opened.ok) windowsHandle = opened.handle;
else windowsBindFailure = opened.reason;
}
let linuxHandle: import("./health-sentinel-linux").ParentPidfd | null = null;
let linuxBindFailure: string | null = null;
if (process.platform === "linux") {
const { openParentPidfd } = await import("./health-sentinel-linux");
const opened = openParentPidfd(targetPid, () => !gone && process.ppid === targetPid);
if (opened.kind === "bound") linuxHandle = opened.handle;
else if (opened.kind === "refused") linuxBindFailure = opened.reason;
else appendBoundedLog(logPath, `pidfd unavailable: ${opened.reason}; using live parent PID check`);
}
const lag = startLagMonitor();
try {
await runHealthSentinel({
targetPid,
now: Date.now,
sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
probe: async (target, timeoutMs) => {
lag.reset();
const outcome = await probeHealth(target, timeoutMs);
return outcome.kind === "timeout" ? { ...outcome, sentinelLagMs: lag.read() } : outcome;
},
readRuntime: () => readSentinelRuntime(runtimePath),
targetExited,
closeTarget: () => { linuxHandle?.close(); windowsHandle?.close(); },
isTargetGone: () => gone || (windowsHandle?.exited() ?? false),
bindTarget: () => {
// POSIX: Bun reads the parent id live, and it stops naming the proxy once the proxy exits.
// Windows: it is the creating parent's id, so the handle below carries the identity.
if (process.ppid !== targetPid) return { ok: false, reason: "the sentinel is not the watched process's child" };
if (linuxBindFailure) return { ok: false, reason: linuxBindFailure };
if (process.platform !== "win32") return { ok: true };
return windowsHandle ? { ok: true } : { ok: false, reason: windowsBindFailure ?? "the parent process cannot be opened" };
},
terminateTarget: () => {
if (process.platform === "win32") {
if (!windowsHandle) return "failed";
return windowsHandle.terminate(1) ? "terminated" : "failed";
}
if (gone || process.ppid !== targetPid) return "not_target";
if (linuxHandle) return linuxHandle.terminate();
try {
process.kill(targetPid, "SIGKILL");
return "terminated";
} catch {
return "failed";
}
},
withOwnershipLease: run => {
let lease: { release(): void };
try {
lease = acquireOwnershipMutationLease(statePaths, { waitMs: LEASE_WAIT_MS, processImage: () => null });
} catch {
return "busy";
}
try { return run(); } finally { lease.release(); }
},
readHistory: () => {
const entries = readRecoveryEntries(historyPath);
return entries === null ? null : countedRecoveryTimes(entries);
},
writeHistory: entry => writeRecoveryEntry(historyPath, entry),
log: line => appendBoundedLog(logPath, linuxHandle && line.startsWith("watching pid ") ? `${line} (pidfd)` : line),
timing: parseTimingOverride(env[HEALTH_SENTINEL_ENV.timing]),
});
} finally {
lag.stop();
}
return 0;
}
if (import.meta.main) process.exit(await runHealthSentinelProcess());