@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
177 lines (163 loc) • 8.59 kB
text/typescript
/**
* Start the opt-in service health sentinel beside a managed service proxy (#6671).
*
* The sentinel (health-sentinel-main.ts) is a separate process with its own event loop. It asks
* this proxy's `/healthz` and, if the proxy stays alive but stops answering, terminates it so the
* service manager restarts it. This module decides whether to start one, starts it, and stops it
* on shutdown. It never blocks startup: spawning is synchronous and fire-and-forget, and every
* failure is logged and ignored.
*
* Only a managed service child gets a sentinel. A foreground `ocx start` has nobody to restart it,
* a desktop sidecar has the app's own supervisor, and a sibling proxy is not the service.
*/
import { existsSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import type { OcxConfig } from "../types";
import { serviceHealthRecoveryEnabled } from "../config/feature-flags";
import { getConfigDir } from "../config/paths";
import { getRuntimePortPath, readRuntimePort } from "../config/process-state";
import { isManagedServiceEnvironment } from "../config/serving-runtimes";
import { didRunDrainStartHooks, registerDrainStartHook, registerOptionalShutdownHook } from "../lib/optional-shutdown-hooks";
import { isStandaloneBinary } from "../lib/standalone";
import { HEALTH_SENTINEL_ENV, readRecoveryEntries, type RecoveryEntry } from "./health-sentinel-main";
import { serviceStatePaths } from "./state";
/** Private argv the compiled binary handles before CLI dispatch (src/cli/index.ts). */
export const HEALTH_SENTINEL_STANDALONE_ARG = "__health-sentinel";
export const HEALTH_SENTINEL_HISTORY_FILE = "service-health-recovery.json";
export const HEALTH_SENTINEL_LOG_FILE = "service-health-sentinel.log";
/** A recovery this recent is announced by the replacement's first log lines. */
const RECENT_RECOVERY_NOTICE_MS = 15 * 60_000;
/** Variables the sentinel may inherit. Credentials and proxy settings are deliberately absent. */
const INHERITED_ENV = [
"PATH", "Path", "SystemRoot", "SYSTEMROOT", "windir", "WINDIR", "SystemDrive", "ComSpec",
"TEMP", "TMP", "TMPDIR", "HOME", "USERPROFILE", "LOCALAPPDATA", "APPDATA", "LANG", "TZ",
] as const;
export type HealthSentinelLaunchInput = {
enabled: boolean;
managedService: boolean;
standalone: boolean;
sibling: boolean;
execPath: string;
entryPath: string;
entryExists: boolean;
};
export type HealthSentinelLaunchDecision =
| { launch: true; argv: string[] }
/** `reason` is null when the feature is simply off, so nothing is logged. */
| { launch: false; reason: string | null };
export function decideHealthSentinelLaunch(input: HealthSentinelLaunchInput): HealthSentinelLaunchDecision {
if (!input.enabled) return { launch: false, reason: null };
if (!input.managedService) return { launch: false, reason: "this proxy is not running as the managed service" };
if (input.sibling) return { launch: false, reason: "this proxy is a sibling of the service" };
if (input.standalone) return { launch: true, argv: [input.execPath, HEALTH_SENTINEL_STANDALONE_ARG] };
if (!input.entryExists) return { launch: false, reason: "the sentinel entry is missing from this installation" };
return { launch: true, argv: [input.execPath, input.entryPath] };
}
export function healthSentinelEnvironment(
parent: NodeJS.ProcessEnv,
paths: { targetPid: number; runtimePath: string; historyPath: string; logPath: string; statePaths: readonly string[] },
): Record<string, string> {
const env: Record<string, string> = {};
for (const key of INHERITED_ENV) {
const value = parent[key];
if (value !== undefined) env[key] = value;
}
env[HEALTH_SENTINEL_ENV.targetPid] = String(paths.targetPid);
env[HEALTH_SENTINEL_ENV.runtimePath] = paths.runtimePath;
env[HEALTH_SENTINEL_ENV.historyPath] = paths.historyPath;
env[HEALTH_SENTINEL_ENV.logPath] = paths.logPath;
env[HEALTH_SENTINEL_ENV.statePaths] = JSON.stringify(paths.statePaths);
const timing = parent[HEALTH_SENTINEL_ENV.timing];
if (timing !== undefined) env[HEALTH_SENTINEL_ENV.timing] = timing;
return env;
}
/** The line a replacement prints when the sentinel recently terminated its predecessor. */
export function recentRecoveryNotice(entries: readonly RecoveryEntry[], selfPid: number, now: number): string | null {
const latest = entries.at(-1);
if (!latest || latest.pid === selfPid || latest.state === "skipped" || latest.state === "failed") return null;
if (now - latest.at < 0 || now - latest.at > RECENT_RECOVERY_NOTICE_MS) return null;
const seconds = Math.round((now - latest.at) / 1000);
const what = latest.state === "terminated" ? "terminated" : "tried to terminate";
return `⚠️ The service health sentinel ${what} the previous proxy (pid ${latest.pid}) ${seconds}s ago because it stopped answering /healthz. Requests it held were not replayed and may already have reached the upstream.`;
}
type SpawnedSentinel = { pid: number; stdin: unknown; kill(): void; unref(): void };
export type HealthSentinelHandle = { pid: number; stop(): void };
let active: { child: SpawnedSentinel; detachHooks: () => void } | null = null;
export function startServiceHealthSentinel(
config: Pick<OcxConfig, "serviceHealthRecovery">,
deps: {
env?: NodeJS.ProcessEnv;
spawn?: (argv: string[], env: Record<string, string>) => SpawnedSentinel;
log?: (line: string) => void;
now?: () => number;
} = {},
): HealthSentinelHandle | null {
const env = deps.env ?? process.env;
const log = deps.log ?? ((line: string) => console.log(line));
const enabled = serviceHealthRecoveryEnabled(config);
if (!enabled) return null;
// A drain that already began (a stop or restart during startup) must not gain a sentinel.
if (didRunDrainStartHooks()) return null;
const configDir = getConfigDir();
const historyPath = join(configDir, HEALTH_SENTINEL_HISTORY_FILE);
const notice = recentRecoveryNotice(readRecoveryEntries(historyPath) ?? [], process.pid, (deps.now ?? Date.now)());
if (notice) log(notice);
const entryPath = fileURLToPath(new URL("./health-sentinel-main.ts", import.meta.url));
const runtime = readRuntimePort(process.pid);
const decision = decideHealthSentinelLaunch({
enabled,
managedService: isManagedServiceEnvironment(env),
standalone: isStandaloneBinary(),
sibling: runtime?.siblingOfPort !== undefined,
execPath: process.execPath,
entryPath,
entryExists: existsSync(entryPath),
});
if (!decision.launch) {
if (decision.reason) log(`service health recovery is enabled but not started: ${decision.reason}`);
return null;
}
stopServiceHealthSentinel();
const childEnv = healthSentinelEnvironment(env, {
targetPid: process.pid,
runtimePath: getRuntimePortPath(),
historyPath,
logPath: join(configDir, HEALTH_SENTINEL_LOG_FILE),
statePaths: serviceStatePaths(),
});
let child: SpawnedSentinel;
try {
child = (deps.spawn ?? ((argv, childEnvironment) => Bun.spawn(argv, {
// The sentinel's stdin is this process's lifeline: it sees EOF when this process exits.
stdin: "pipe",
stdout: "ignore",
stderr: "ignore",
env: childEnvironment,
windowsHide: true,
})))(decision.argv, childEnv);
} catch (error) {
log(`service health recovery could not start its sentinel: ${error instanceof Error ? error.name : "error"}`);
return null;
}
child.unref();
// Keep the subprocess and its stdin sink referenced for this process's lifetime.
const handle = { child, detachHooks: () => {} };
active = handle;
// Withdraw as soon as any drain begins: a deliberate stop whose drain stalls must stay a stop.
const detachDrain = registerDrainStartHook("service-health-sentinel", stopServiceHealthSentinel);
const detachShutdown = registerOptionalShutdownHook("service-health-sentinel", stopServiceHealthSentinel);
handle.detachHooks = () => { detachDrain(); detachShutdown(); };
log(`service health recovery: sentinel pid ${child.pid} watches this proxy (pid ${process.pid})`);
return { pid: child.pid, stop: stopServiceHealthSentinel };
}
/** Stop the sentinel before a deliberate shutdown so it never judges a draining process. */
export function stopServiceHealthSentinel(): void {
const current = active;
if (!current) return;
active = null;
current.detachHooks();
try { current.child.kill(); } catch { /* already gone */ }
const sink = current.child.stdin as { end?: () => unknown } | null;
try { sink?.end?.(); } catch { /* already closed */ }
}