@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
1,028 lines • 51.6 kB
text/typescript
/**
* Runtime-state-first proxy liveness with identity checking.
*
* Historically `ensure`/`start` probed only `config.port` and accepted ANY 2xx /healthz:
* a proxy that started on a fallback port was invisible (duplicate starts, Codex synced
* back to a dead port), and a foreign app answering 200 on the configured port counted
* as "our proxy". Liveness now (1) prefers the pid + runtime-port record and (2) requires
* the /healthz body to identify as opencodex.
*
* Lives outside cli.ts (which dispatches argv at module top level) so tests can import it.
*/
import { loadConfig } from "../config";
import { isLoopbackHostname, isWildcardHostname } from "../codex/loopback-target";
import { normalizeBindHostname } from "./bind-host";
import {
hasPublishedOwnershipRecords, isOcxCommandLine, readAlivePid, readProcessCommandLine, readRuntimePort,
runtimeOwnerScope, sameRuntimeOwnerScope, verifyPidIdentity, type RuntimeOwnerScope,
} from "../config/process-state";
import { createServer, isIP } from "node:net";
import { networkInterfaces } from "node:os";
import { loopbackFamilyAvailability } from "../lib/loopback-family";
import {
LOCAL_ATTESTATION_CHALLENGE_HEADER,
LOCAL_ATTESTATION_PROOF_HEADER,
createLocalAttestationChallenge,
verifyLocalAttestationProof,
} from "../lib/local-management-attestation";
import { directLocalHttpFetch, isConnectPhaseTimeout } from "./direct-local-http";
export interface HealthzIdentity {
service?: unknown;
status?: unknown;
version?: unknown;
uptime?: unknown;
pid?: unknown;
port?: unknown;
/**
* Which listener answered: the standalone/hub server omits it, and the connected-client
* machine listener reports `"client"` (src/client/machine-listener.ts). It is the only
* on-the-wire way to tell those two apart, because both bind `config.port ?? 10100`.
*/
role?: unknown;
restartCapability?: unknown;
providerReloadCapability?: unknown;
asideSyncCapability?: unknown;
guiPairCapability?: unknown;
/** Present on a package-tree-fenced 503: the version on disk an in-place respawn will run. */
installedVersion?: unknown;
error?: unknown;
}
export type EndpointLiveness = "live" | "dead" | "unknown";
/**
* Whether this process can bind `hostname:port` right now. A successful bind proves no process
* owns that address, which no probe of it can: connecting to this machine's own non-loopback
* address (a Tailscale IP, a LAN IP) with nothing listening TIMES OUT on some hosts instead of
* being refused, and a timeout is indistinguishable from a slow owner. The listener is closed
* immediately. Any bind error (EADDRINUSE, EADDRNOTAVAIL, EACCES) answers false, so an address
* we cannot bind stays "unknown" and startup keeps refusing.
*/
const BIND_PROOF_TIMEOUT_MS = 1_500;
export function canBindAddress(hostname: string, port: number, timeoutMs = BIND_PROOF_TIMEOUT_MS, signal?: AbortSignal): Promise<boolean> {
if (!Number.isInteger(port) || port <= 0 || port > 65_535 || timeoutMs <= 0 || signal?.aborted) return Promise.resolve(false);
return new Promise<boolean>(resolve => {
const server = createServer(socket => socket.destroy());
let settled = false;
const finish = (bound: boolean): void => {
if (settled) return;
settled = true;
clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
try { server.close(); } catch { bound = false; }
resolve(bound);
};
const timer = setTimeout(() => finish(false), timeoutMs);
const onAbort = () => finish(false);
signal?.addEventListener("abort", onAbort, { once: true });
server.once("error", () => finish(false));
server.once("listening", () => finish(true));
try { server.listen({ host: hostname, port, exclusive: true, ipv6Only: hostname === "::" }); } catch { finish(false); }
});
}
export interface LivenessIo {
fetchFn?: typeof fetch;
readPidFn?: () => number | null;
/**
* Full identity check of the passed candidate pid; must return the SAME pid or null.
* Destructive callers only ever receive pids that passed this gate.
*/
verifyPidFn?: (candidatePid: number) => number | null;
/** Fresh command-line evidence: undefined/empty is unreadable, never confirmed foreign. */
readCommandLineFn?: typeof readProcessCommandLine;
loopbackAvailabilityFn?: typeof loopbackFamilyAvailability;
/**
* Test seam for the bind proof (see `canBindAddress`): whether `hostname:port` can be bound by
* this process right now, which proves no process owns that address. `hostname` is the raw
* socket address (IPv6 unbracketed) and `timeoutMs` the remaining discovery budget.
*/
bindProofFn?: (hostname: string, port: number, timeoutMs: number, signal?: AbortSignal) => Promise<boolean>;
readRuntimeFn?: (pid?: number) => { pid?: number; port: number; hostname?: string; ownerScope?: unknown } | null;
/** This process, as the self-record proof in {@link observeProxyLiveness} needs it (#8552). */
ownProcessFn?: () => OwnProcess;
configFn?: () => { port?: number; hostname?: string };
timeoutMs?: number;
/**
* How many times to retry a probe that failed with a transport error (timeout /
* connection refused). Definitive answers (non-OK HTTP, foreign /healthz body, pid
* mismatch) do not retry. Default 1 = no retry. Stop paths should pass 2–3 (#764).
*/
attempts?: number;
sleepFn?: (ms: number) => Promise<void>;
/**
* Absolute wall-clock deadline for discovery. When set, each probe attempt aborts
* once the remaining budget cannot cover another fetch — so multi-candidate
* `findLiveProxy` under `SERVICE_STOP_LIVENESS` cannot overrun the stop-path
* verification window (#764 / CodeRabbit).
*/
deadlineAt?: number;
nowFn?: () => number;
signal?: AbortSignal;
/** Enable the Windows-only loopback bind proof for startup ownership discovery. */
windowsLoopbackBindProof?: boolean;
/** Test seam for the Windows bind policy. */
platform?: NodeJS.Platform;
/**
* Also accept a proxy whose `/healthz` is fenced by the package-tree guard (#5496).
*
* Opt-in, because such a proxy is ours but not healthy: `ocx restart` and the stop paths
* must find it, while ensure, update health waits and replacement waits must not count it.
* A fenced body is never trusted by itself. It is accepted only when this home's runtime
* record names the same pid and port and the listener proves possession of that record's
* attestation secret for a fresh challenge.
*/
acceptPackageTreeFenced?: boolean;
/** Test seam for the fenced-identity challenge. */
createChallengeFn?: () => string;
}
/**
* Operator override for the per-probe fetch ceilings below (`OCX_PROBE_TIMEOUT_MS`),
* integer milliseconds in [1, MAX_PROBE_TIMEOUT_MS].
*
* Some hosts put a security layer (content filter, EDR network extension) in front of
* loopback TCP that adds a fixed per-connect cost, measured at about one second on an
* affected macOS machine. The shipped 750 ms probe then aborts before a healthy proxy can
* answer, and every CLI liveness consumer reports the proxy as down while a direct
* `curl /healthz` succeeds.
*
* The override only raises: each ceiling keeps its shipped floor (750 ms for the shared
* default, 1500 ms for the stop/start ownership budgets that guard against a duplicate
* proxy, #764, #5004), so a small value can never shorten them. Values above the 30 s
* ceiling are ignored rather than clamped: a stop multiplies its budget by the attempt
* count, and a typo must not turn a stop into a wait of minutes or days. Parsed once at
* module load; anything malformed falls back to the defaults and never breaks startup.
*/
export const MAX_PROBE_TIMEOUT_MS = 30_000;
const SHARED_PROBE_FLOOR_MS = 750;
const OWNERSHIP_PROBE_FLOOR_MS = 1500;
export function parseProbeTimeoutOverrideMs(raw: string | undefined): number | undefined {
const trimmed = raw?.trim();
if (!trimmed || !/^\d+$/.test(trimmed)) return undefined;
const n = Number(trimmed);
return n > 0 && n <= MAX_PROBE_TIMEOUT_MS ? n : undefined;
}
/** The ceiling for a probe whose shipped value is `floorMs`, raised by a valid override only. */
export function probeCeilingMs(floorMs: number, override: number | undefined): number {
return Math.max(floorMs, override ?? 0);
}
const probeTimeoutOverrideMs = parseProbeTimeoutOverrideMs(process.env.OCX_PROBE_TIMEOUT_MS);
/** Default per-probe fetch ceiling shared by liveness and readiness probes. */
export const DEFAULT_PROBE_TIMEOUT_MS = probeCeilingMs(SHARED_PROBE_FLOOR_MS, probeTimeoutOverrideMs);
/** Default probe options for service stop / orphan cleanup — a just-bound proxy can miss a single 750ms probe. */
export const SERVICE_STOP_LIVENESS: Pick<LivenessIo, "timeoutMs" | "attempts"> = {
timeoutMs: probeCeilingMs(OWNERSHIP_PROBE_FLOOR_MS, probeTimeoutOverrideMs),
attempts: 3,
};
/**
* Probe budget for a decision whose wrong answer starts a DUPLICATE proxy (#5004).
*
* `start` used the 750ms single-attempt default for both the pre-bind owner probe and
* (implicitly) the busy-port question behind the ephemeral hop. On Windows that answered
* "nothing is listening" for a proxy the previous command had just refused to shadow, and
* the hop then spawned a second instance that took over this home's pid/runtime records
* and re-pointed Codex at itself. A single unanswered probe is not evidence of absence
* when the failure mode is a duplicate instance, so the start path borrows the numbers
* the stop path already uses for the mirror-image decision.
*/
export const START_OWNERSHIP_LIVENESS: Pick<LivenessIo, "timeoutMs" | "attempts" | "windowsLoopbackBindProof"> = {
timeoutMs: probeCeilingMs(OWNERSHIP_PROBE_FLOOR_MS, probeTimeoutOverrideMs),
attempts: 3,
windowsLoopbackBindProof: true,
};
type LivenessFetch = (
input: string | URL | Request,
init?: RequestInit,
) => Promise<Response>;
export interface LiveProxy {
pid: number | null;
port: number;
/** Candidate bind scope, with the default loopback made explicit; never a probe alias. */
hostname?: string;
/** Successful recovery probe address, when discovery checked both loopback families. */
probeHostname?: string;
/** Whether the successful probe used runtime-port metadata or the configured listen port. */
source: "runtime" | "config";
/**
* Version the live proxy reported on `/healthz`, when it reported one.
*
* Carried so a stale `ocx` on PATH can be detected without a second request: the
* identity probe already parsed and validated this body. Absent for a legacy proxy whose
* healthz body predates the field.
*/
version?: string;
/**
* Role the live listener reported on `/healthz`, when it reported one: `"client"` for the
* connected-client machine listener, absent for a standalone or hub proxy.
*
* Liveness deliberately still ACCEPTS a client-role listener — see `isOpencodexHealthz`.
* The role is carried so a caller that needs a management plane (src/cli/runtime-api.ts)
* can refuse one, while `stop` and orphan cleanup keep finding the process they must act
* on. Absent for a proxy whose healthz body predates the field.
*/
role?: string;
/**
* The proxy answered with an attested package-tree fence (#5496): it is this home's process,
* but it is refusing traffic until it restarts. Only set for callers that opted in through
* `LivenessIo.acceptPackageTreeFenced`.
*/
packageTreeFenced?: true;
}
/**
* A /healthz identity proves only that a proxy holds the port. Before a destructive orphan stop,
* require that listener to prove possession of this home's runtime-record secret as well.
*
* The verdict is three-valued on purpose. "proven" is the only answer that authorizes the
* caller's action; "refuted" means the answer was definitive (no record, a pid/port
* mismatch, or a failed proof); "indeterminate" means transport, deadline, or an
* unattestable record left the question open, which a shared-write decision must treat
* the same as a live owner it simply could not verify (#6198).
*/
export type HomeOwnershipProof = "proven" | "refuted" | "indeterminate";
export async function proveLiveProxyOwnedByHome(live: LiveProxy, io: LivenessIo = {}): Promise<HomeOwnershipProof> {
if (live.pid === null) return "indeterminate";
return attestFencedIdentity(
`http://${probeHostname(live.probeHostname ?? live.hostname)}:${live.port}/healthz`,
live.port,
live.pid,
io,
io.fetchFn ?? directLocalHttpFetch,
io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS,
);
}
/**
* Host to probe for a given bind hostname: wildcards answer on IPv4 loopback, and raw
* IPv6 addresses must be bracketed or the composed URL is invalid.
*
* The wildcard test is `isWildcardHostname`, not a list of spellings. This function used to
* know exactly three (`0.0.0.0`, `::`, `[::]`) while the bind-scope predicate knew every
* all-zero form, so `ocx` composed `http://0.0.0.0.:10100` or `http://*:10100` — unreachable
* URLs — for a config the server itself treated as a wildcard bind. One predicate, both sides.
*/
export function probeHostname(hostname: string | undefined): string {
const trimmed = (hostname ?? "").trim();
if (!trimmed || isWildcardHostname(trimmed)) return "127.0.0.1";
if (trimmed.startsWith("[") && trimmed.endsWith("]")) return trimmed;
return trimmed.includes(":") ? `[${trimmed}]` : trimmed;
}
/**
* True when a /healthz body identifies an opencodex proxy. Accepts the explicit
* `service: "opencodex"` marker, plus the legacy `{status, version, uptime}` trio so a
* still-running pre-identity proxy (e.g. right after `ocx update`) is not mistaken for a
* foreign server and shadow-started over.
*
* The connected-client machine listener answers with the same `service: "opencodex"` marker
* and an extra `role: "client"`, and it is accepted here on purpose. Liveness answers "is one
* of our processes listening on this port", which is exactly what `ocx stop`, orphan cleanup,
* and duplicate-start avoidance need: rejecting the client role would make those paths blind to
* a real opencodex process and let them shadow-start over it. Callers that additionally need a
* management plane discriminate on `LiveProxy.role` instead of narrowing this predicate.
*/
export function isOpencodexHealthz(body: HealthzIdentity | null): boolean {
if (!body) return false;
if (body.service === "opencodex") return true;
if (body.service !== undefined) return false;
return body.status === "ok" && typeof body.version === "string" && typeof body.uptime === "number";
}
/**
* True for the exact 503 body the package-tree guard serves on `/healthz`. Shape only: this is
* what the listener CLAIMS, and whoever holds the port can claim it. Callers must attest it.
*/
export function isPackageTreeFencedHealthz(body: HealthzIdentity | null | undefined): boolean {
if (!body || body.service !== "opencodex" || body.status !== "restart_required") return false;
const error = body.error as { code?: unknown } | null | undefined;
return typeof error === "object" && error !== null && error.code === "package_tree_changed"
&& typeof body.pid === "number" && Number.isSafeInteger(body.pid) && body.pid > 0;
}
/**
* Prove that a fenced listener is the process this home's runtime record describes: the record
* must name the same pid and port and carry an attestation secret, and the listener must answer a
* fresh challenge with a proof bound to that secret, pid and port. The pid in the 503 body only
* selects which record to check; it is never accepted on its own.
*/
async function attestFencedIdentity(
url: string,
port: number,
pid: number,
io: LivenessIo,
fetchFn: LivenessFetch,
timeoutMs: number,
): Promise<HomeOwnershipProof> {
const readRuntimeFn = io.readRuntimeFn ?? readRuntimePort;
let record: ReturnType<NonNullable<LivenessIo["readRuntimeFn"]>>;
try {
record = readRuntimeFn(pid);
} catch {
return "indeterminate";
}
// The typed seam omits the secret; the production record (readRuntimePort) carries it.
const secret: unknown = record ? Reflect.get(record, "attestationSecret") : undefined;
if (!record || record.pid !== pid || record.port !== port) return "refuted";
if (typeof secret !== "string" || secret.length === 0) return "indeterminate";
const challenge = (io.createChallengeFn ?? createLocalAttestationChallenge)();
// One proof failure is definitive and never retried; a transport failure only means
// the listener did not answer yet, so it gets the same bounded retry the identity
// probe uses — "did not answer" is not "not ours" (#6198). The challenge is minted
// once: a retried attempt proves the same fresh nonce, not a replayed proof.
const sleepFn = io.sleepFn ?? ((ms: number) => new Promise<void>(r => setTimeout(r, ms)));
const nowFn = io.nowFn ?? Date.now;
const requestedAttempts = Math.trunc(io.attempts ?? 1);
const attempts = Number.isNaN(requestedAttempts)
? 1
: Math.max(1, Math.min(requestedAttempts, 5));
for (let attempt = 1; attempt <= attempts; attempt++) {
const remainingMs = io.deadlineAt === undefined ? timeoutMs : Math.min(timeoutMs, io.deadlineAt - nowFn());
if (remainingMs <= 0) return "indeterminate";
try {
const res = await fetchFn(url, {
headers: { [LOCAL_ATTESTATION_CHALLENGE_HEADER]: challenge },
signal: AbortSignal.timeout(remainingMs),
});
const body = (await res.json().catch(() => null)) as HealthzIdentity | null;
// The second answer must still be the same fenced (or by now healthy) process.
if (!isOpencodexHealthz(body) && !isPackageTreeFencedHealthz(body)) return "refuted";
if (body?.pid !== pid) return "refuted";
return verifyLocalAttestationProof(secret, challenge, pid, port, res.headers.get(LOCAL_ATTESTATION_PROOF_HEADER))
? "proven"
: "refuted";
} catch {
if (attempt >= attempts) return "indeterminate";
await sleepFn(100);
}
}
return "indeterminate";
}
/** A bounded version string safe to carry beyond the untrusted health response. */
export function isHealthzVersion(value: unknown): value is string {
return typeof value === "string"
&& value.length <= 64
&& /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/.test(value);
}
/**
* "Nothing is listening" is narrower than "the probe failed". Only a connect-phase refusal
* proves the endpoint is free; a timeout, reset, or other transport failure leaves the
* question open.
*/
export function isConnectionRefused(error: unknown): boolean {
const visit = (current: unknown, depth: number): boolean => {
if (depth >= 4) return false;
if (current === null || (typeof current !== "object" && typeof current !== "function")) return false;
const record = current as { code?: unknown; cause?: unknown; errors?: unknown };
if (record.errors !== undefined) {
// One connect attempt fanned out over several addresses reports a single AggregateError.
// Only a unanimous refusal proves the endpoint is free: a bundle that mixes ECONNREFUSED
// with a timeout means one address answered nothing at all, and an address whose state is
// unreadable is unknown, not absence. Collapsing it to "refused" is how a second runtime
// gets started on a port that already has one.
return Array.isArray(record.errors) && record.errors.length > 0 && record.errors.every(error => visit(error, depth + 1));
}
if (record.code === "ECONNREFUSED" || record.code === "ConnectionRefused") return true;
if (typeof record.code === "string" && record.code.endsWith("ECONNREFUSED")) return true;
return visit(record.cause, depth + 1);
};
return visit(error, 0);
}
/**
* A connect the host refused to start (Windows filtering can deny `::1` outright) says no more
* about a listener than a connect timeout does, so it qualifies for the same Windows bind proof.
*/
function isConnectAccessDenied(error: unknown): boolean {
const record = error as { code?: unknown; syscall?: unknown } | null;
return record?.code === "EACCES" && record.syscall === "connect";
}
export async function classifyHealthz(
url: string,
fetchFn: LivenessFetch,
timeoutMs: number,
): Promise<EndpointLiveness> {
return (await classifyHealthzDetailed(url, fetchFn, timeoutMs)).verdict;
}
/** {@link classifyHealthz} plus whether an unknown verdict was a connect that timed out (#8318). */
async function classifyHealthzDetailed(
url: string,
fetchFn: LivenessFetch,
timeoutMs: number,
): Promise<{ verdict: EndpointLiveness; connectTimeout: boolean }> {
const answer = (verdict: EndpointLiveness, connectTimeout = false) => ({ verdict, connectTimeout });
try {
const response = await fetchFn(url, { signal: AbortSignal.timeout(timeoutMs) });
if (response.status !== 200) return answer("unknown");
const body = (await response.json().catch(() => undefined)) as HealthzIdentity | null | undefined;
if (body === undefined) return answer("unknown");
return answer(isOpencodexHealthz(body) ? "live" : "dead");
} catch (error) {
return isConnectionRefused(error) ? answer("dead") : answer("unknown", isConnectPhaseTimeout(error));
}
}
/** Whether `address` (unbracketed) is assigned to one of this machine's network interfaces. */
export function isLocalInterfaceAddress(address: string): boolean {
const canonical = (value: string) => isIP(value) === 6 ? new URL(`http://[${value.split("%")[0]}]/`).hostname.slice(1, -1) : value;
const wanted = canonical(address);
try {
return Object.values(networkInterfaces()).some(entries =>
(entries ?? []).some(entry => canonical(entry.address) === wanted));
} catch {
return false;
}
}
/**
* Tri-state probe of one endpoint, the in-process counterpart of
* `src/update/proxy-liveness-probe.mjs`. Only a connect-phase refusal or a clean 200 that is
* not ours proves "dead"; a timeout, reset, non-200 or unreadable body leaves the question
* open. Loopback endpoints are checked on both IPv4 and IPv6 because a listener may bind only
* one family. Runs in-process because a compiled standalone binary cannot fork `execPath -e`.
*/
export async function probeEndpointLiveness(
endpoint: { port: number; hostname?: string },
io: Pick<LivenessIo, "fetchFn" | "timeoutMs" | "bindProofFn"> & {
/** Test seam for {@link isLocalInterfaceAddress}. */
localAddressFn?: (address: string) => boolean;
} = {},
): Promise<EndpointLiveness> {
if (!Number.isFinite(endpoint.port) || endpoint.port <= 0 || endpoint.port > 65535) return "dead";
const fetchFn = io.fetchFn ?? directLocalHttpFetch;
const timeoutMs = io.timeoutMs ?? 1500;
let sawUnknown = false;
for (const hostname of loopbackProbeHosts(endpoint.hostname)) {
const { verdict, connectTimeout } = await classifyHealthzDetailed(
`http://${hostname}:${endpoint.port}/healthz`,
fetchFn,
timeoutMs,
);
if (verdict === "live") return "live";
if (verdict === "unknown" && !(connectTimeout && await provenFreeByBind(hostname, endpoint.port, io))) sawUnknown = true;
}
return sawUnknown ? "unknown" : "dead";
}
/**
* #8318: a connect to this machine's own non-loopback address (a Tailscale IP) with nothing
* listening can time out instead of being refused, which left `ocx resolve` unknown forever. Only
* a connect-phase timeout on a specific address of a local interface qualifies, and only an
* exclusive bind that succeeds (and is released at once) proves absence; EADDRINUSE or any other
* bind error keeps the verdict unknown. Loopback and wildcard binds keep their refusal-only rule,
* because a wildcard listener can share the port with a specific bind on some platforms.
*/
async function provenFreeByBind(
probeHost: string,
port: number,
io: { bindProofFn?: LivenessIo["bindProofFn"]; localAddressFn?: (address: string) => boolean },
): Promise<boolean> {
const address = probeHost.replace(/^\[(.*)\]$/, "$1");
if (isIP(address) === 0 || recoveryLoopbackFamily(address) !== null || isWildcardHostname(address)) return false;
if (!(io.localAddressFn ?? isLocalInterfaceAddress)(address)) return false;
return await (io.bindProofFn ?? canBindAddress)(address, port, BIND_PROOF_TIMEOUT_MS);
}
/** Identity-checked /healthz probe; null when unreachable, non-OK, or not our proxy. */
export async function proxyIdentityAt(
port: number,
opts: { hostname?: string; expectedPid?: number } = {},
io: LivenessIo = {},
): Promise<{ pid: number | null; version?: string; role?: string; packageTreeFenced?: true } | null> {
return proxyIdentityAtObserved(port, opts, io);
}
/** Preserve negative evidence for recovery without changing identity-only callers. */
async function proxyIdentityAtObserved(
port: number,
opts: { hostname?: string; expectedPid?: number },
io: LivenessIo,
observe?: (result: EndpointLiveness, onlyConnectTimeouts?: boolean) => void,
): ReturnType<typeof proxyIdentityAt> {
const negative = (result: "dead" | "unknown", onlyConnectTimeouts = false): null => {
observe?.(result, onlyConnectTimeouts);
return null;
};
const fetchFn = io.fetchFn ?? directLocalHttpFetch;
const sleepFn = io.sleepFn ?? ((ms: number) => new Promise<void>(r => setTimeout(r, ms)));
const nowFn = io.nowFn ?? Date.now;
const baseTimeoutMs = io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
const requestedAttempts = Math.trunc(io.attempts ?? 1);
const attempts = Number.isNaN(requestedAttempts)
? 1
: Math.max(1, Math.min(requestedAttempts, 5));
let sawUnknown = false;
let sawOtherFailure = false;
for (let attempt = 1; attempt <= attempts; attempt++) {
const remainingMs = io.deadlineAt === undefined ? baseTimeoutMs : io.deadlineAt - nowFn();
if (remainingMs <= 0 || io.signal?.aborted) return negative("unknown");
const timeoutMs = Math.min(baseTimeoutMs, remainingMs);
try {
const url = `http://${probeHostname(opts.hostname)}:${port}/healthz`;
const timeout = AbortSignal.timeout(timeoutMs);
const res = await fetchFn(url, {
signal: io.signal ? AbortSignal.any([io.signal, timeout]) : timeout,
});
if (!res.ok && io.acceptPackageTreeFenced && res.status === 503) {
const fenced = (await res.json().catch(() => null)) as HealthzIdentity | null;
if (!isPackageTreeFencedHealthz(fenced)) return negative("unknown");
const fencedPid = fenced!.pid as number;
if (opts.expectedPid !== undefined && fencedPid !== opts.expectedPid) return negative("unknown");
const attestMs = io.deadlineAt === undefined ? baseTimeoutMs : Math.min(baseTimeoutMs, io.deadlineAt - nowFn());
if (attestMs <= 0) return negative("unknown");
if ((await attestFencedIdentity(url, port, fencedPid, io, fetchFn, attestMs)) !== "proven") return negative("unknown");
const fencedVersion = isHealthzVersion(fenced?.version) ? fenced!.version as string : undefined;
return {
pid: fencedPid,
...(fencedVersion === undefined ? {} : { version: fencedVersion }),
packageTreeFenced: true,
};
}
if (!res.ok) return negative("unknown");
const body = (await res.json().catch(() => undefined)) as HealthzIdentity | null | undefined;
if (!isOpencodexHealthz(body ?? null)) return negative(res.status === 200 && body !== undefined ? "dead" : "unknown");
const pid = typeof body?.pid === "number" ? body.pid : null;
if (opts.expectedPid !== undefined && pid !== null && pid !== opts.expectedPid) return negative("unknown");
// Whoever holds the port controls this response. Only carry bounded semver text into
// diagnostics; dropping anything else prevents terminal controls reaching human output.
const version = isHealthzVersion(body?.version) ? body.version : undefined;
// Same guard for the role, for the same reason: absent on a standalone/hub proxy and on
// a legacy body, and never coerced from a non-string.
const role = typeof body?.role === "string" ? body.role : undefined;
return {
pid,
...(version === undefined ? {} : { version }),
...(role === undefined ? {} : { role }),
};
} catch (error) {
// Transport failure (timeout / refused) — retry while budget remains; a proxy that
// has only just begun listening can miss a single short probe (#764).
if (!isConnectionRefused(error)) {
sawUnknown = true;
if (!isConnectPhaseTimeout(error) && !isConnectAccessDenied(error)) sawOtherFailure = true;
}
if (io.signal?.aborted) return negative("unknown");
if (attempt >= attempts) return negative(sawUnknown ? "unknown" : "dead", sawUnknown && !sawOtherFailure);
if (io.deadlineAt !== undefined && io.deadlineAt - nowFn() <= 0) return negative("unknown");
await sleepFn(100);
}
}
return negative("unknown");
}
/** Startup identity and absence share one evidence history for this literal endpoint. */
export async function observeStartupEndpoint(port: number, opts: { hostname: string; expectedPid?: number }, io: LivenessIo) {
let verdict: EndpointLiveness = "unknown";
let connectTimedOut = false;
const identity = await proxyIdentityAtObserved(port, opts, io, (result, onlyConnectTimeouts) => {
verdict = result;
connectTimedOut = onlyConnectTimeouts === true;
});
const bindProof = !identity && connectTimedOut ? await proveWindowsLoopbackVacant(opts.hostname, port, io) : undefined;
return { identity, verdict: bindProof ? "dead" as const : verdict, bindProof };
}
/** Undefined means the endpoint is outside the Windows loopback proof policy. */
async function proveWindowsLoopbackVacant(hostname: string, port: number, io: LivenessIo): Promise<boolean | undefined> {
if (!io.windowsLoopbackBindProof || (io.platform ?? process.platform) !== "win32") return undefined;
const address = hostname.replace(/^\[(.*)\]$/, "$1");
const family = isIP(address);
if (!family || recoveryLoopbackFamily(address) !== family) return undefined;
const now = io.nowFn ?? Date.now;
const deadline = Math.min(io.deadlineAt ?? Infinity, now() + BIND_PROOF_TIMEOUT_MS);
const bind = io.bindProofFn ?? canBindAddress;
// Winsock can allow a specific exclusive bind beside an earlier nonexclusive wildcard
// listener under the same user. Require the exact bind and rule out that wildcard too.
for (const host of [address, family === 4 ? "0.0.0.0" : "::"]) {
const remaining = deadline - now();
if (remaining <= 0 || io.signal?.aborted) return false;
try { if (!await bind(host, port, remaining, io.signal)) return false; }
catch { return false; }
}
return now() < deadline && !io.signal?.aborted;
}
export type ProxyLivenessObservation =
| { kind: "live"; live: LiveProxy }
| { kind: "unknown"; live: null }
| { kind: "absent"; live: null; loopbackBindProven?: true };
export interface OwnProcess {
pid: number;
/** {@link hasPublishedOwnershipRecords}: this process has written ownership records itself. */
publishedOwnership: boolean;
scope: RuntimeOwnerScope;
}
function currentOwnProcess(): OwnProcess {
return { pid: process.pid, publishedOwnership: hasPublishedOwnershipRecords(), scope: runtimeOwnerScope() };
}
/** Recovery additionally covers both available loopback families and fresh PID identity. */
export async function observeProxyLiveness(io: LivenessIo = {}): Promise<ProxyLivenessObservation> {
let alivePid: number | null = null, runtimePid: number | undefined, runtimeScope: unknown, unknown = false;
let loopbackBindProven = false;
const live = await discoverLiveProxy(io, {
pid: pid => { alivePid = pid; },
runtimePid: (pid, ownerScope) => { runtimePid = pid; runtimeScope = ownerScope; },
endpoint: (result, bindProven) => {
if (result === "unknown") unknown = true;
if (bindProven) loopbackBindProven = true;
},
});
if (live) return { kind: "live", live };
// #8552: in a container the proxy is PID 1 on every start, so the record the previous container
// left names this very process; signalling it succeeds and its command line is ocx. Such a
// record is stale only when this process never published ownership itself (so the writer was an
// earlier lifetime of the PID number) AND the runtime record's owner ran where this process runs
// (same host and boot, plus the same network namespace or container). That scope check keeps a
// RollingUpdate safe: the old pod is also PID 1 and its listener sits in another network
// namespace, which the endpoint probe cannot see. A record without a scope (an older version,
// which may still be serving during an upgrade) keeps the refusal.
const own = (io.ownProcessFn ?? currentOwnProcess)();
const ownEarlierLifetime = !own.publishedOwnership && runtimePid === own.pid && sameRuntimeOwnerScope(runtimeScope, own.scope);
const candidates = new Set<number>();
if (alivePid !== null) candidates.add(alivePid);
if (runtimePid !== undefined && Number.isSafeInteger(runtimePid) && runtimePid > 0) candidates.add(runtimePid);
for (const pid of candidates) {
if (unknown) break;
if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) { unknown = true; break; }
if (ownEarlierLifetime && pid === own.pid) continue;
if (pid !== alivePid) {
try { process.kill(pid, 0); } catch (error) {
if ((error as NodeJS.ErrnoException).code === "ESRCH") continue;
// Permission or an indeterminate probe is not proof that the process is dead.
unknown = true;
break;
}
}
// A failed verifier alone conflates foreign with unreadable. Only fresh positive
// evidence of a foreign command line releases a reused PID; never signal it.
const commandLine = (io.readCommandLineFn ?? readProcessCommandLine)(pid);
unknown = !commandLine?.trim() || isOcxCommandLine(commandLine);
if (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt) unknown = true;
}
if (io.signal?.aborted || (io.deadlineAt !== undefined && (io.nowFn ?? Date.now)() >= io.deadlineAt)) unknown = true;
if (unknown) return { kind: "unknown", live: null };
return { kind: "absent", live: null, ...(loopbackBindProven ? { loopbackBindProven: true } : {}) };
}
/**
* Locate the live proxy: pid file → runtime-port record → identity probe. Falls back to
* the configured port ONLY when no runtime record answers, so a fallback-port proxy is
* found and a foreign listener on the configured port is rejected.
*/
export async function findLiveProxy(io: LivenessIo = {}): Promise<LiveProxy | null> {
return discoverLiveProxy(io);
}
async function discoverLiveProxy(io: LivenessIo, observer?: {
pid: (pid: number | null) => void;
runtimePid: (pid: number | undefined, ownerScope: unknown) => void;
endpoint: (result: EndpointLiveness, loopbackBindProven?: boolean) => void;
}): Promise<LiveProxy | null> {
// Prefer the cheap alive-pid check: the Windows cmdline probe (WMIC/PowerShell) is too
// expensive for waitForProxy's 150ms poll loop, and /healthz identity is the real trust gate.
const readPidFn = io.readPidFn ?? readAlivePid;
const verifyPidFn = io.verifyPidFn ?? verifyPidIdentity;
const readRuntimeFn = io.readRuntimeFn ?? readRuntimePort;
const configFn = io.configFn ?? loadConfig;
const nowFn = io.nowFn ?? Date.now;
const deadlineAt = io.deadlineAt;
const probeIo: LivenessIo = io;
const budgetExhausted = (): boolean =>
io.signal?.aborted === true || (deadlineAt !== undefined && nowFn() >= deadlineAt);
const expired = (): null => { observer?.endpoint("unknown"); return null; };
// The proof stays inside the discovery deadline: it is skipped once the budget is spent, its
// own timeout is capped by what remains, and a proof that finishes past the deadline is
// discarded, so an exhausted discovery still answers "unknown". A socket takes the raw
// address, never the bracketed URL spelling `probeHostname` composes for IPv6.
const bindProof = async (hostname: string, port: number): Promise<boolean> => {
const remainingMs = deadlineAt === undefined ? BIND_PROOF_TIMEOUT_MS : Math.min(BIND_PROOF_TIMEOUT_MS, deadlineAt - nowFn());
if (remainingMs <= 0 || budgetExhausted()) return false;
const socketHost = hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
// A hostname can resolve to different addresses for the probe and the bind.
if (isIP(socketHost) === 0) return false;
const bound = await (io.bindProofFn ?? canBindAddress)(socketHost, port, remainingMs, io.signal);
return bound && !budgetExhausted();
};
const probe = async (port: number, opts: { hostname?: string; expectedPid?: number }) => {
// Missing/blank means an actual loopback bind, not a later config's bind scope.
const bindHostname = opts.hostname?.trim() || "127.0.0.1";
const hostname = normalizeBindHostname(opts.hostname);
if (!observer || (!recoveryLoopbackFamily(hostname) && !isWildcardHostname(hostname))) {
// Recovery only (an observer is present): a transport miss on a specific non-loopback bind
// address is "unknown", and a bind proof is the one thing that turns it into "dead".
let verdict: EndpointLiveness | undefined;
const identity = await proxyIdentityAtObserved(port, opts, probeIo, observer ? (r => { verdict = r; }) : undefined);
if (observer && !identity && verdict !== undefined) {
if (verdict === "unknown" && await bindProof(hostname, port)) verdict = "dead";
observer.endpoint(verdict);
}
return identity ? { ...identity, hostname: bindHostname,
...(observer ? { probeHostname: probeHostname(opts.hostname) } : {}) } : null;
}
// Parallel families share the existing per-probe attempts/timeout and absolute
// deadline. A slow family cannot multiply the serial discovery budget.
const hosts = recoveryProbeHosts(hostname).filter(host =>
(io.loopbackAvailabilityFn ?? loopbackFamilyAvailability)(host) !== "unavailable");
const identities = await Promise.all(hosts.map(async hostname => {
const { identity, verdict, bindProof } = await observeStartupEndpoint(port, { ...opts, hostname }, probeIo);
if (!identity) observer.endpoint(verdict, bindProof);
return identity ? { ...identity, hostname: bindHostname, probeHostname: hostname } : null;
}));
return identities.find(identity => identity !== null) ?? null;
};
// The cheap pid is discovery-only. Before it can appear in a returned (killable) result
// it must pass the full identity check AND the verifier must echo the exact candidate —
// a pidfile rewrite between discovery and verification can never swap in another process.
const killablePid = (candidate: number | null): number | null => {
if (candidate === null) return null;
const verified = verifyPidFn(candidate);
return verified === candidate ? verified : null;
};
const verifiedReportedPid = (reported: number | null): number | null => {
if (reported === null) return null;
if (!Number.isSafeInteger(reported) || reported <= 0) return null;
const verified = verifyPidFn(reported);
return verified === reported ? verified : null;
};
const pid = readPidFn();
observer?.pid(pid);
let probedPort: number | null = null;
if (pid) {
const runtime = readRuntimeFn(pid);
if (runtime?.port) {
if (budgetExhausted()) return expired();
probedPort = runtime.port;
const identity = await probe(runtime.port, { hostname: runtime.hostname, expectedPid: pid });
if (identity) {
// healthz confirmed the pid itself → trusted; a pidless legacy body did not,
// so the cheap pid must pass full identity verification before it is returned.
const trusted = identity.pid === pid ? pid : killablePid(pid);
return {
pid: trusted,
port: runtime.port,
hostname: identity.hostname,
...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }),
source: "runtime",
...(identity.version === undefined ? {} : { version: identity.version }),
...(identity.role === undefined ? {} : { role: identity.role }),
...(identity.packageTreeFenced ? { packageTreeFenced: true as const } : {}),
};
}
}
}
// Orphan recovery: the pid file can be lost/corrupt while the proxy is alive (crash of a
// sibling command, manual deletion). The runtime record still says where it listens —
// identity-probe it so ensure/update/stop see the live proxy instead of shadowing it.
const record = readRuntimeFn();
observer?.runtimePid(record?.pid, record?.ownerScope);
if (record?.port && record.port !== probedPort) {
if (budgetExhausted()) return expired();
const expectedPid = typeof record.pid === "number" ? record.pid : undefined;
const identity = await probe(record.port, { hostname: record.hostname, expectedPid });
// Only the healthz-reported pid is authoritative here. The record's pid may be stale
// (its process dead, the port reused by a pidless legacy proxy) — synthesizing it
// would hand destructive callers (stopProxy → kill fallback) a reusable pid.
if (identity) {
return {
pid: verifiedReportedPid(identity.pid),
port: record.port,
hostname: identity.hostname,
...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }),
source: "runtime",
...(identity.version === undefined ? {} : { version: identity.version }),
...(identity.role === undefined ? {} : { role: identity.role }),
...(identity.packageTreeFenced ? { packageTreeFenced: true as const } : {}),
};
}
}
const config = configFn();
const port = config.port ?? 10100;
if (budgetExhausted()) return expired();
// Port zero requests an OS-assigned port; it cannot name an existing listener.
if (observer && port === 0) return null;
const identity = await probe(port, { hostname: config.hostname });
if (identity) {
return {
pid: verifiedReportedPid(identity.pid) ?? killablePid(pid),
port,
hostname: identity.hostname,
...(identity.probeHostname === undefined ? {} : { probeHostname: identity.probeHostname }),
source: "config",
...(identity.version === undefined ? {} : { version: identity.version }),
...(identity.role === undefined ? {} : { role: identity.role }),
...(identity.packageTreeFenced ? { packageTreeFenced: true as const } : {}),
};
}
return null;
}
// Recovery classification is intentionally local: it must not broaden admission or bind policy.
function recoveryLoopbackFamily(hostname: string): 4 | 6 | null {
if (isLoopbackHostname(hostname)) return hostname.includes(":") ? 6 : 4;
const address = hostname.replace(/^\[(.*)\]$/, "$1");
const family = isIP(address);
if (family === 4) return address.startsWith("127.") ? 4 : null;
if (family !== 6) return null;
// URL serializes validated IPv6 literals canonically, including dotted IPv4-mapped forms.
const canonical = new URL(`http://[${address.split("%")[0]}]/`).hostname.slice(1, -1);
if (canonical === "::1") return 6;
return /^::ffff:7f[0-9a-f]{2}:[0-9a-f]{1,4}$/.test(canonical) ? 4 : null;
}
function recoveryProbeHosts(hostname: string): string[] {
if (isWildcardHostname(hostname)) return ["127.0.0.1", "[::1]"];
const family = recoveryLoopbackFamily(hostname);
if (family) return [probeHostname(hostname), family === 6 ? "127.0.0.1" : "[::1]"];
return [probeHostname(hostname)];
}
/**
* Loopback addresses to ask about a port whose holder must be identified before a
* decision that is destructive when it answers "nobody".
*
* A listener and a probe can disagree about what "loopback" means, and on Windows they
* do. `startServer` canonicalizes a `localhost` bind to 127.0.0.1 precisely because
* Windows resolves the name IPv6-first (src/server/index.ts), while `probeHostname`
* hands the literal name back and leaves the family choice to the resolver. A probe that
* lands on `::1` while the listener holds `127.0.0.1` reports an empty port that another
* process is demonstrably serving. Both literal addresses are therefore asked, cheapest
* question first: the extra one costs a refused connection, and skipping it costs a
* duplicate proxy.
*
* A non-loopback bind (a LAN address, a named host) gets exactly one candidate — the
* address it was configured with. Guessing another interface for it would answer a
* different question than the caller asked.
*/
export function loopbackProbeHosts(hostname: string | undefined): string[] {
const normalized = normalizeBindHostname(hostname);
const primary = probeHostname(normalized);
if (isLoopbackHostname(normalized) || isWildcardHostname(normalized)) {
return primary === "[::1]" ? ["[::1]", "127.0.0.1"] : ["127.0.0.1", "[::1]"];
}
return [primary];
}
/**
* Identity-checked answer to "who holds this exact port", independent of the pid file
* and the runtime-port record.
*
* `findLiveProxy` answers "is a proxy of this home alive", and it can only do that from
* recorded state plus the configured port. This answers the narrower question a start
* has to ask before it walks away from a busy port: an opencodex listening THERE, right
* now, whatever this home's records say about it. Returns null only after every
* candidate address has failed the identity check with the caller's full probe budget.
*/
export async function probePortOwner(
port: number,
opts: { hostname?: string } = {},
io: LivenessIo = {},
): Promise<{ pid: number | null; hostname: string; version?: string; role?: string } | null> {
for (const hostname of loopbackProbeHosts(opts.hostname)) {
const identity = await proxyIdentityAt(port, { hostname }, io);
if (identity) return { ...identity, hostname };
}
return null;
}
// ─────────────────────────────────────────────────────────────────────────────
// Readiness (/readyz) strict probe.
//
// Liveness (/healthz) confirms the process answers; readiness confirms the
// post-startup Codex sync has settled. A readiness probe is identity-checked the
// same way liveness is, AND additionally enforces the full /readyz contract so
// an adversarial or malformed body can never count as ready:
//
// - HTTP 200 is required for status="ready"; HTTP 503 is required for pending
// or failed. Any other HTTP/body-status pairing is an invalid contract.
// - body.service must be exactly "opencodex".
// - body.version must be a non-empty string.
// - body.uptime must be a finite nonnegative number.
// - body.pid must be a positive integer; when `expectedPid` is supplied it must
// match exactly.
// - body.port must be an integer in 1..65535 and equal the probed port.
// - body.status must be exactly one of pending|ready|failed.
//
// Any unreachable, foreign, legacy, malformed, mismatched, or self-inconsistent
// response returns `null` so callers can never treat an invalid identity/contract
// as ready.
// ─────────────────────────────────────────────────────────────────────────────
interface ReadyzBody {
service?: unknown;
version?: unknown;
uptime?: unknown;
pid?: unknown;
port?: unknown;
status?: unknown;
// Remote protocol metadata is intentionally additive here. Ordinary
// readiness remains compatible with legacy standalone servers; `ocx connect`
// validates these fields separately in src/remote/protocol.ts.
protocol?: unknown;
minimumClientProtocol?: unknown;
managementUrl?: unknown;
}
export interface ReadinessProbeResult {
/** True ONLY for a valid 200 + status="ready" body with a matching pid. */
ready: boolean;
/** Fixed sanitized status. A foreign/unreadable body yields a `null` RESULT, never a `null` status. */
status: "ready" | "pending" | "failed";
/** Positive integer pid from a valid body. */
pid: number;
/** Integer port from a valid body. */
port: number;
}
export interface ReadinessProbeIo {
fetchFn?: typeof fetch;
timeoutMs?: number;
}
const READYZ_STATUS_VALUES = new Set<"ready" | "pending" | "failed">(["ready", "pending", "failed"]);
/**
* Validate a parsed /readyz body against the strict contract. Returns the
* sanitized probe result, or `null` when the body is foreign, legacy,
* malformed, or fails the pid/port checks. Pure (no I/O) so it is fully
* deterministic and unit-testable.
*/
export function validateReadyzBody(
body: unknown,
port: number,
opts: { expectedPid?: number } = {},
): ReadinessProbeResult | null {
if (!body || typeof body !== "object") return null;
const b = body as ReadyzBody;
if (b.service !== "opencodex") return null;
if (typeof b.version !== "string" || b.version.length === 0) return null;
if (typeof b.uptime !== "number" || !Number.isFinite(b.uptime) || b.uptime < 0) return null;
if (typeof b.pid !== "number" || !Number.isInteger(b.pid) || b.pid <= 0) return null;
if (
typeof b.port !== "number"
|| !Number.isInteger(b.port)
|| b.port < 1
|| b.port > 65535
|| b.port !== port
) return null;
if (typeof b.status !== "string" || !READYZ_STATUS_VALUES.has(b.status as "ready" | "pending" | "failed")) return null;
const status = b.status as "ready" | "pending" | "failed";
if (opts.expectedPid !== undefined && b.pid !== opts.expectedPid) return null;
return { ready: status === "ready", status, pid: b.pid, port: b.port };
}
/**
* Identity- and contract-checked /readyz probe. Returns `null` when the
* endpoint is unreachable or the body fails the strict contract (foreign 200,
* legacy health-only body, non-JSON, missing/malformed/mismatched fields,
* wrong port/pid, or an HTTP/body-status inconsistency). Returns
* `{ready:false, ...}` when the body is ours but pending or failed. Returns
* `{ready:true, ...}` ONLY for a valid 200 body with `status:"ready"` and (when
* requested) a matching pid.
*/
export async function probeReadiness(
port: number,
opts: { hostname?: string; expectedPid?: number } = {},
io: ReadinessProbeIo = {},
): Promise<ReadinessProbeResult | null> {
const fetchFn = io.fetchFn ?? directLocalHttpFetch;
try {
const res = await fetchFn(`http://${probeHostname(opts.hostname)}:${port}/readyz`, {
signal: AbortSignal.timeout(io.timeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS),
});
// Parse even on 503: /readyz returns JSON with a sanitized status while pending.
const body = (await res.json().catch(() => null)) as unknown;
const parsed = validateReadyzBody(body, port, opts);
if (!parsed) return null;
// HTTP/body-status consistency: ready requires 200; pending/failed require 503.
if (parsed.status === "ready" && res.status !== 200) return null;
if (parsed.status !== "ready" && res.status !== 503) return null;
return parsed;
} catch {
return null;
}
}