@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
464 lines (441 loc) • 22.1 kB
text/typescript
import { execFileSync } from "node:child_process";
import { isIP } from "node:net";
import { loadConfig } from "../config";
import { readRuntimePort } from "../config/process-state";
import { directLocalHttpFetch } from "../server/direct-local-http";
import { configuredAdminToken } from "./admin-secrets";
import { PROXY_STOP_EXIT_GRACE_MS } from "./shutdown-budget";
export function isProcessAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
export function waitForExit(pid: number, timeoutMs: number): boolean {
const deadline = Date.now() + timeoutMs;
const marker = new Int32Array(new SharedArrayBuffer(4));
while (Date.now() < deadline) {
if (!isProcessAlive(pid)) return true;
Atomics.wait(marker, 0, 0, 50);
}
return !isProcessAlive(pid);
}
/** Injectable seams so the graceful-stop flow is unit-testable without a live proxy. */
export interface GracefulStopIo {
fetchFn?: typeof fetch;
readRuntime?: (pid: number) => { port: number; hostname?: string } | null;
waitExit?: (pid: number, timeoutMs: number) => boolean;
env?: Record<string, string | undefined>;
exitTimeoutMs?: number;
/**
* Nonce of the pending-teardown receipt this caller claimed.
*
* `ocx stop` sets it because it restores shared client config itself, only after
* proving a stopped Task Scheduler did not respawn the proxy (#3008). The nonce is what
* makes the deferral an owned obligation rather than a flag anyone can set: the proxy
* honours it only when it names the receipt actually on disk. Direct callers omit it
* and keep the self-contained behaviour.
*/
deferSharedTeardownNonce?: string;
/**
* Whether a `sharedTeardown: "not-owned"` answer confirms this stop's teardown.
*
* A sibling's `not-owned` is a truthful final answer: it owns no shared teardown, so
* nothing is left for this caller either. `ocx stop` of a connected Link Child sets this
* false — only an old build answers `not-owned` for a link connection, and it says so
* while preserving the loopback route this caller must restore itself. The caller decides
* from the connection state read at the POST: a connect or disconnect committed mid-stop
* changes what the answer means for it. Defaults to true; the option only narrows a stop
* that sent no deferral nonce.
*/
acceptNotOwnedTeardown?: boolean;
/**
* Endpoint the caller already resolved for this pid.
*
* `ocx stop` records this same snapshot in its pending-teardown receipt. Re-reading the
* runtime file here could pick up a different one, which would make the receipt name an
* endpoint the stop never contacted — and recovery probes exactly that endpoint.
*/
runtimeEndpoint?: { hostname: string; port: number };
/**
* Send the stop only to `runtimeEndpoint`: without one, no request is sent and the runtime
* record is not read again. A caller whose deferral receipt names the endpoint it resolved sets
* this, so a record published in between cannot direct the request, and the receipt, elsewhere.
*/
runtimeEndpointOnly?: boolean;
/**
* Asked again immediately before the forced-stop fallback signals `pid`; false refuses the
* signal (killProxy throws). A caller that proved which process it may stop rechecks that here,
* so a pid reused after the graceful attempt is never signalled.
*/
mayTerminate?: () => boolean;
/**
* Windows checked stop (#7771): Part B confirms the port after this stop returns and
* owns the wait, so the 15 s waitForStoppedPort here is skipped (030 "Time budget").
*/
skipPortWait?: boolean;
/**
* Whether a graceful stop that did not confirm may fall back to the platform kill ladder.
*
* `ocx uninstall` sets this false when its Part A did not end stopped/absent: a manager
* that would not stop can respawn the proxy, and `taskkill /T /F` under it is exactly the
* forced stop the checked Windows stop exists to prevent. A 409 refusal never reaches the
* fallback either way. Defaults to true, which is every other caller's behaviour.
*/
allowForcedFallback?: boolean;
/**
* The forced ladder, injectable so the rule above can be proven without signalling a
* process. Production callers omit it and get {@link killProxy}.
*/
forceKill?: (pid: number) => void;
/**
* The post-stop port wait, injectable so a test can present a verified listener without
* starting one. Production callers omit it and get `reclaimListenPort`.
*/
reclaimPort?: (
port: number,
hostname: string,
opts: { timeoutMs?: number; intervalMs?: number; scanIntervalMs?: number; killOcxHolders?: boolean; onlyKillPids?: number[] },
) => Promise<boolean>;
}
/**
* Host to POST /api/stop against: follow the recorded bind hostname only when it is
* an IP literal (a proxy bound to ::1 or a LAN IP is unreachable on 127.0.0.1).
* DNS names are not safe here because resolution may change between bind and stop,
* while this request carries the management token. Loopback aliases, DNS names, and
* wildcard binds therefore all fall back to IPv4 loopback.
*/
export function gracefulStopHost(hostname: string | undefined): string {
const trimmed = (hostname ?? "").trim();
const lower = trimmed.toLowerCase();
if (!trimmed || lower === "localhost" || trimmed === "127.0.0.1" || trimmed === "0.0.0.0" || trimmed === "::" || trimmed === "[::]") {
return "127.0.0.1";
}
if (lower === "::1" || lower === "[::1]") return "[::1]";
const unbracketed = trimmed.startsWith("[") && trimmed.endsWith("]")
? trimmed.slice(1, -1)
: trimmed;
const ipVersion = isIP(unbracketed);
if (ipVersion === 6) return `[${unbracketed}]`;
if (ipVersion === 4) return unbracketed;
return "127.0.0.1";
}
/**
* `"refused"` forbids forced stop. `"teardown-unconfirmed"` means the process exited,
* but its assigned shared teardown was not confirmed; callers must not kill it again.
*/
export type GracefulStopResult = boolean | "refused" | "teardown-unconfirmed";
/**
* The server's own explanation for the most recent 409, captured so `stopProxy` can report
* the real reason. There is more than one: a scheduler wrapper under another home, or the
* proxy being the installed service itself (#4023). Module-scoped because
* `GracefulStopResult` is a public contract with several callers, and widening it to carry
* the text would change every one of them for a message only this file reports.
*/
let lastRefusalMessage: string | null = null;
/**
* The server's machine-readable reason for the most recent 409, captured alongside the
* message so a refusal that arrives without a body still names the right cause. Without it
* the fallback has to guess, and guessing "ownership" sent operators to re-check
* CODEX_HOME for a refusal the scheduler wrapper had issued (#4169).
*/
let lastRefusalCode: string | null = null;
/** The server's explanation for the most recent 409, or `null` when it sent none. */
export function lastStopRefusalMessage(): string | null {
return lastRefusalMessage;
}
/** The server's `code` for the most recent 409, or `null` when it sent none. */
export function lastStopRefusalCode(): string | null {
return lastRefusalCode;
}
/**
* Wording for a refusal whose body carried no message. Each branch mirrors a refusal the
* management API can return from `POST /api/stop`; the default stays cause-neutral because
* naming the wrong cause is worse than naming none — it costs the operator the time they
* spend acting on it.
*
* These name the cause only. The command belongs to {@link refusalNextStep}, because the
* only callers of `stopProxy` are `ocx stop` and the service manager's own cleanup, and a
* message that told either of them to run `ocx stop` would be the #4169 loop again.
*/
function refusalFallbackMessage(code: string | null): string {
switch (code) {
case "respawnable_service":
return "The running proxy refused to stop: a service manager that can respawn it owns "
+ "the process.";
case "self_unload_service":
return "The running proxy refused to stop: it is the installed service itself, so "
+ "stopping the manager from inside it would end the process before native Codex is "
+ "restored.";
case "service_state_unknown":
return "The running proxy refused to stop: the service manager state could not be read, "
+ "so it cannot tell whether a wrapper would respawn it.";
default:
return "The running proxy refused to stop and sent no reason.";
}
}
/**
* What is actually left to do when `ocx stop` is the command that received the refusal.
*
* Every refusal `POST /api/stop` produces is written for an API client, so it recommends
* `ocx stop` — which is the command already running when the CLI prints it. That is the
* loop #4169 reports: the endpoint points at `ocx stop`, `ocx stop` repeats the endpoint,
* and neither names the wrapper that is refusing. `ocx stop` has already asked the service
* manager to stop by the time this is reached, so the remaining question is always what the
* service manager is doing, and no branch may answer with the command that just failed.
*/
export function refusalNextStep(code: string | null): string {
switch (code) {
case "respawnable_service":
return "This stop already asked the service manager to stop, so running `ocx stop` "
+ "again is not the missing step. Run `ocx service status` to see whether a wrapper "
+ "is still installed and able to respawn the proxy.";
case "self_unload_service":
return "This stop already asked the service manager to stop, so running `ocx stop` "
+ "again is not the missing step. Run `ocx service status` to see whether the service "
+ "is still registered.";
case "service_state_unknown":
return "Run `ocx service status` to see the query error, repair the service manager "
+ "access, then retry.";
default:
return "Run `ocx service status` to inspect the service state.";
}
}
/**
* A proxy declined shutdown (HTTP 409). There is more than one reason it can say no — a
* scheduler wrapper under another home, or the proxy being the installed service itself
* (#4023) — so the server's own message is carried through rather than guessed at.
*
* The refusal's `code` travels on the error because the reporting caller has to act on the
* cause, not re-parse prose: the message is the server's, and it recommends a command the
* CLI has already run.
*/
export class ProxyOwnershipRefusedError extends Error {
readonly code: string | null;
constructor(message: string, code: string | null = null) {
super(message);
this.code = code;
}
}
/**
* Ask a running proxy to stop itself via the management API (`POST /api/stop`), which
* drains in-flight turns, restores native Codex, and cleans its pid/runtime files.
* This is the only way to get a GRACEFUL stop on Windows, where the POSIX
* SIGTERM-then-SIGKILL ladder does not exist and `taskkill /F` gives the proxy no
* chance to run its shutdown handlers. Returns false when the proxy can't be reached
* or doesn't exit in time — callers fall back to {@link killProxy}. Returns `"refused"`
* when the proxy declines the stop (HTTP 409), which callers must NOT force past.
* True requires the expected shared-teardown response and an observed exit. It does not
* attest the process exit code or completion of every drain/shutdown hook.
*/
export async function stopProxyGracefully(pid: number, io: GracefulStopIo = {}): Promise<GracefulStopResult> {
return (await stopProxyGracefullyDetailed(pid, io)).result;
}
/**
* The refusal a single stop attempt received, carried back to that attempt's caller.
*
* Module-scoped state cannot do this job: two overlapping stops race, and the first would
* report the second's cause. The exported accessors stay as observational state for callers
* that only want the last refusal, but the error text is built from this per-call value.
*/
type StopRefusal = { message: string | null; code: string | null };
export async function stopRequestFetch(input: string | URL | Request, init?: RequestInit): Promise<Response> {
let status: number | undefined;
try {
return await directLocalHttpFetch(input, init, { onResponseStatus: value => { status = value; } });
} catch (error) {
// A received refusal forbids forced stop even if its explanatory body fails.
if (status === 409) return new Response(null, { status });
throw error;
}
}
async function stopProxyGracefullyDetailed(
pid: number,
io: GracefulStopIo = {},
): Promise<{ result: GracefulStopResult; refusal: StopRefusal }> {
const refusal: StopRefusal = { message: null, code: null };
const done = (result: GracefulStopResult): { result: GracefulStopResult; refusal: StopRefusal } =>
({ result, refusal });
const readRuntime = io.readRuntime ?? readRuntimePort;
const runtime = io.runtimeEndpoint ?? (io.runtimeEndpointOnly ? null : readRuntime(pid));
if (!runtime?.port) return done(false);
const env = io.env ?? process.env;
const headers: Record<string, string> = {};
const token = configuredAdminToken(env.OPENCODEX_HOME?.trim() || undefined, env as NodeJS.ProcessEnv);
if (token) headers["x-opencodex-api-key"] = token;
const fetchFn = io.fetchFn ?? stopRequestFetch;
let sharedTeardownConfirmed = false;
try {
// `ocx stop` asks the proxy NOT to restore shared client config: it does that itself,
// after verifying a stopped Task Scheduler did not respawn the proxy (#3008). Letting
// the child do it means a survivor found seconds later has already lost its config.
const stopUrl = `http://${gracefulStopHost(runtime.hostname)}:${runtime.port}/api/stop`
+ (io.deferSharedTeardownNonce
? `?deferSharedTeardown=1&teardownNonce=${encodeURIComponent(io.deferSharedTeardownNonce)}`
: "");
const res = await fetchFn(stopUrl, {
method: "POST",
headers,
// Hung proxies with many CLOSE_WAIT clients can be slow to accept; give them
// longer than a health poll so we prefer drain over taskkill /F.
signal: AbortSignal.timeout(io.exitTimeoutMs ? Math.min(io.exitTimeoutMs, 10_000) : 10_000),
});
// 409 is the proxy REFUSING to stop. There is more than one reason it can say no — a
// respawning service manager, the proxy being the installed service itself, or an
// unreadable scheduler state — so both the message and the code are captured rather
// than assumed. That is a policy answer, not a dead endpoint — escalating to SIGTERM
// here would run the daemon's cleanup and strip shared config out from under the
// still-running service. Report the refusal instead of forcing.
if (res.status === 409) {
const parsed = await res.json()
.then(body => {
const record = body as { message?: unknown; code?: unknown } | null;
const message = record?.message;
const code = record?.code;
return {
message: typeof message === "string" && message.trim() ? message.trim() : null,
code: typeof code === "string" && code.trim() ? code.trim() : null,
};
})
.catch(() => ({ message: null, code: null }));
refusal.message = parsed.message;
refusal.code = parsed.code;
lastRefusalMessage = parsed.message;
lastRefusalCode = parsed.code;
return done("refused");
}
if (!res.ok) return done(false);
const body: unknown = await res.json().catch(() => null);
const expectedTeardown = io.deferSharedTeardownNonce ? "deferred" : "performed";
// A sibling instance answers `not-owned`: it has no shared teardown to perform, so there is
// nothing left over for this caller either. A deferral it was sent is still not confirmed.
// A connected Link Child's stop refuses the same answer: only an old build sends it for a
// link connection, and it preserves the route this caller must restore.
sharedTeardownConfirmed = body !== null
&& typeof body === "object"
&& !Array.isArray(body)
&& "success" in body && body.success === true
&& "sharedTeardown" in body && (body.sharedTeardown === expectedTeardown
|| (!io.deferSharedTeardownNonce
&& (io.acceptNotOwnedTeardown ?? true)
&& body.sharedTeardown === "not-owned"));
} catch {
return done(false);
}
const waitExit = io.waitExit ?? waitForExit;
// Honor the server's own drain window: /api/stop answers 200 first, then drains for
// config.shutdownTimeoutMs. Waiting less than that hard-kills mid-drain.
const exitTimeoutMs = io.exitTimeoutMs ?? drainDeadlineMs();
if (!waitExit(pid, exitTimeoutMs)) return done(false);
return done(sharedTeardownConfirmed ? true : "teardown-unconfirmed");
}
function drainDeadlineMs(): number {
try {
return Math.max(0, loadConfig().shutdownTimeoutMs ?? 5000) + PROXY_STOP_EXIT_GRACE_MS;
} catch {
return 5000 + PROXY_STOP_EXIT_GRACE_MS;
}
}
/** Graceful-first stop: management-API drain, then the platform kill ladder. */
export async function stopProxy(pid: number, io: GracefulStopIo = {}): Promise<boolean> {
if (!isProcessAlive(pid)) return false;
const runtime = io.skipPortWait ? null : io.runtimeEndpoint ?? (io.runtimeEndpointOnly ? null : readRuntimePort(pid));
const { result: graceful, refusal } = await stopProxyGracefullyDetailed(pid, io);
if (graceful === "refused") {
// The proxy refused on purpose. Forcing would strip shared config while whatever owns
// the process keeps it alive. The server's own message is preferred; the fallback is
// selected from its code so an empty body still names the right cause. Both come from
// THIS attempt, so an overlapping stop cannot lend it the wrong reason.
throw new ProxyOwnershipRefusedError(
refusal.message ?? refusalFallbackMessage(refusal.code),
refusal.code,
);
}
if (graceful === "teardown-unconfirmed") {
// Exit was observed, so do not enter the forced-stop fallback. Returning false keeps
// shared restoration with `ocx stop` instead of claiming that the proxy completed it.
await waitForStoppedPort(runtime, pid, io);
return false;
}
if (graceful) {
await waitForStoppedPort(runtime, pid, io);
return true;
}
if (io.allowForcedFallback === false) {
// The caller forbade the forced ladder (a manager that would not stop may respawn the
// proxy). Report the unconfirmed stop; the caller keeps its failure recorded.
await waitForStoppedPort(runtime, pid, io);
return false;
}
// Written as a literal killProxy(pid, ...) call so the refusal-before-kill ordering stays
// checkable in source (tests/providers/xai/grok-lifecycle.test.ts); the seam only swaps it.
if (io.forceKill) io.forceKill(pid);
else killProxy(pid, io.mayTerminate);
await waitForStoppedPort(runtime, pid, io);
return false;
}
/**
* After stop/kill, wait for the former listen port to become bindable (Windows drain).
*
* The wait is OBSERVATIONAL whenever the caller forbade the forced ladder
* (`allowForcedFallback === false`). Reclaiming with `killOcxHolders` would terminate a
* verified listener through `killProxy`, which is the forced stop that refusal exists to
* prevent: a manager that would not stop may respawn the proxy, so a survivor must be
* reported rather than killed. The wait still runs and still returns; it only stops being
* allowed to change anything.
*/
async function waitForStoppedPort(
runtime: { port: number; hostname?: string } | null | undefined,
stoppedPid?: number,
io: GracefulStopIo = {},
): Promise<void> {
if (!runtime?.port) return;
const mayKill = io.allowForcedFallback !== false;
try {
const reclaim = io.reclaimPort ?? (await import("../server/port-reclaim")).reclaimListenPort;
await reclaim(runtime.port, runtime.hostname ?? "127.0.0.1", {
timeoutMs: 15_000,
intervalMs: 100,
scanIntervalMs: 500,
// Only the process we just stopped — never kill a newly started twin proxy.
killOcxHolders: mayKill && !!(stoppedPid && stoppedPid > 0),
onlyKillPids: mayKill && stoppedPid && stoppedPid > 0 ? [stoppedPid] : [],
...(io.forceKill ? { killFn: io.forceKill } : io.mayTerminate ? { killFn: (pid: number) => killProxy(pid, io.mayTerminate) } : {}),
});
} catch {
/* best-effort — callers that need a hard guarantee reclaim again before bind */
}
}
/** taskkill in the forced fallback; it had no timeout before #7771. */
export const FORCED_TASKKILL_TIMEOUT_MS = 5_000;
export function killProxy(pid: number, mayTerminate?: () => boolean): void {
if (!isProcessAlive(pid)) return;
const authorize = () => {
if (mayTerminate && mayTerminate() !== true) throw Object.assign(new Error("Process termination refused"), { code: "EUPDATEAUTHORITY" });
};
if (process.platform === "win32") {
// Windows process.kill(SIGTERM/SIGINT) is TerminateProcess — not a graceful signal.
// Graceful drain happens only via stopProxyGracefully() (POST /api/stop). This path
// is the hard fallback: taskkill /T /F so the process tree exits (ghost LISTEN /
// CLOSE_WAIT are then cleared by reclaimListenPort / SetTcpEntry).
const taskkill = `${process.env.SystemRoot ?? "C:\\Windows"}\\System32\\taskkill.exe`;
const args = ["/PID", String(pid), "/T", "/F"];
authorize();
try {
// Capped (#7771): the checked stop's budget counts the forced kill at 5 s.
execFileSync(taskkill, args, { stdio: "pipe", windowsHide: true, timeout: FORCED_TASKKILL_TIMEOUT_MS });
} catch (err) {
if (isProcessAlive(pid)) throw err;
}
} else {
authorize();
process.kill(pid, "SIGTERM");
if (!waitForExit(pid, 5000)) {
authorize();
process.kill(pid, "SIGKILL");
}
}
if (!waitForExit(pid, 5000)) throw new Error(`process ${pid} did not exit`);
}