@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
420 lines (391 loc) • 19.2 kB
text/typescript
import { readFileSync } from "node:fs";
import { loadConfig } from "../config";
import { readRuntimePort } from "../config/process-state";
import { readClientConnectionState } from "../client/state";
import { appendDebugLogLine } from "../lib/debug-log-buffer";
import { redactUserPath } from "../lib/redact";
import { isRecyclingForExit, isShutdownDraining } from "../server/lifecycle";
import { probeEndpointLiveness, type EndpointLiveness } from "../server/proxy-liveness";
import type { OcxConfig } from "../types";
import { shouldSyncCodexOnStart } from "./desired-state";
import { injectCodexConfig, type CodexInjectResult, type InjectCodexOptions } from "./inject";
import { standaloneCodexRoutingTarget } from "./inject/routing-target";
import { journaledInjectedOpenaiBaseUrl, journaledInjectedRealtimeWsBaseUrl, journalOwner } from "./journal";
import { effectiveLoopbackListenerPort, shouldInjectApiAuthHeader } from "./loopback-target";
import { CODEX_CONFIG_PATH } from "./paths";
import {
detectCodexRoutingDrift,
type CodexRoutingDrift,
type CodexRoutingDriftTarget,
type JournaledCodexRouting,
} from "./routing-drift";
import { siblingOfLivePort } from "./sibling-start";
/**
* The live owner re-points Codex routing that a dead instance left on another loopback port.
*
* Startup sync is the last time an owner looks at `config.toml`. When something else re-points the
* opencodex-owned routing at its own port and then dies, every new Codex thread fails with
* "Connection refused" while this proxy keeps serving its own port. This loop closes that gap.
*
* Cost: one read of `config.toml` every 10 s from an unref'd timer, and nothing else while the
* bytes are unchanged. Probes, gates and the journal run only while owned routing names a port this
* process does not serve. Nothing here touches the request path.
*
* It writes only when every foreign target has been dead on at least two probes spanning 20 s, a
* final probe still says dead, and every gate is open. The write is the ordinary injector with no
* catalog path (so no catalog gather and the current `model_catalog_json` stays), a short lock
* timeout, and a synchronous guard that re-reads `config.toml` under the lock and aborts when the
* routing changed since the probe. A refusal that raced another write of `config.toml` is the same
* abort, not a refusal. A live foreign opencodex is never fought; an unknown answer never advances
* the streak. The caps pause the loop for at most an hour and never stop it. Detection, gates and
* probes read the journal read-only, so watching never deletes an unreadable journal (a heal write
* goes through the injector's ordinary journal handling). Started only from `handleStart`'s owner
* path and stopped first in its exit cleanup.
*/
export const ROUTING_HEAL_TICK_MS = 10_000;
export const ROUTING_HEAL_MIN_DEAD_PROBES = 2;
export const ROUTING_HEAL_MIN_DEAD_SPAN_MS = 20_000;
export const ROUTING_HEAL_LOCK_TIMEOUT_MS = 1_000;
/** While the bytes stay equal, a live foreign owner, an unknown answer or a closed gate is looked at again only this often. */
export const ROUTING_HEAL_RECHECK_MS = 30_000;
export const ROUTING_HEAL_REFUSED_BACKOFF_MS = 10 * 60_000;
export const ROUTING_HEAL_WINDOW_MS = 60 * 60_000;
/** Rate cap: write attempts (busy ones included) per rolling window; more pause until the oldest leaves it. */
export const ROUTING_HEAL_MAX_ATTEMPTS = 6;
/**
* Flap cap: this many successful heals in one window means another writer keeps re-pointing it, so a
* further heal waits until the oldest leaves the window, then needs a fresh dead streak.
*/
export const ROUTING_HEAL_MAX_HEALS = 3;
export type CodexRoutingHealGate =
| "sibling"
| "exiting"
| "runtime-record"
| "codex-off"
| "admission-token"
| "target-not-served"
| "client";
export interface CodexRoutingHealGates {
siblingOfLivePort(): number | null;
/** True while the process is recycling for exit or draining for shutdown. */
exiting(): boolean;
/** The port in this home's runtime record when that record names this process. */
runtimePortOfThisProcess(): number | null;
loadConfig(): OcxConfig;
/** Any client-connection state other than disconnected (connected, link, invalid). */
clientConnected(): boolean;
/** The journal names a connected client as its owner (read-only). */
clientJournalOwner(): boolean;
}
export type CodexRoutingHealGateResult =
| { readonly open: true; readonly config: OcxConfig; readonly targetPort: number }
| { readonly open: false; readonly gate: CodexRoutingHealGate };
/** Cheapest first; `loadConfig` and the journal only when the in-memory gates are open. */
export function evaluateCodexRoutingHealGates(
port: number,
ownPorts: ReadonlySet<number>,
gates: CodexRoutingHealGates,
): CodexRoutingHealGateResult {
if (gates.siblingOfLivePort() !== null) return { open: false, gate: "sibling" };
if (gates.exiting()) return { open: false, gate: "exiting" };
if (gates.runtimePortOfThisProcess() !== port) return { open: false, gate: "runtime-record" };
const config = gates.loadConfig();
// Codex ON and not hub-gated; the injector re-checks this under its lock.
if (!shouldSyncCodexOnStart(config)) return { open: false, gate: "codex-off" };
if (shouldInjectApiAuthHeader(config)) return { open: false, gate: "admission-token" };
const targetPort = Number(new URL(standaloneCodexRoutingTarget(port, config).baseUrl).port);
if (!ownPorts.has(targetPort)) return { open: false, gate: "target-not-served" };
if (gates.clientConnected() || gates.clientJournalOwner()) return { open: false, gate: "client" };
return { open: true, config, targetPort };
}
export interface CodexRoutingHealRecord {
readonly at: string;
readonly fromUrls: readonly string[];
readonly toPort: number;
readonly probes: number;
readonly deadForMs: number;
}
export interface CodexRoutingHealerHandle {
stop(): void;
lastHeal(): CodexRoutingHealRecord | null;
}
export interface CodexRoutingHealerDeps {
/** Schedule `fn` after `ms`; defaults to an unref'd setTimeout. */
scheduleFn?: (fn: () => void, ms: number) => { cancel(): void };
/** Monotonic milliseconds. */
now?: () => number;
/** `config.toml` text, or null when absent or unreadable. */
readConfig?: () => string | null;
readJournaled?: () => JournaledCodexRouting;
probe?: (target: { port: number; hostname: string }) => Promise<EndpointLiveness>;
inject?: (port: number, config: OcxConfig, options: InjectCodexOptions) => Promise<CodexInjectResult>;
gates?: Partial<CodexRoutingHealGates>;
log?: Pick<Console, "warn">;
debugLine?: (line: string) => void;
}
/** Thrown by the under-lock guard: the routing changed after it was proven dead. */
class RoutingHealAborted extends Error {
constructor() {
super("Codex routing changed before the heal could be written.");
this.name = "RoutingHealAborted";
}
}
function defaultSchedule(fn: () => void, ms: number): { cancel(): void } {
const timer = setTimeout(fn, ms);
if (typeof timer.unref === "function") timer.unref();
return { cancel: () => clearTimeout(timer) };
}
function readCodexConfigText(): string | null {
try {
return readFileSync(CODEX_CONFIG_PATH, "utf8");
} catch {
return null;
}
}
const defaultGates: CodexRoutingHealGates = {
siblingOfLivePort,
exiting: () => isRecyclingForExit() || isShutdownDraining(),
runtimePortOfThisProcess: () => readRuntimePort(process.pid)?.port ?? null,
loadConfig,
clientConnected: () => readClientConnectionState().kind !== "disconnected",
clientJournalOwner: () => journalOwner({ readOnly: true })?.kind === "client",
};
/** The first line of an injector or error message, with home paths and secret-shaped values masked. */
function logDetail(message: string): string {
return redactUserPath(message.split("\n")[0] ?? "");
}
function endpointLabel(target: CodexRoutingDriftTarget): string {
return `${target.hostname.includes(":") ? `[${target.hostname}]` : target.hostname}:${target.port}`;
}
function endpointKey(target: Pick<CodexRoutingDriftTarget, "hostname" | "port">): string {
return JSON.stringify([target.hostname, target.port]);
}
export function startCodexRoutingHealer(options: {
port: number;
config: OcxConfig;
deps?: CodexRoutingHealerDeps;
}): CodexRoutingHealerHandle {
const { port, config } = options;
const deps = options.deps ?? {};
let stopped = false;
let pending: { cancel(): void } | undefined;
let last: CodexRoutingHealRecord | null = null;
const handle: CodexRoutingHealerHandle = {
stop() {
stopped = true;
pending?.cancel();
pending = undefined;
},
lastHeal: () => last,
};
// Admission-token (non-loopback) routing is never healed, and the bind scope is fixed for the
// process lifetime, so such an owner starts no timer at all.
if (shouldInjectApiAuthHeader(config)) return handle;
const listenerPort = effectiveLoopbackListenerPort(config, port);
const ownPorts: ReadonlySet<number> = new Set(listenerPort === null ? [port] : [port, listenerPort]);
const scheduleFn = deps.scheduleFn ?? defaultSchedule;
const clock = deps.now ?? (() => performance.now());
const readConfig = deps.readConfig ?? readCodexConfigText;
const readJournaled = deps.readJournaled ?? ((): JournaledCodexRouting => ({
openaiBaseUrl: journaledInjectedOpenaiBaseUrl({ readOnly: true }),
realtimeWsBaseUrl: journaledInjectedRealtimeWsBaseUrl({ readOnly: true }),
}));
const probe = deps.probe ?? (target => probeEndpointLiveness(target));
const inject = deps.inject ?? injectCodexConfig;
const gates: CodexRoutingHealGates = { ...defaultGates, ...deps.gates };
const log = deps.log ?? console;
const debugLine = deps.debugLine ?? appendDebugLogLine;
// The verdict for the last bytes read. Unchanged bytes with a non-foreign verdict cost nothing.
let cache: { content: string; kind: CodexRoutingDrift["kind"] } | null = null;
let streak: { key: string; firstDeadAt: number; probes: number } | null = null;
let recheckAt = 0;
/** After a real refusal; cleared as soon as routing is not foreign any more. */
let backoffUntil = 0;
/** While a cap is reached: until the oldest counted attempt or heal leaves the window. */
let pausedUntil = 0;
const announcedLive = new Set<string>();
const attempts: number[] = [];
const heals: number[] = [];
let rateCapAnnounced = false;
let flapCapAnnounced = false;
const warn = (line: string) => {
try { log.warn(line); } catch { /* logging never breaks the loop */ }
try { debugLine(line); } catch { /* the debug buffer is best-effort */ }
};
const detect = (content: string): CodexRoutingDrift =>
detectCodexRoutingDrift(content, { ownPorts, journaled: readJournaled });
/** "live" wins over "unknown", which wins over "dead": all must be dead to proceed. */
const probeAll = async (targets: readonly CodexRoutingDriftTarget[]): Promise<{ verdict: EndpointLiveness; liveEndpoints: CodexRoutingDriftTarget[] }> => {
const byEndpoint = new Map<string, CodexRoutingDriftTarget>();
for (const target of targets) byEndpoint.set(endpointKey(target), target);
const liveEndpoints: CodexRoutingDriftTarget[] = [];
let unknown = false;
for (const target of byEndpoint.values()) {
let result: EndpointLiveness;
try { result = await probe({ port: target.port, hostname: target.hostname }); }
catch { result = "unknown"; }
if (result === "live") liveEndpoints.push(target);
else if (result === "unknown") unknown = true;
}
return { verdict: liveEndpoints.length > 0 ? "live" : unknown ? "unknown" : "dead", liveEndpoints };
};
const standDownForLive = (liveEndpoints: readonly CodexRoutingDriftTarget[]) => {
streak = null;
recheckAt = clock() + ROUTING_HEAL_RECHECK_MS;
for (const target of liveEndpoints) {
const key = endpointKey(target);
if (announcedLive.has(key)) continue;
announcedLive.add(key);
warn(`Codex routing points at another running opencodex on ${endpointLabel(target)}; leaving it.`);
}
};
/** A cap pauses probing and writing until `until`; the streak starts over once it has passed. */
const pauseUntil = (until: number) => {
streak = null;
pausedUntil = until;
};
const heal = async (
content: string,
targets: readonly CodexRoutingDriftTarget[],
proven: { firstDeadAt: number; probes: number },
) => {
const now = clock();
while (attempts.length > 0 && now - attempts[0]! >= ROUTING_HEAL_WINDOW_MS) attempts.shift();
while (heals.length > 0 && now - heals[0]! >= ROUTING_HEAL_WINDOW_MS) heals.shift();
if (heals.length >= ROUTING_HEAL_MAX_HEALS) {
pauseUntil(heals[0]! + ROUTING_HEAL_WINDOW_MS);
if (!flapCapAnnounced) warn(`Codex routing was re-pointed at a dead port ${heals.length} times within an hour; the self-heal pauses until that hour has passed so it does not fight another writer. Run 'ocx sync' to repair routing now.`);
flapCapAnnounced = true;
return;
}
flapCapAnnounced = false;
if (attempts.length >= ROUTING_HEAL_MAX_ATTEMPTS) {
pauseUntil(attempts[0]! + ROUTING_HEAL_WINDOW_MS);
if (!rateCapAnnounced) warn("Codex routing self-heal paused after repeated attempts; it resumes within the hour. Run 'ocx sync' to repair now.");
rateCapAnnounced = true;
return;
}
rateCapAnnounced = false;
// One final look right before the write, then the gates again: the probe may have taken seconds.
const final = await probeAll(targets);
if (stopped) return; // the exit cleanup ran while the probe was out
if (final.verdict === "live") { standDownForLive(final.liveEndpoints); return; }
if (final.verdict !== "dead") { recheckAt = clock() + ROUTING_HEAL_RECHECK_MS; return; }
const gate = evaluateCodexRoutingHealGates(port, ownPorts, gates);
if (!gate.open) {
streak = null;
if (gate.gate === "sibling") stopped = true;
return;
}
attempts.push(clock());
const deadEndpoints = new Set(targets.map(endpointKey));
const guard = () => {
const current = readConfig();
const drift = current === null ? null : detect(current);
if (current !== content || drift?.kind !== "foreign"
|| drift.targets.some(target => !deadEndpoints.has(endpointKey(target)))) {
throw new RoutingHealAborted();
}
};
/*
* Whether a failed write raced another writer. A coordinated home re-reads its admission under
* the lock BEFORE this guard runs, so a config.toml rewritten between the plan and the lock
* comes back as a plain refusal. Moved bytes are the same abort as the guard's; only a refusal
* over the very bytes that were proven dead backs off.
*/
const raced = (): boolean => {
const current = readConfig();
if (current !== content) return true;
const drift = detect(current);
return drift.kind !== "foreign" || drift.targets.some(target => !deadEndpoints.has(endpointKey(target)));
};
let result: CodexInjectResult;
try {
result = await inject(port, gate.config, { lockTimeoutMs: ROUTING_HEAL_LOCK_TIMEOUT_MS, beforeClientWrite: guard });
} catch (error) {
streak = null;
cache = null;
if (error instanceof RoutingHealAborted || raced()) return;
backoffUntil = clock() + ROUTING_HEAL_REFUSED_BACKOFF_MS;
warn(`Codex routing self-heal failed (${logDetail(error instanceof Error ? error.message : String(error))}); retrying in 10 minutes. Run 'ocx sync' to repair now.`);
return;
}
if (result.success && result.status !== "skipped" && result.configApplied !== false) {
heals.push(clock());
streak = null;
cache = null;
const fromUrls = [...new Set(targets.map(target => target.url))];
last = {
at: new Date().toISOString(),
fromUrls,
toPort: gate.targetPort,
probes: proven.probes + 1,
deadForMs: Math.round(clock() - proven.firstDeadAt),
};
const from = [...new Set(targets.map(endpointLabel))].join(", ");
warn(`Codex routing pointed at ${from}, where no opencodex answers; re-pointed it at this proxy on port ${gate.targetPort}. Codex threads opened meanwhile keep the old address until you reopen them.`);
return;
}
// Busy lock: keep the proven streak so the next tick re-probes and tries again.
if (result.retryable) return;
streak = null;
cache = null;
// Skipped under the lock (Codex OFF, hub, sibling), an external provider appeared, or a race.
if (result.success || raced()) return;
backoffUntil = clock() + ROUTING_HEAL_REFUSED_BACKOFF_MS;
warn(`Codex routing self-heal was refused (${logDetail(result.message)}); retrying in 10 minutes. Run 'ocx sync' to repair now.`);
};
const evaluate = async () => {
if (gates.siblingOfLivePort() !== null) { stopped = true; return; }
const content = readConfig();
if (content === null) { cache = null; streak = null; return; }
const unchanged = cache !== null && cache.content === content;
if (unchanged && cache!.kind !== "foreign") return;
const drift = detect(content);
cache = { content, kind: drift.kind };
if (drift.kind !== "foreign") {
streak = null;
recheckAt = 0;
backoffUntil = 0;
announcedLive.clear();
return;
}
const now = clock();
if (now < backoffUntil || now < pausedUntil) return;
if (unchanged && now < recheckAt) return;
const gate = evaluateCodexRoutingHealGates(port, ownPorts, gates);
if (!gate.open) {
streak = null;
recheckAt = now + ROUTING_HEAL_RECHECK_MS;
if (gate.gate === "sibling") stopped = true;
return;
}
const probed = await probeAll(drift.targets);
if (stopped) return;
if (probed.verdict === "live") { standDownForLive(probed.liveEndpoints); return; }
// Unknown never advances the streak and never resets it; it is asked again at the recheck pace.
if (probed.verdict !== "dead") { recheckAt = clock() + ROUTING_HEAL_RECHECK_MS; return; }
recheckAt = 0;
const probedAt = clock();
const key = JSON.stringify([...new Set(drift.targets.map(endpointKey))].sort());
if (!streak || streak.key !== key) {
streak = { key, firstDeadAt: probedAt, probes: 1 };
return;
}
streak.probes += 1;
if (streak.probes < ROUTING_HEAL_MIN_DEAD_PROBES || probedAt - streak.firstDeadAt < ROUTING_HEAL_MIN_DEAD_SPAN_MS) return;
await heal(content, drift.targets, streak);
};
const tick = async () => {
pending = undefined;
if (stopped) return;
try { await evaluate(); } catch {
// A background check never throws into the timer; an unreadable config.json is looked at
// again at the slower recheck pace rather than on every tick.
recheckAt = clock() + ROUTING_HEAL_RECHECK_MS;
}
if (!stopped) pending = scheduleFn(() => { void tick(); }, ROUTING_HEAL_TICK_MS);
};
pending = scheduleFn(() => { void tick(); }, ROUTING_HEAL_TICK_MS);
return handle;
}