UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

1,083 lines (1,017 loc) • 47.5 kB
import { randomUUID } from "node:crypto"; import { chmodSync, mkdirSync, readFileSync, statSync, unlinkSync } from "node:fs"; import { execFile } from "node:child_process"; import { readFile as readFileAsync } from "node:fs/promises"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { atomicWriteFile, isMissingPathError } from "../config/atomic-write"; import { getConfigPath } from "../config/paths"; import { readBoundedResponseBytes } from "../lib/bounded-body"; import { linkDir, linkKnownHostsPath } from "../link/paths"; import { buildTunnelArgv } from "../link/ssh-argv"; import { defaultSshCaptureOpener, openSshCapture, retainSshCapture, type SshCaptureOpener } from "../link/ssh-capture"; import { createSshRunner, type SshChild, type SshRunner } from "../link/ssh-runner"; import { CLIENT_TUNNEL_RETRY_POLICY, classifySshStderr, dueForSpawn, failedSpawnTunnel, IDLE, reduceTunnel, type StderrClass, type TunnelState, } from "../link/tunnel-state"; import { isLinkPort } from "../link/ports"; import { inspectIpv4LoopbackListener, type LoopbackOwnerLookupResult } from "../server/port-reclaim"; import type { LinkDiagnosticSink } from "../link/diagnostics"; import { createTunnelHistoryBook, exitCause, type TunnelCause } from "../link/tunnel-history"; import { isLinkConnection, readClientConnectionState } from "./state"; import { clientLinkStatePath, readClientLinkState, type ClientLinkState } from "./link-state"; import type { LinkTunnelGate, LinkAdmissionResult } from "./link-relay"; import { nextLinkConnectionGeneration } from "../link/relay-auth"; /** * The client-owned `ssh -N -L 127.0.0.1:<tunnelPort>:127.0.0.1:<peerListenerPort> <alias>` * of a client-initiated link. Two owners use it: the dashboard join (a short-lived tunnel that * lives only until the in-process connect finishes) and the client runtime supervisor. */ export interface ClientLinkTunnelSpec { linkId: string; alias: string; tunnelPort: number; peerListenerPort: number; } export interface ClientLinkTunnelHandle { readonly pid: number; readonly startTime: string | null; readonly exited: Promise<number>; /** TERM, wait up to 5 s, then KILL; removes the pidfile this handle wrote. Idempotent. */ stop(): Promise<void>; } export interface ClientLinkTunnelDeps { runner?: SshRunner; configDir?: string; knownHostsFile?: string; /** Opt-in SSH capture (src/link/ssh-capture.ts); defaults to the one for this configuration home. */ sshCapture?: SshCaptureOpener; setTimer?: (callback: () => void, ms: number) => ReturnType<typeof setTimeout>; clearTimer?: (timer: ReturnType<typeof setTimeout>) => void; } export type OrphanTunnelResult = | { tunnel: "reaped" } | { tunnel: "absent" } | { tunnel: "owned"; pid?: number } | { tunnel: "unresolved"; pid: number }; /** A macOS process as `ps -o ppid= -o args=` shows it: the parent pid and the space-joined argv. */ export interface DarwinProcessInfo { ppid: number; startTime: string; args: string; } export interface OrphanReapDeps { configDir?: string; platform?: NodeJS.Platform; readProcessArgv?: (pid: number) => readonly string[] | null; /** Stable process start identity; unavailable identity means the orphan is never signalled. */ readProcessStartTime?: (pid: number, platform: NodeJS.Platform) => string | null; /** macOS: the parent pid and argv of `pid`, or null when they cannot be read. */ readProcessInfo?: (pid: number) => DarwinProcessInfo | null; isAlive?: (pid: number) => boolean; signal?: (pid: number, signal: NodeJS.Signals) => void; sleep?: (ms: number) => Promise<void>; } /** * What the periodic keyed probe last saw, for display only: it never changes a connected tunnel. * `home_not_ready` is a Home that admitted the key but reports its own startup readiness as pending * or failed; its link still works. */ export type ClientTunnelProbeReason = "unauthorized" | "home_unreachable" | "home_not_ready"; export type ClientLinkSupervisorStatus = | { kind: "stopped" } | { kind: "tunnel"; linkId: string; state: TunnelState; pid: number | null; probe?: ClientTunnelProbeReason } | { kind: "failed"; reason: "sidecar_invalid" }; export interface ClientLinkTunnelStatusProjection { alias: string; state: "failed"; since: string; reason: "sidecar_invalid"; } /** The supervisor is also the relay's tunnel gate: requests wait on it while it reconnects. */ export interface ClientLinkSupervisor extends LinkTunnelGate { start(): void; /** Stops the tunnel (TERM, up to 5 s, KILL). The runtime calls this before stopping its listener. */ stop(): Promise<void>; status(): ClientLinkSupervisorStatus; /** Changes on every transition into connected, initial or recovery; undefined while not connected. */ connectionGeneration(): string | undefined; } /** Read-only status bridge for a client whose persisted sidecar cannot be trusted. */ export function clientLinkTunnelStatus( path: string = clientLinkStatePath(), now: () => number = Date.now, ): ClientLinkTunnelStatusProjection | null { try { readClientLinkState(path); return null; } catch { return { alias: "unknown", state: "failed", since: new Date(now()).toISOString(), reason: "sidecar_invalid" }; } } /** `connectedLinkId` when the connection state could not be read (a write in flight, a bad file). */ export const CONNECTION_UNREADABLE = "unreadable"; export interface ClientLinkSupervisorDeps extends ClientLinkTunnelDeps, OrphanReapDeps { readSidecar?: () => ClientLinkState | null; /** * Current link id of a connected link-transport client, null when that no longer holds, or * `CONNECTION_UNREADABLE` when the connection state could not be read. */ connectedLinkId?: () => string | null | typeof CONNECTION_UNREADABLE; /** Called once after the tunnel stopped because the link ended (the runtime recycles here). */ onLinkEnded?: () => void; /** Test seam: called once the start-up check (leftover tunnel, first spawn) has finished. */ onInitialized?: () => void; /** The link key for the keyed readiness probe. The runtime passes its cached key source. */ linkKey?: () => string | null; /** Async local IPv4 LISTEN ownership; an unknown or foreign owner never receives the link key. */ inspectLoopbackListener?: (port: number, pid: number) => Promise<LoopbackOwnerLookupResult>; diagnostic?: LinkDiagnosticSink; fetchImpl?: typeof fetch; now?: () => number; random?: () => number; warn?: (message: string) => void; } export function clientTunnelPidfilePath(configDir?: string): string { return join(linkDir(configDir), "client-tunnel.pid"); } /** * Pidfile body at `clientTunnelPidfilePath()`: `{ version: 1, linkId, pid, argv, ownerPid }`. * `ownerPid` is the process that spawned the tunnel. A tunnel is an orphan only while its owner * is gone; a live owner means the tunnel is managed and `reapOrphanTunnel` reports "owned". */ export interface ClientTunnelPidfile { version: 1; linkId: string; pid: number; argv: string[]; ownerPid: number; /** Captured at spawn; legacy pidfiles without it cannot authorize a signal. */ startTime: string | null; } const STOP_TIMEOUT_MS = 5_000; /** A Child supervises one tunnel; its history needs one fixed key, never the link id. */ const CHILD_TUNNEL_HISTORY_KEY = "child"; const REAP_POLL_MS = 100; const TIMER_MS = 1_000; /** Consecutive unreadable reads before the supervisor acts on them. */ export const INVALID_READ_TICKS = 3; /** How often a connected tunnel is probed. The result is display-only. */ export const CONNECTED_PROBE_MS = 30_000; const PROBE_TIMEOUT_MS = 5_000; const PROBE_BACKOFF_MAX_MS = 5_000; /** A Home `/readyz` body is a few hundred bytes; a larger one is not the Home's. */ const PROBE_BODY_MAX_BYTES = 4_096; /** Requests that may wait on a reconnecting tunnel at once; more are answered 503 at once. */ export const CLIENT_LINK_MAX_HOLDS = 64; const PROCESS_IDENTITY_TIMEOUT_MS = 2_000; const execFileAsync = promisify(execFile); function sameArgv(left: readonly string[], right: readonly string[]): boolean { return left.length === right.length && left.every((value, index) => value === right[index]); } function parsePidfile(value: unknown): ClientTunnelPidfile | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; const raw = value as Record<string, unknown>; if (raw.version !== 1 || typeof raw.linkId !== "string" || typeof raw.pid !== "number" || !Number.isSafeInteger(raw.pid) || raw.pid < 1 || !Array.isArray(raw.argv) || raw.argv.length === 0 || raw.argv.some(item => typeof item !== "string") || typeof raw.ownerPid !== "number" || !Number.isSafeInteger(raw.ownerPid) || raw.ownerPid < 1 || (raw.startTime !== undefined && raw.startTime !== null && (typeof raw.startTime !== "string" || raw.startTime.length === 0 || raw.startTime.length > 128))) return null; return { version: 1, linkId: raw.linkId, pid: raw.pid, argv: raw.argv as string[], ownerPid: raw.ownerPid, startTime: typeof raw.startTime === "string" ? raw.startTime : null, }; } function readPidfile(path: string): ClientTunnelPidfile | null { try { return parsePidfile(JSON.parse(readFileSync(path, "utf8")) as unknown); } catch (error) { if (isMissingPathError(error)) return null; return null; } } function writePidfile(path: string, value: ClientTunnelPidfile): void { const dir = dirname(path); mkdirSync(dir, { recursive: true, mode: 0o700 }); atomicWriteFile(path, `${JSON.stringify(value)}\n`); if (process.platform !== "win32") chmodSync(path, 0o600); } function removePidfileIfPid(path: string, pid: number, startTime: string | null): void { const current = readPidfile(path); if (current?.pid !== pid || current.startTime !== startTime) return; try { unlinkSync(path); } catch (error) { if (!isMissingPathError(error)) throw error; } } function defaultSignal(pid: number, signal: NodeJS.Signals): void { try { process.kill(pid, signal); } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code !== "ESRCH") throw error; } } function defaultIsAlive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch (error) { return (error as NodeJS.ErrnoException).code === "EPERM"; } } function linuxProcessArgv(pid: number): readonly string[] | null { try { return parseLinuxArgv(readFileSync(`/proc/${pid}/cmdline`).toString()); } catch (error) { if (isMissingPathError(error)) return null; return null; } } function parseLinuxArgv(content: string): readonly string[] | null { const values = content.split("\0"); if (values.at(-1) === "") values.pop(); return values.length > 0 ? values : null; } function parseLinuxStartTime(stat: string): string | null { const afterCommand = stat.lastIndexOf(") "); if (afterCommand < 0) return null; const startTime = stat.slice(afterCommand + 2).trim().split(/\s+/)[19]; return startTime && /^\d+$/.test(startTime) ? startTime : null; } /** Field 22 of /proc/<pid>/stat: kernel start ticks survive argv and parent changes. */ function linuxProcessStartTime(pid: number): string | null { try { return parseLinuxStartTime(readFileSync(`/proc/${pid}/stat`, "utf8")); } catch { return null; } } function parseDarwinProcessInfo(output: string): DarwinProcessInfo | null { const match = /^\s*(\d+)\s+(\S+\s+\S+\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.+)$/.exec(output.replace(/\r?\n$/, "")); return match ? { ppid: Number(match[1]), startTime: match[2]!, args: match[3]! } : null; } /** One `ps` snapshot carries parent, second-resolution start time and the full argv. */ function darwinProcessInfo(pid: number): DarwinProcessInfo | null { try { const result = Bun.spawnSync(["/bin/ps", "-ww", "-o", "ppid=", "-o", "lstart=", "-o", "args=", "-p", String(pid)], { stdin: "ignore", stdout: "pipe", stderr: "ignore", }); if (result.exitCode !== 0) return null; return parseDarwinProcessInfo(result.stdout.toString()); } catch { return null; } } type AdoptedIdentityVerdict = "match" | "mismatch" | "unknown"; /** An unreadable identity denies this admission; only readable disagreement releases adoption. */ async function inspectAdoptedIdentity(record: ClientTunnelPidfile, deps: OrphanReapDeps): Promise<AdoptedIdentityVerdict> { if (!record.startTime) return "unknown"; const platform = deps.platform ?? process.platform; const check = async (): Promise<AdoptedIdentityVerdict> => { if (platform === "linux") { const [argv, startTime] = await Promise.all([ deps.readProcessArgv ? Promise.resolve().then(() => deps.readProcessArgv!(record.pid)) : readFileAsync(`/proc/${record.pid}/cmdline`, "utf8").then(parseLinuxArgv), deps.readProcessStartTime ? Promise.resolve().then(() => deps.readProcessStartTime!(record.pid, platform)) : readFileAsync(`/proc/${record.pid}/stat`, "utf8").then(parseLinuxStartTime), ]); if (argv === null || startTime === null) return "unknown"; return sameArgv(argv, record.argv) && startTime === record.startTime ? "match" : "mismatch"; } if (platform === "darwin") { const info = deps.readProcessInfo ? await Promise.resolve().then(() => deps.readProcessInfo!(record.pid)) : await execFileAsync("/bin/ps", ["-ww", "-o", "ppid=", "-o", "lstart=", "-o", "args=", "-p", String(record.pid)], { encoding: "utf8", timeout: PROCESS_IDENTITY_TIMEOUT_MS, maxBuffer: 64 * 1024, windowsHide: true, }).then(result => parseDarwinProcessInfo(result.stdout)); if (info === null) return "unknown"; return info.args === record.argv.join(" ") && info.startTime === record.startTime ? "match" : "mismatch"; } return "unknown"; }; let timer: ReturnType<typeof setTimeout> | undefined; const deadline = new Promise<"unknown">(resolve => { timer = setTimeout(() => resolve("unknown"), PROCESS_IDENTITY_TIMEOUT_MS); timer.unref?.(); }); try { return await Promise.race([check().catch(() => "unknown" as const), deadline]); } finally { if (timer) clearTimeout(timer); } } function processStartTime(pid: number, platform: NodeJS.Platform, deps: OrphanReapDeps): string | null { if (deps.readProcessStartTime) return deps.readProcessStartTime(pid, platform); if (platform === "linux") return linuxProcessStartTime(pid); if (platform === "darwin") return (deps.readProcessInfo ?? darwinProcessInfo)(pid)?.startTime ?? null; return null; } function timerDeps(deps: ClientLinkTunnelDeps): Required<Pick<ClientLinkTunnelDeps, "setTimer" | "clearTimer">> { return { setTimer: deps.setTimer ?? ((callback, ms) => setTimeout(callback, ms)), clearTimer: deps.clearTimer ?? (timer => clearTimeout(timer)), }; } async function stopChild(child: SshChild, deps: ClientLinkTunnelDeps): Promise<void> { const { setTimer, clearTimer } = timerDeps(deps); try { child.kill("SIGTERM"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } let timer: ReturnType<typeof setTimeout> | undefined; let exited = false; const exitedPromise = child.exited.then(() => { exited = true; }, () => { exited = true; }); const timeout = new Promise<void>(resolve => { timer = setTimer(resolve, STOP_TIMEOUT_MS); }); await Promise.race([exitedPromise, timeout]); if (timer !== undefined) clearTimer(timer); if (!exited) { try { child.kill("SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } } } export function spawnClientLinkTunnel(spec: ClientLinkTunnelSpec, deps: ClientLinkTunnelDeps = {}): ClientLinkTunnelHandle { if (!isLinkPort(spec.tunnelPort)) throw new Error("client tunnel port is outside the link range"); const knownHostsFile = deps.knownHostsFile ?? linkKnownHostsPath(deps.configDir); const runner = deps.runner ?? createSshRunner(); // Off unless the operator opened a capture window on this machine; then only the level changes. const capture = openSshCapture(deps.sshCapture ?? defaultSshCaptureOpener(linkDir(deps.configDir)), "child", spec.linkId); const argv = buildTunnelArgv({ alias: spec.alias, direction: "L", bindPort: spec.tunnelPort, targetPort: spec.peerListenerPort, knownHostsFile, ...(capture ? { logLevel: capture.logLevel } : {}), route: null, }); const child = capture ? runner.spawnTunnel(argv, { stderrLine: capture.stderrLine }) : runner.spawnTunnel(argv); const pidfile = clientTunnelPidfilePath(deps.configDir); const startTime = processStartTime(child.pid, process.platform, deps); try { writePidfile(pidfile, { version: 1, linkId: spec.linkId, pid: child.pid, argv: [...argv], ownerPid: process.pid, startTime }); } catch (error) { try { child.kill("SIGTERM"); } catch (killError) { if ((killError as NodeJS.ErrnoException).code !== "ESRCH") throw killError; } throw error; } let stopPromise: Promise<void> | undefined; const handleExit = (): void => { removePidfileIfPid(pidfile, child.pid, startTime); }; const handle = { pid: child.pid, startTime, exited: child.exited, stderr: child.stderr, stop(): Promise<void> { if (stopPromise) return stopPromise; stopPromise = stopChild(child, deps).finally(() => removePidfileIfPid(pidfile, child.pid, startTime)); return stopPromise; }, } satisfies ClientLinkTunnelHandle & { stderr?: Promise<string> }; void child.exited.then(handleExit, handleExit); return handle; } type TunnelIdentity = | { kind: "gone" } | { kind: "other" } | { kind: "ours"; orphaned: boolean; startTime: string } | { kind: "unknown" }; /** * Whether the pidfile's process is still our tunnel. Linux compares `/proc/<pid>/cmdline` with * the recorded argv. macOS compares `ps` args with the argv joined by spaces and reports the * process orphaned only when launchd (pid 1) is its parent. Elsewhere a live process is unknown. */ function tunnelIdentity( pidfile: ClientTunnelPidfile, platform: NodeJS.Platform, deps: OrphanReapDeps, isAlive: (pid: number) => boolean, ): TunnelIdentity { if (platform === "linux") { const actualArgv = (deps.readProcessArgv ?? linuxProcessArgv)(pidfile.pid); if (!actualArgv) return { kind: "gone" }; if (!sameArgv(actualArgv, pidfile.argv)) return { kind: "other" }; const startTime = processStartTime(pidfile.pid, platform, deps); if (!pidfile.startTime || !startTime) return { kind: "unknown" }; return startTime === pidfile.startTime ? { kind: "ours", orphaned: true, startTime } : { kind: "other" }; } if (!isAlive(pidfile.pid)) return { kind: "gone" }; if (platform !== "darwin") return { kind: "unknown" }; const info = (deps.readProcessInfo ?? darwinProcessInfo)(pidfile.pid); if (!info) return { kind: "unknown" }; if (info.args !== pidfile.argv.join(" ")) return { kind: "other" }; if (!pidfile.startTime || !info.startTime) return { kind: "unknown" }; return info.startTime === pidfile.startTime ? { kind: "ours", orphaned: info.ppid === 1, startTime: info.startTime } : { kind: "other" }; } /** * Settle a leftover tunnel pidfile before a new tunnel starts. * * - A pidfile whose process is gone, or is provably another program (a reused pid, as after a * reboot), is stale: it is removed and the result is `absent`, so a new tunnel starts. * - While the owner lives, a tunnel that is still ours (or cannot be told apart) is `owned`. * - After the owner exits, a proven orphan is reaped: Linux on an exact `/proc` argv match, macOS * on an exact `ps` argv match with launchd as the parent. TERM, up to five seconds, KILL. * - Anything else is `unresolved`: never signalled; the caller watches it. */ export async function reapOrphanTunnel(deps: OrphanReapDeps = {}): Promise<OrphanTunnelResult> { const path = clientTunnelPidfilePath(deps.configDir); const pidfile = readPidfile(path); if (!pidfile) return { tunnel: "absent" }; const isAlive = deps.isAlive ?? defaultIsAlive; const platform = deps.platform ?? process.platform; const identity = tunnelIdentity(pidfile, platform, deps, isAlive); if (identity.kind === "gone" || identity.kind === "other") { removePidfileIfPid(path, pidfile.pid, pidfile.startTime); return { tunnel: "absent" }; } if (isAlive(pidfile.ownerPid)) return { tunnel: "owned", pid: pidfile.pid }; if (identity.kind !== "ours" || !identity.orphaned) return { tunnel: "unresolved", pid: pidfile.pid }; // The pid may have been reused since the first read. Recheck the saved start identity and // orphan proof immediately before each signal, including the escalation after the wait. const stillOurOrphan = (): boolean => { const current = tunnelIdentity(pidfile, platform, deps, isAlive); return current.kind === "ours" && current.orphaned && current.startTime === identity.startTime; }; if (!stillOurOrphan()) return { tunnel: "unresolved", pid: pidfile.pid }; const signal = deps.signal ?? defaultSignal; const sleep = deps.sleep ?? ((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms))); try { signal(pidfile.pid, "SIGTERM"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } for (let waited = 0; waited < STOP_TIMEOUT_MS && isAlive(pidfile.pid); waited += REAP_POLL_MS) await sleep(REAP_POLL_MS); if (isAlive(pidfile.pid)) { if (!stillOurOrphan()) return { tunnel: "unresolved", pid: pidfile.pid }; try { signal(pidfile.pid, "SIGKILL"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } } removePidfileIfPid(path, pidfile.pid, pidfile.startTime); return { tunnel: "reaped" }; } function defaultConnectedLinkId(): string | null | typeof CONNECTION_UNREADABLE { const state = readClientConnectionState(); if (state.kind === "invalid" || state.kind === "mismatched") return CONNECTION_UNREADABLE; if (state.kind !== "connected" || !isLinkConnection(state.value)) return null; return state.value.link?.linkId ?? null; } function fileSignature(path: string): string | null { try { const stat = statSync(path); return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`; } catch (error) { return isMissingPathError(error) ? "missing" : null; } } /** * `read` again only when the file at `path` changed, so a steady link costs one stat per check * instead of a parse. A read that throws, a value `keep` refuses, or a failed stat is never cached. */ export function readWhenFileChanges<T>(path: () => string, read: () => T, keep: (value: T) => boolean = () => true): () => T { let cached: { signature: string; value: T } | null = null; return () => { const signature = fileSignature(path()); if (signature !== null && cached?.signature === signature) return cached.value; cached = null; const value = read(); if (signature !== null && keep(value)) cached = { signature, value }; return value; }; } /** `ready` and `home_not_ready` both prove the link works; the rest do not. */ type ProbeResult = "ready" | ClientTunnelProbeReason; /** Whether a 503 `/readyz` body is the Home's own readiness answer (`service: "opencodex"`). */ async function isOpencodexReadiness(response: Response, signal: AbortSignal): Promise<boolean> { try { const { bytes, oversized } = await readBoundedResponseBytes(response, { maxBytes: PROBE_BODY_MAX_BYTES, signal }); if (oversized) return false; const body = JSON.parse(new TextDecoder().decode(bytes)) as unknown; return typeof body === "object" && body !== null && (body as { service?: unknown }).service === "opencodex"; } catch { return false; } } /** * `GET /readyz` through the tunnel with the link key; the key is sent as a header only. The Home's * link listener answers 401 before it reaches `/readyz`, so both a 200 and a 503 carrying the * Home's readiness body prove that the forward reaches that listener and that the key is admitted. * The caller first proves the local LISTEN socket belongs to its SSH process; an arbitrary 200 * from a competing process never earns a keyed request. * The 503 only means the Home's own startup readiness is pending or failed, which does not stop * relayed requests; it is reported as `home_not_ready`, for display. */ async function probeTunnel(fetchImpl: typeof fetch, tunnelPort: number, key: string, stop: AbortSignal): Promise<ProbeResult> { const signal = AbortSignal.any([stop, AbortSignal.timeout(PROBE_TIMEOUT_MS)]); try { const response = await fetchImpl(`http://127.0.0.1:${tunnelPort}/readyz`, { headers: { "x-opencodex-api-key": key }, cache: "no-store", redirect: "manual", signal, }); if (response.status === 503) return await isOpencodexReadiness(response, signal) ? "home_not_ready" : "home_unreachable"; try { await response.body?.cancel(); } catch { /* the body is not needed */ } if (response.status === 200) return "ready"; return response.status === 401 || response.status === 403 ? "unauthorized" : "home_unreachable"; } catch { return "home_unreachable"; } } /** * Own the Child's SSH lifecycle and reconnect holds. Each connected() call checks fresh listener * ownership and fences results against tunnel replacement or stop; diagnostics cannot admit it. */ export function createClientLinkSupervisor(deps: ClientLinkSupervisorDeps = {}): ClientLinkSupervisor { const sidecarPath = (): string => clientLinkStatePath(deps.configDir); const readSidecar = deps.readSidecar ?? readWhenFileChanges(sidecarPath, () => readClientLinkState(sidecarPath())); const connectedLinkId = deps.connectedLinkId ?? readWhenFileChanges(getConfigPath, defaultConnectedLinkId, value => value !== CONNECTION_UNREADABLE); const now = deps.now ?? (() => Date.now()); const random = deps.random ?? Math.random; const policy = CLIENT_TUNNEL_RETRY_POLICY; const isAlive = deps.isAlive ?? defaultIsAlive; const linkKey = deps.linkKey ?? (() => null); const ownerLookup = deps.inspectLoopbackListener ?? inspectIpv4LoopbackListener; const fetchImpl = deps.fetchImpl ?? fetch; const setSupervisorTimer = deps.setTimer ?? ((callback: () => void, ms: number) => { const interval = setInterval(callback, ms); (interval as { unref?: () => void }).unref?.(); return interval as unknown as ReturnType<typeof setTimeout>; }); const clearSupervisorTimer = deps.clearTimer ?? ((timer: ReturnType<typeof setTimeout>) => clearInterval(timer as unknown as ReturnType<typeof setInterval>)); // Capture files an earlier process wrote are deleted on time even if this one never spawns. if (!deps.sshCapture) retainSshCapture(linkDir(deps.configDir)); let timer: ReturnType<typeof setTimeout> | undefined; let started = false; let stopping = false; // Told once when stop() begins or the link ends (endLink), before the tunnel is signalled (the link // WebSocket drains here). const stoppingListeners: Array<() => void> = []; const notifyStopping = (): void => { for (const listener of stoppingListeners.splice(0)) { try { listener(); } catch { /* a listener never delays the stop */ } } }; let initialized = false; let initializing = false; let onLinkEndedCalled = false; let child: ClientLinkTunnelHandle | undefined; /** A leftover tunnel this supervisor may not signal: watched, and replaced once it dies. */ let adopted: number | null = null; let adoptedRecord: ClientTunnelPidfile | null = null; let tunnelPort: number | null = null; let ownerGeneration = 0; /** Minted on every transition into connected from the process-wide sequence (relay-auth.ts). */ let connection: string | undefined; let state: TunnelState = IDLE; /** * Child tunnel transitions go to the same bounded diagnostics file as the Home's, with the same * fixed causes (#7297). The port is remembered past stop, which clears the live port first. */ let lastTunnelPort: number | undefined; // The record is built at the transition but delivered after the current synchronous step, so a // sink that re-enters the supervisor (stop, status) never runs in the middle of a transition. const transitionSink = deps.diagnostic; const history = createTunnelHistoryBook({ diagnostic: transitionSink ? record => queueMicrotask(() => { try { transitionSink(record); } catch { /* best effort */ } }) : undefined, tunnelPort: () => lastTunnelPort, }); let linkId: string | null = null; let failure: ClientLinkSupervisorStatus | undefined; let tickFlight: Promise<void> | undefined; /** The one keyed probe in flight, which tunnel it probes, and the abort that stop() fires. */ let probeFlight: { child: ClientLinkTunnelHandle | undefined; adopted: number | null; abort: AbortController } | undefined; let probe: ClientTunnelProbeReason | null = null; let nextProbeAt = 0; let probeFailures = 0; let invalidReads = 0; /** A leftover tunnel pidfile has not been settled yet; nothing may spawn until it is. */ let reapPending = true; const waiters = new Set<(connected: boolean) => void>(); /** The tunnel is being (re)established, so a request may wait for it instead of failing. */ const pending = (): boolean => !stopping && failure === undefined && ((started && !initialized) || state.kind === "connecting" || state.kind === "reconnecting"); const invalidateOwnerProof = (): void => { ownerGeneration += 1; }; const ownerCandidate = (port: number): { pid: number; generation: number } | null => { if (port !== tunnelPort) return null; const pid = child?.pid ?? adopted; if (pid === null || pid === undefined || (adopted !== null && !adoptedRecord)) return null; return { pid, generation: ownerGeneration }; }; const listenerOwnedByTunnel = async (port: number, context?: { requestId?: string }): Promise<LinkAdmissionResult> => { const candidate = ownerCandidate(port); if (!candidate) return { kind: "unavailable" }; const checkId = randomUUID(); const report = (kind: LinkAdmissionResult["kind"], elapsedMs = 0): LinkAdmissionResult => { const check = { kind, elapsedMs, checkId, generation: candidate.generation, state: state.kind }; if (kind !== "match") try { deps.diagnostic?.({ event: "owner_lookup", outcome: kind, requestId: context?.requestId, checkId, generation: candidate.generation, state: state.kind, elapsedMs }); } catch { /* best effort */ } return check; }; if (adopted !== null) { const record = adoptedRecord; const identity = record ? await inspectAdoptedIdentity(record, deps) : "unknown"; if (identity !== "match") { if (identity === "mismatch" && record && adopted === candidate.pid && ownerGeneration === candidate.generation) { // The old PID may now name another process. Never signal it; discard only our stale // pidfile and let the next supervisor tick launch a fresh tunnel. abortProbe(); removePidfileIfPid(clientTunnelPidfilePath(deps.configDir), candidate.pid, record.startTime); adopted = null; adoptedRecord = null; tunnelPort = null; linkId = null; probe = null; invalidateOwnerProof(); setState(IDLE); } return report(identity === "mismatch" ? "identity_mismatch" : "identity_unknown"); } } const result = await Promise.resolve().then(() => ownerLookup(port, candidate.pid)) .catch((): LoopbackOwnerLookupResult => ({ kind: "lookup_error", elapsedMs: 0 })); if (stopping || failure !== undefined || ownerGeneration !== candidate.generation || ownerCandidate(port)?.pid !== candidate.pid) { return report("stale", result.elapsedMs); } return report(result.kind, result.elapsedMs); }; const connected = async (context?: { requestId?: string }): Promise<LinkAdmissionResult> => { if (!started || stopping || failure !== undefined || state.kind !== "connected" || linkId === null || tunnelPort === null) { return { kind: "unavailable" }; } const result = await listenerOwnedByTunnel(tunnelPort, context); // State may change during the asynchronous lookup or an optional diagnostic callback. if (result.kind === "match" && (!started || stopping || failure !== undefined || state.kind !== "connected" || result.generation !== ownerGeneration)) { return { ...result, kind: "stale" }; } return result; }; const settleWaiters = (): void => { if (waiters.size === 0) return; const ready = !stopping && failure === undefined && state.kind === "connected"; if (!ready && pending()) return; for (const settle of [...waiters]) settle(ready); }; const setState = (next: TunnelState, cause: TunnelCause | null = null): void => { const before = state; if (next.kind === "connected" && state.kind !== "connected") connection = nextLinkConnectionGeneration(); state = next; // Records only a real change and swallows sink errors; diagnostics never steer the lifecycle. history.observe(CHILD_TUNNEL_HISTORY_KEY, before, next, cause, now()); settleWaiters(); }; const awaitingReady = (): boolean => state.kind === "connecting" || state.kind === "reconnecting" || (state.kind === "failed" && state.inFlight === true); const readCurrent = (): { sidecar: ClientLinkState | null; invalid: boolean } => { try { return { sidecar: readSidecar(), invalid: false }; } catch { deps.warn?.("client link sidecar could not be read"); return { sidecar: null, invalid: true }; } }; const readConnectedLinkId = (): string | null | typeof CONNECTION_UNREADABLE => { try { return connectedLinkId(); } catch { return CONNECTION_UNREADABLE; } }; /** Drops the probe in flight; its answer, if it still comes, is ignored. */ const abortProbe = (): void => { const flight = probeFlight; probeFlight = undefined; flight?.abort.abort(); }; /** Stops our own child only; the tunnel state is left for the caller to decide. */ const killChild = async (): Promise<void> => { const current = child; invalidateOwnerProof(); child = undefined; probe = null; abortProbe(); if (current) await current.stop(); }; const stopTunnel = async (): Promise<void> => { invalidateOwnerProof(); adopted = null; adoptedRecord = null; tunnelPort = null; setState(reduceTunnel(state, { type: "stop" })); await killChild(); }; const endLink = async (): Promise<void> => { // The link ended and the runtime recycles: listeners drain before the tunnel goes, as on stop(). notifyStopping(); await stopTunnel(); if (onLinkEndedCalled || stopping) return; linkId = null; onLinkEndedCalled = true; deps.onLinkEnded?.(); }; const onExit = (stderrClass: StderrClass): void => { setState(reduceTunnel(state, { type: "exit", now: now(), stderrClass }, random, policy), exitCause(stderrClass)); }; const spawn = (sidecar: ClientLinkState): void => { if (stopping || child || adopted !== null || reapPending) return; const timestamp = now(); if (state.kind === "failed" && !dueForSpawn(state, timestamp)) return; try { child = spawnClientLinkTunnel({ linkId: sidecar.linkId, alias: sidecar.alias, tunnelPort: sidecar.tunnelPort, peerListenerPort: sidecar.peerListenerPort, }, deps); } catch { // A due retry that cannot start counts as a failed retry, so the backoff keeps its step. lastTunnelPort = sidecar.tunnelPort; setState(failedSpawnTunnel(state, "forward", timestamp, policy), "spawn_failed"); deps.warn?.("client link tunnel could not be started"); return; } invalidateOwnerProof(); linkId = sidecar.linkId; tunnelPort = sidecar.tunnelPort; lastTunnelPort = sidecar.tunnelPort; history.spawned(CHILD_TUNNEL_HISTORY_KEY); probe = null; probeFailures = 0; nextProbeAt = timestamp; setState(reduceTunnel(state, { type: "spawn", now: timestamp }, random, policy)); const current = child; void current.exited.then(async () => { if (child !== current) return; invalidateOwnerProof(); child = undefined; probe = null; const stderr = (current as ClientLinkTunnelHandle & { stderr?: Promise<string> }).stderr ? await (current as ClientLinkTunnelHandle & { stderr?: Promise<string> }).stderr!.catch(() => "") : ""; onExit(classifySshStderr(stderr)); }).catch(() => { if (child !== current) return; invalidateOwnerProof(); child = undefined; probe = null; onExit("network"); }); }; /** * Applies one keyed probe's answer. While the tunnel is being established an answer that proves * the link (`ready`, or `home_not_ready`) promotes it to connected; otherwise the next probe backs * off from one to five seconds, or up to 30 seconds once the link reads failed (a revoked key or a * stopped Home costs one probe per 30 s). While connected the answer is display-only, and the next * probe is 30 seconds out. */ const applyProbe = (result: ProbeResult, wasConnected: boolean): void => { const timestamp = now(); const reason = result === "ready" ? null : result; if (wasConnected) { if (state.kind === "connected") probe = reason; nextProbeAt = timestamp + CONNECTED_PROBE_MS; return; } if ((result === "ready" || result === "home_not_ready") && awaitingReady()) { probe = reason; probeFailures = 0; nextProbeAt = timestamp + CONNECTED_PROBE_MS; setState(reduceTunnel(state, { type: "ready", now: timestamp }, random, policy)); return; } probe = reason; probeFailures += 1; const cap = state.kind === "failed" ? CONNECTED_PROBE_MS : PROBE_BACKOFF_MAX_MS; nextProbeAt = timestamp + Math.min(cap, 1_000 * 2 ** Math.min(probeFailures - 1, 16)); }; /** * Starts one keyed probe only after a local socket-owner check. A slow Home never delays the * check that notices a disconnect, and stop() aborts it instead of waiting for probe timeout. */ const startProbe = (tunnelPort: number): void => { const flight = { child, adopted, abort: new AbortController() }; probeFlight = flight; const wasConnected = state.kind === "connected"; void (async () => { if ((await listenerOwnedByTunnel(tunnelPort)).kind !== "match") return; if (probeFlight !== flight || stopping || flight.abort.signal.aborted || child !== flight.child || adopted !== flight.adopted) return; const key = linkKey(); if (!key) return; const result = await probeTunnel(fetchImpl, tunnelPort, key, flight.abort.signal); // An aborted probe, or one whose listener changed while it ran, proves nothing. if (probeFlight !== flight || stopping || child !== flight.child || adopted !== flight.adopted || (await listenerOwnedByTunnel(tunnelPort)).kind !== "match") return; applyProbe(result, wasConnected); })().catch(() => undefined).finally(() => { if (probeFlight === flight) probeFlight = undefined; }); }; /** * Settles a leftover tunnel pidfile once, before this supervisor's first spawn: a proven orphan * is reaped and a live tunnel that may not be ours is watched without a keyed probe until its * process identity and LISTEN ownership are verified. It needs a valid, matching read * after the reap; without one it stays pending and the next check that has one runs it again. */ const settleLeftover = async (): Promise<void> => { const orphan = await reapOrphanTunnel(deps); if (stopping) return; const afterReap = readCurrent(); if (afterReap.invalid || !afterReap.sidecar || readConnectedLinkId() !== afterReap.sidecar.linkId) return; reapPending = false; if ((orphan.tunnel === "owned" || orphan.tunnel === "unresolved") && orphan.pid !== undefined && isAlive(orphan.pid)) { // A leftover tunnel that may not be ours to stop: watch it, probe only after identity and // socket ownership match, and start our own once it dies. It is never signalled. adopted = orphan.pid; const record = readPidfile(clientTunnelPidfilePath(deps.configDir)); const identity = record?.pid === adopted && record.linkId === afterReap.sidecar.linkId ? tunnelIdentity(record, deps.platform ?? process.platform, deps, isAlive) : null; adoptedRecord = identity?.kind === "ours" ? record : null; invalidateOwnerProof(); linkId = afterReap.sidecar.linkId; tunnelPort = afterReap.sidecar.tunnelPort; lastTunnelPort = afterReap.sidecar.tunnelPort; nextProbeAt = now(); setState({ kind: "connecting", since: now() }); return; } spawn(afterReap.sidecar); }; const tick = async (): Promise<void> => { if (stopping || !initialized) return; const current = readCurrent(); const connected = current.invalid ? null : readConnectedLinkId(); if (current.invalid || connected === CONNECTION_UNREADABLE) { // A single unreadable read (a write in progress, a transient I/O error) must not end a // healthy link; only INVALID_READ_TICKS in a row do. invalidReads += 1; if (invalidReads < INVALID_READ_TICKS) return; if (current.invalid) { failure = { kind: "failed", reason: "sidecar_invalid" }; await stopTunnel(); } else if (child || linkId || adopted !== null) { await endLink(); } settleWaiters(); return; } invalidReads = 0; if (!current.sidecar || connected !== current.sidecar.linkId) { if (child || linkId || adopted !== null) await endLink(); return; } failure = undefined; const sidecar = current.sidecar; if (reapPending) { // The start-up read was unreadable or did not match, so a leftover tunnel was never // settled. Settle it now, before anything spawns, so an orphan cannot keep the port. await settleLeftover(); settleWaiters(); return; } if (adopted !== null && !isAlive(adopted)) { // The leftover tunnel is gone: start our own on this tick. invalidateOwnerProof(); adopted = null; adoptedRecord = null; tunnelPort = null; probe = null; setState(IDLE); } // A tick changes state only when the outage limit fails the link, so its cause is the timeout. setState(reduceTunnel(state, { type: "tick", now: now() }, random, policy), "timeout"); // An adopted tunnel is never killed. Failed identity or socket-owner proof keeps it // unprobed and unable to promote, even when the process stays alive after a timeout. if (state.kind === "failed" && !state.inFlight && adopted !== null) setState({ ...state, inFlight: true }); if (state.kind === "failed" && !state.inFlight && child) await killChild(); // A probe still running for a tunnel that has since exited or died answers nothing useful. if (probeFlight && (probeFlight.child !== child || probeFlight.adopted !== adopted)) abortProbe(); // While requests are held the backoff does not apply: one probe per check (one a second), so a // forward that just came up releases them at once. Held requests exist only while it is down. if ((child || adopted !== null) && (state.kind === "connected" || awaitingReady()) && !probeFlight && (now() >= nextProbeAt || waiters.size > 0)) { startProbe(sidecar.tunnelPort); } if (!child && adopted === null && (state.kind === "idle" || dueForSpawn(state, now()))) spawn(sidecar); settleWaiters(); }; const runTick = (): void => { if (tickFlight) return; tickFlight = tick().finally(() => { tickFlight = undefined; }); }; const initialize = async (): Promise<void> => { if (initializing || initialized || stopping) return; initializing = true; try { const current = readCurrent(); if (current.invalid) { failure = { kind: "failed", reason: "sidecar_invalid" }; return; } // An unreadable or mismatched read leaves the leftover tunnel for the first check that // reads a matching link (reapPending), so no tunnel is ever spawned over an unreaped one. if (!current.sidecar || readConnectedLinkId() !== current.sidecar.linkId) return; await settleLeftover(); } finally { initialized = true; initializing = false; settleWaiters(); try { deps.onInitialized?.(); } catch { /* a test seam never alters the supervisor */ } } }; return { start(): void { if (started) return; started = true; stopping = false; timer = setSupervisorTimer(runTick, TIMER_MS); void initialize().catch(() => { failure = { kind: "failed", reason: "sidecar_invalid" }; initialized = true; initializing = false; settleWaiters(); }); }, async stop(): Promise<void> { if (stopping) { if (tickFlight) await tickFlight; return; } stopping = true; notifyStopping(); abortProbe(); settleWaiters(); if (timer !== undefined) { clearSupervisorTimer(timer); timer = undefined; } if (tickFlight) await tickFlight; await stopTunnel(); failure = undefined; }, onStopping(listener: () => void): void { stoppingListeners.push(listener); }, connectionGeneration(): string | undefined { if (!started || stopping || failure !== undefined || state.kind !== "connected") return undefined; return connection; }, status(): ClientLinkSupervisorStatus { if (failure) return failure; if (!child && !linkId && adopted === null) return { kind: "stopped" }; return { kind: "tunnel", linkId: linkId ?? "", state, pid: child?.pid ?? adopted, ...(probe ? { probe } : {}), }; }, pending, connected, waitForConnected(timeoutMs: number, signal?: AbortSignal): Promise<boolean> { if (state.kind === "connected") return connected().then(result => result.kind === "match"); if (!pending() || waiters.size >= CLIENT_LINK_MAX_HOLDS || signal?.aborted || !(timeoutMs > 0)) { return Promise.resolve(false); } return new Promise<boolean>(resolve => { let holdTimer: ReturnType<typeof setTimeout> | undefined; const onAbort = (): void => settle(false); const settle = (connected: boolean): void => { if (!waiters.delete(settle)) return; if (holdTimer !== undefined) clearTimeout(holdTimer); signal?.removeEventListener("abort", onAbort); resolve(connected); }; waiters.add(settle); holdTimer = setTimeout(() => settle(false), timeoutMs); (holdTimer as { unref?: () => void }).unref?.(); signal?.addEventListener("abort", onAbort, { once: true }); }); }, }; }