@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
515 lines (478 loc) • 21.2 kB
text/typescript
import { spawn, type ChildProcess } from "node:child_process";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { ensureRestrictedDir } from "../paths";
import { FABRIC_LIMITS } from "./constants";
import {
FABRIC_PRODUCER_PROTOCOL_MAX_BYTES,
FABRIC_PRODUCER_REQUEST_MAX_BYTES,
FABRIC_PRODUCER_STDERR_MAX_BYTES,
parseProducerProtocolLine,
type IsolatedProducerResult,
} from "./producer-protocol";
import type { FabricHarnessProducerKind, FabricPatchExecutorInput, SyntheticPatchV1 } from "./types";
import { FabricTaskError } from "./types";
const CHILD_ENTRY = join(dirname(fileURLToPath(import.meta.url)), "producer-child.ts");
/**
* Bounded drain window between a child's `exit` and our decision. `close` also
* waits for the child's stdio to end, and a descendant holding an inherited pipe
* can delay it forever — so a missing `close` must not keep the run pending.
* The bound only has to cover an ordinary stdio flush after process death: the
* kernel releases a dead child's pipe ends immediately, so a `close` slower
* than this almost always means a descendant still holds a pipe. A slow-but-
* normal pipe that outlives the bound costs an inconclusive verdict plus
* deferred scratch cleanup — a bounded price that does not grow with the wait.
*/
const EXIT_DRAIN_MS = 250;
/**
* Bounded wait for exit/close after a parent-owned SIGKILL. Neither event is
* guaranteed — a child in uninterruptible sleep, or a kill() that failed, produces
* neither — and a latched killReason must not leave the run pending forever.
*/
const KILL_CONFIRM_MS = 2_000;
type FabricProducerIsolationLimits = {
totalTimeoutMs: number;
inactivityTimeoutMs: number;
};
let testIsolationLimits: FabricProducerIsolationLimits | undefined;
interface IsolateRequest {
harnessKind?: FabricHarnessProducerKind;
executorModulePath?: string;
scratchRoot: string;
totalTimeoutMs: number;
inactivityTimeoutMs: number;
executorInput?: FabricPatchExecutorInput;
now?: () => number;
}
/**
* The environment an isolated producer child runs with.
*
* Exported so a test that spawns `producer-child.ts` directly cannot drift from
* the environment production actually uses. The Windows loader state and the
* scratch-owned temp paths below are load-bearing, and a test carrying its own
* literal copy of this object silently loses them.
*/
export function minimalFabricChildEnv(scratchRoot: string): Record<string, string> {
const childTempDir = join(scratchRoot, ".tmp");
ensureRestrictedDir(childTempDir, scratchRoot);
const env: Record<string, string> = {
TZ: "UTC",
NO_COLOR: "1",
OCX_FABRIC_SCRATCH_ROOT: scratchRoot,
// Executors commonly use os.tmpdir() through libraries they import. Keep
// those writes inside the same scratch boundary instead of forwarding the
// user's ambient temp directory (Windows) or falling back to /tmp (POSIX).
TEMP: childTempDir,
TMP: childTempDir,
TMPDIR: childTempDir,
};
if (process.platform !== "win32") return env;
// Windows has no equivalent of "run with an (almost) empty environment". A
// CreateProcess child inherits nothing here, and the loader itself reads the
// environment: without SystemRoot it cannot resolve the system DLLs the Bun
// executable links against, so the child dies before its entry module runs.
// The parent then sees an immediate non-zero close with no protocol line and
// reports harness_failure -- which is what turned every CL-07 producer case
// into "inconclusive" on the Windows leg while POSIX stayed green.
//
// These are OS-owned loader state, not caller-supplied configuration. Temp
// state is deliberately not forwarded; it is rooted in scratch above.
for (const name of ["SystemRoot", "windir"] as const) {
const value = process.env[name];
if (value) env[name] = value;
}
return env;
}
function killChild(child: ChildProcess): void {
try {
child.kill("SIGKILL");
} catch {
/* ignore */
}
}
/** Whether the producer rejected while its child might still be running. */
export function isUnconfirmedProducerTermination(error: unknown): boolean {
return error instanceof FabricTaskError
&& (error as FabricTaskError & { unconfirmedTermination?: boolean }).unconfirmedTermination === true;
}
/**
* The release signal attached to an unconfirmed-termination rejection: resolves
* once every monitored inherited stdio pipe reports its natural close. This is
* observation only: closing a pipe does not prove a descendant exited and must
* not authorize deletion. Undefined when nothing monitorable remained.
*/
export function producerTerminationSignal(error: unknown): Promise<void> | undefined {
if (!error || typeof error !== "object") return undefined;
const signal = (error as { stdioRelease?: unknown }).stdioRelease;
return signal instanceof Promise ? signal : undefined;
}
/** Run a fabric patch producer in an isolated child process with parent-owned timeouts. */
export async function runIsolatedFabricProducer(request: IsolateRequest): Promise<IsolatedProducerResult> {
const now = request.now ?? (() => Date.now());
let lastActivityAt = now();
// Budget enforcement must not follow wall-clock adjustments; telemetry still does.
const budgetNow = request.now ?? (() => performance.now());
const startedAt = request.now ? lastActivityAt : budgetNow();
const totalDeadline = startedAt + request.totalTimeoutMs;
let inactivityDeadline = startedAt + request.inactivityTimeoutMs;
return await new Promise<IsolatedProducerResult>((resolve, reject) => {
let child: ChildProcess;
try {
child = spawn(process.execPath, ["run", CHILD_ENTRY], {
env: minimalFabricChildEnv(request.scratchRoot),
stdio: ["pipe", "pipe", "pipe"],
});
} catch (error) {
reject(new FabricTaskError(
error instanceof Error ? error.message : String(error),
"harness_failure",
"harness",
));
return;
}
let stdoutBuffer = "";
let stderrBytes = 0;
let settled = false;
let childClosed = false;
let childExitedAt: number | undefined;
let receivedResult: SyntheticPatchV1 | undefined;
let killReason: FabricTaskError | undefined;
let reapTimer: ReturnType<typeof setTimeout> | undefined;
let killWatchdog: ReturnType<typeof setTimeout> | undefined;
// Resolves once every still-open inherited stdio pipe reports its natural
// close. This is not a writer-termination proof or scratch deletion lease.
// Undefined when no open pipe can be monitored.
let stdioReleaseSignal: Promise<void> | undefined;
// Keep monitorable pipes open but unref'd so they never extend process
// lifetime; a stream without unref() is destroyed instead, and its
// self-inflicted close must not count toward the release signal.
const armStdioRelease = (): void => {
const waiters: Promise<void>[] = [];
let unmonitorable = false;
for (const stream of [child.stdout, child.stderr]) {
if (!stream || stream.destroyed) continue;
const unref = (stream as unknown as { unref?: unknown }).unref;
if (typeof unref === "function") {
waiters.push(new Promise<void>((resolve) => stream.once("close", resolve)));
unref.call(stream);
} else {
unmonitorable = true;
try { stream.destroy(); } catch { /* already closed */ }
}
}
if (waiters.length > 0 && !unmonitorable) {
stdioReleaseSignal = Promise.all(waiters).then(() => undefined);
}
};
const finish = (fn: () => void) => {
// A latched failure owns settlement, but scratch cleanup must wait for close.
if (settled || (killReason && !childClosed)) return;
settled = true;
clearTimeout(totalTimer);
clearTimeout(inactivityTimer);
if (reapTimer) clearTimeout(reapTimer);
if (killWatchdog) clearTimeout(killWatchdog);
if (killReason) reject(killReason);
else fn();
};
const settleTimeout = (error: FabricTaskError) => {
if (settled || killReason) return;
killReason = error;
if (childClosed) finish(() => reject(error));
else {
killChild(child);
// SIGKILL does not guarantee exit/close: an uninterruptible child, or a
// kill() that failed, emits neither, and the latched killReason would
// otherwise keep this run pending forever. Bound the wait; on expiry
// reject with the original reason flagged unconfirmed so the caller
// defers scratch cleanup instead of racing a child that may live.
killWatchdog = setTimeout(() => {
if (childClosed || settled) return;
settled = true;
clearTimeout(totalTimer);
clearTimeout(inactivityTimer);
if (reapTimer) clearTimeout(reapTimer);
if (killWatchdog) {
clearTimeout(killWatchdog);
killWatchdog = undefined;
}
armStdioRelease();
try { child.unref(); } catch { /* fake children may lack unref */ }
const reason = killReason! as FabricTaskError & { unconfirmedTermination?: boolean; stdioRelease?: Promise<void> };
reason.unconfirmedTermination = true;
reason.stdioRelease = stdioReleaseSignal;
reject(killReason);
}, KILL_CONFIRM_MS);
}
};
const expiredDeadline = (at: number): FabricTaskError | undefined => {
// Choose the earliest deadline, regardless of which timer/data callback ran first.
if (at >= inactivityDeadline && inactivityDeadline <= totalDeadline) {
return new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment");
}
if (at >= totalDeadline) {
return new FabricTaskError("total timeout exceeded", "timeout", "environment");
}
return undefined;
};
const onInactivityTimeout = () => {
settleTimeout(expiredDeadline(budgetNow())
?? new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment"));
};
const armInactivity = () => {
clearTimeout(inactivityTimer);
inactivityTimer = setTimeout(onInactivityTimeout, request.inactivityTimeoutMs);
};
let inactivityTimer: ReturnType<typeof setTimeout> = setTimeout(onInactivityTimeout, request.inactivityTimeoutMs);
const totalTimer = setTimeout(() => {
settleTimeout(expiredDeadline(budgetNow())
?? new FabricTaskError("total timeout exceeded", "timeout", "environment"));
}, request.totalTimeoutMs);
const handleProtocolLine = (line: string) => {
if (settled || killReason) return;
try {
const message = parseProducerProtocolLine(line);
if (message.type === "activity" || message.type === "result") {
// Bytes drained after `exit` are judged at the exit timestamp: the
// process met its budgets when it died, so the drain must not
// condemn data it wrote while still inside them.
const at = childExitedAt ?? budgetNow();
const expired = expiredDeadline(at);
if (expired) {
settleTimeout(expired);
return;
}
if (message.type === "activity") {
lastActivityAt = request.now ? at : now();
inactivityDeadline = at + request.inactivityTimeoutMs;
if (childExitedAt === undefined) armInactivity();
return;
}
}
if (message.type === "result") {
receivedResult = message.patch;
return;
}
if (message.type === "error") {
const fabricCode = message.code === "inactivity_timeout"
? "inactivity_timeout"
: message.code === "timeout"
? "timeout"
: message.code === "sandbox_violation"
? "sandbox_violation"
: message.code === "budget_exhausted"
? "budget_exhausted"
: "harness_failure";
const attribution = message.attribution === "environment" ? "environment" : "harness";
const fabricError = new FabricTaskError(message.message, fabricCode, attribution);
settleTimeout(fabricError);
return;
}
} catch (error) {
settleTimeout(new FabricTaskError(
error instanceof Error ? error.message : String(error),
"harness_failure",
"harness",
));
}
};
const consumeStdout = (chunk: string) => {
if (settled || killReason) return;
stdoutBuffer += chunk;
if (Buffer.byteLength(stdoutBuffer, "utf8") > FABRIC_PRODUCER_PROTOCOL_MAX_BYTES) {
settleTimeout(new FabricTaskError("producer protocol output exceeded limit", "budget_exhausted", "environment"));
stdoutBuffer = "";
return;
}
let newlineIdx = stdoutBuffer.indexOf("\n");
while (newlineIdx >= 0) {
const line = stdoutBuffer.slice(0, newlineIdx);
stdoutBuffer = stdoutBuffer.slice(newlineIdx + 1);
handleProtocolLine(line);
newlineIdx = stdoutBuffer.indexOf("\n");
}
};
child.stdout?.on("data", (chunk: Buffer | string) => {
consumeStdout(chunk.toString("utf8"));
});
child.stdout?.on("error", (error) => {
settleTimeout(new FabricTaskError(error.message, "harness_failure", "harness"));
});
child.stderr?.on("data", (chunk: Buffer | string) => {
stderrBytes += Buffer.byteLength(chunk.toString("utf8"), "utf8");
if (stderrBytes > FABRIC_PRODUCER_STDERR_MAX_BYTES) {
stderrBytes = FABRIC_PRODUCER_STDERR_MAX_BYTES;
}
});
child.stderr?.on("error", (error) => {
settleTimeout(new FabricTaskError(error.message, "harness_failure", "harness"));
});
child.on("error", (error) => {
if (child.pid === undefined) {
// A spawn failure has no process to supervise and may never emit close.
childClosed = true;
finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness")));
return;
}
settleTimeout(new FabricTaskError(error.message, "harness_failure", "harness"));
});
child.stdin?.on("error", (error: NodeJS.ErrnoException) => {
if (error.code === "EPIPE") return;
settleTimeout(new FabricTaskError(error.message, "harness_failure", "harness"));
});
const decide = (code: number | null, signal: NodeJS.Signals | null, reaped = false) => {
if (settled) return;
if (killReason) {
const reason = killReason as FabricTaskError & { unconfirmedTermination?: boolean; stdioRelease?: Promise<void> };
if (reaped) {
// The kill was answered by exit but the pipes stayed open: a
// descendant can still hold them, so scratch cleanup must defer to
// the same release contract as an unconfirmed kill.
reason.unconfirmedTermination = true;
reason.stdioRelease = stdioReleaseSignal;
}
finish(() => reject(reason));
return;
}
if (stdoutBuffer.trim()) {
try {
handleProtocolLine(stdoutBuffer.trim());
if (settled) return;
} catch {
/* fall through */
}
}
if (reaped) {
// `close` never followed `exit`: the pipes outlived the producer, which
// may mean a descendant escaped supervision — but the drain also cannot
// rule out a stalled event loop or a slow pipe, so this is reported as
// an inconclusive harness failure rather than a sandbox escape. Either
// way the result is rejected: it must never resolve while a descendant
// might still be alive to mutate scratch after cleanup. The recorded
// exit status only narrows the message; the deferral contract is the
// same for a clean exit and a nonzero or signaled one.
const failure = new FabricTaskError(
code !== 0 || signal
? `isolated producer exited (${code ?? signal ?? "unknown"}); its stdio never closed`
: "isolated producer exited but its stdio never closed",
"harness_failure",
"harness",
) as FabricTaskError & { unconfirmedTermination?: boolean; stdioRelease?: Promise<void> };
// A descendant holding an inherited pipe can still use scratch after
// the direct child exited, so the caller must retain scratch for manual
// review — pipe closure alone cannot prove a descendant terminated.
failure.unconfirmedTermination = true;
failure.stdioRelease = stdioReleaseSignal;
finish(() => reject(failure));
return;
}
// A stored result is accepted only when the child exited normally. A
// signaled or nonzero exit without a latched reason is a harness failure —
// parent-owned kills always carry a killReason, so this is never a timeout.
if (code !== 0 || signal) {
finish(() => reject(new FabricTaskError(
`isolated producer exited (${code ?? signal ?? "unknown"})`,
"harness_failure",
"harness",
)));
return;
}
if (receivedResult) {
finish(() => resolve({ patch: receivedResult!, lastActivityAt }));
return;
}
finish(() => reject(new FabricTaskError("isolated producer returned no result", "harness_failure", "harness")));
};
child.on("exit", (code, signal) => {
if (childClosed || settled) return;
// `exit` ends the budget window even while `close` is still pending on
// stdio: an already-met deadline still applies, and no producer code can
// breach one after this point, so both budget timers are disarmed now.
childExitedAt = budgetNow();
if (!killReason) {
const expired = expiredDeadline(childExitedAt);
if (expired) killReason = expired;
}
clearTimeout(totalTimer);
clearTimeout(inactivityTimer);
// The direct child is confirmed dead, so the kill-confirmation watchdog
// is moot; only the stdio drain still needs its bound.
if (killWatchdog) {
clearTimeout(killWatchdog);
killWatchdog = undefined;
}
// The direct child is dead, but `close` also waits for its stdio to end.
// Bound the drain so a descendant holding an inherited pipe cannot keep
// the run pending, then settle from the recorded exit status.
reapTimer = setTimeout(() => {
if (reapTimer) {
clearTimeout(reapTimer);
reapTimer = undefined;
}
if (childClosed || settled) return;
childClosed = true;
armStdioRelease();
decide(code, signal, true);
}, EXIT_DRAIN_MS);
});
child.on("close", (code, signal) => {
if (childClosed) return;
childClosed = true;
decide(code, signal);
});
const payload = JSON.stringify({
harnessKind: request.harnessKind,
executorModulePath: request.executorModulePath,
scratchRoot: request.scratchRoot,
totalTimeoutMs: request.totalTimeoutMs,
inactivityTimeoutMs: request.inactivityTimeoutMs,
executorInput: request.executorInput
? {
routeContext: request.executorInput.routeContext,
destination: request.executorInput.destination,
routeSubject: request.executorInput.routeSubject,
scratchRoot: request.executorInput.scratchRoot,
}
: undefined,
});
if (Buffer.byteLength(payload, "utf8") > FABRIC_PRODUCER_REQUEST_MAX_BYTES) {
settleTimeout(new FabricTaskError("producer request exceeds protocol limit", "budget_exhausted", "environment"));
return;
}
try {
child.stdin?.write(payload);
child.stdin?.end();
} catch (error) {
settleTimeout(new FabricTaskError(
error instanceof Error ? error.message : String(error),
"harness_failure",
"harness",
));
return;
}
});
}
/** Internal test seam. Production callers cannot arm it without the test-home guard. */
export function setFabricProducerIsolationLimitsForTests(limits?: FabricProducerIsolationLimits): void {
if (process.env.OCX_TEST_HOME_GUARD !== "1") {
throw new Error("fabric isolation limits can only be overridden by the test harness");
}
if (limits && (
!Number.isFinite(limits.totalTimeoutMs)
|| !Number.isFinite(limits.inactivityTimeoutMs)
|| limits.totalTimeoutMs <= 0
|| limits.inactivityTimeoutMs <= 0
|| limits.inactivityTimeoutMs >= limits.totalTimeoutMs
)) {
throw new Error("invalid fabric test isolation limits");
}
testIsolationLimits = limits ? { ...limits } : undefined;
}
/** Default isolation limits for fabric producer child processes. */
export function fabricProducerIsolationLimits(): FabricProducerIsolationLimits {
if (testIsolationLimits) return { ...testIsolationLimits };
return {
totalTimeoutMs: FABRIC_LIMITS.totalTimeoutMs,
inactivityTimeoutMs: FABRIC_LIMITS.inactivityTimeoutMs,
};
}