@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
395 lines (362 loc) • 16.5 kB
text/typescript
import { appendFileSync, mkdirSync } from "node:fs";
import { join } from "node:path";
import { getConfigDir } from "../config";
import { ConfigUninstallInProgressError, recordOwnedConfigPath } from "./config-ownership";
import { redactSecretString, redactUrlForLog } from "./redact";
import { sidecarBreadcrumb, activityBreadcrumb } from "./sidecar-tracker";
import { retainedUtf8Bytes, truncateRetainedUtf8 } from "./admission";
import { enforceAppOwnedMemoryBudget } from "./app-owned-memory";
/**
* Process-level safety net for the long-running proxy daemon.
*
* A single request can trigger an async error inside a Bun.serve streaming
* handler (e.g. a ReadableStream `start(controller)` callback hitting an
* unexpected upstream response shape). Without a handler, Bun's default
* behaviour prints the raw error — shown as `(function (controller, error)
* {"use strict"; ... TypeError: null is not an object` — and can tear down
* the whole proxy, killing every other in-flight Codex session.
*
* We must NOT let one bad stream crash the daemon. These handlers:
* 1. Append the full error + stack to `<configDir>/crash.log` so the exact
* fault (with the JSC `(evaluating 'x.y')` clause and file:line) is
* captured for a precise root-cause fix.
* 2. Keep the process alive — the failed request is already isolated by
* Bun.serve; surviving is strictly better than terminating.
*/
let installed = false;
function crashLogPath(): string {
const dir = getConfigDir();
try {
recordOwnedConfigPath(dir, join(dir, "crash.log"));
mkdirSync(dir, { recursive: true });
} catch (error) {
// During uninstall the log is skipped (both callers swallow this) rather than recreated.
if (error instanceof ConfigUninstallInProgressError) throw error;
/* best-effort: directory usually already exists */
}
return join(dir, "crash.log");
}
export function formatCrashEntry(kind: string, err: unknown, promise?: unknown): string {
const ts = new Date().toISOString();
const detail =
err instanceof Error
? `${err.name}: ${redactDiagnosticText(err.message)}\n${redactDiagnosticText(err.stack ?? "(no stack)")}`
: typeof err === "object"
? redactDiagnosticText(safeStringify(err))
: redactDiagnosticText(String(err));
return `\n[${ts}] ${kind}\n${detail}${diagnose(err)}${diagnosePromise(promise)}${breadcrumb()}\n`;
}
/**
* Bun surfaces some request-time stream/abort errors with only native frames
* (`at <anonymous> (native:1:11)`), so `err.stack` alone cannot locate the
* fault. JSC still records the true throw site on hidden own properties
* (`sourceURL` / `originalLine` / `originalColumn`) and `Bun.inspect` renders a
* code snippet from them — capture both so the next occurrence is pinpointable.
*/
function diagnose(err: unknown): string {
const lines: string[] = [];
try {
const ctor = (err as { constructor?: { name?: string } } | null)?.constructor?.name;
if (ctor && ctor !== "Error" && ctor !== "Object") lines.push(` ctor: ${ctor}`);
if (err && typeof err === "object") {
const e = err as Record<string, unknown>;
const cause = e.cause;
if (cause !== undefined) {
lines.push(` cause: ${redactDiagnosticText(cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause))}`);
}
if (e.code !== undefined) lines.push(` code: ${redactDiagnosticText(String(e.code))}`);
// JSC hidden throw-site fields survive even when the stack is native-only.
const sourceURL = e.sourceURL;
const line = e.line ?? e.originalLine;
const column = e.column ?? e.originalColumn;
if (typeof sourceURL === "string" && sourceURL) {
lines.push(` origin: ${redactUrlForLog(sourceURL)}${line !== undefined ? `:${String(line)}` : ""}${column !== undefined ? `:${String(column)}` : ""}`);
}
}
const stack = err instanceof Error ? err.stack ?? "" : "";
const hasUsableStack = /\((?!native:)[^)]*:\d+:\d+\)/.test(stack);
if (!hasUsableStack) {
const snippet = inspectErr(err);
if (snippet) lines.push(` inspect:\n${snippet.split("\n").map(l => ` ${l}`).join("\n")}`);
}
} catch {
/* diagnosis must never throw */
}
return lines.length ? `\n${lines.join("\n")}` : "";
}
/**
* Bun.inspect renders the JSC source snippet (with the offending line + caret)
* for errors whose throw site is otherwise lost to native frames.
*/
function inspectErr(err: unknown): string {
try {
const bun = (globalThis as { Bun?: { inspect?: (v: unknown, o?: unknown) => string } }).Bun;
if (!bun?.inspect) return "";
return redactDiagnosticText(bun.inspect(err, { depth: 2 }).trim());
} catch {
return "";
}
}
/**
* Inspect the rejected promise itself. Bun sometimes attaches richer context to the promise object
* than to the reason, and the rendered form helps distinguish a fetch/stream teardown from app code.
*/
function diagnosePromise(promise: unknown): string {
if (promise === undefined) return "";
try {
const bun = (globalThis as { Bun?: { inspect?: (v: unknown, o?: unknown) => string } }).Bun;
const rendered = bun?.inspect ? bun.inspect(promise, { depth: 1 }).trim() : String(promise);
if (!rendered || rendered === "Promise { <rejected> }") return "";
return `\n promise: ${redactDiagnosticText(rendered.split("\n").join(" "))}`;
} catch {
return "";
}
}
/**
* Record whether a sidecar (web-search / vision) was in flight when the fault fired. A native-only
* rejection coinciding with sidecar work is the prime suspect; this turns the correlation into a
* logged fact instead of an inference.
*/
function breadcrumb(): string {
try {
const lines: string[] = [];
const b = sidecarBreadcrumb();
if (b.inFlight > 0 || b.lastLabel) {
lines.push(` sidecar: inFlight=${b.inFlight} last=${b.lastLabel || "-"} sinceMs=${b.sinceMs}`);
}
const a = activityBreadcrumb();
if (a.note) lines.push(` activity: ${a.note} sinceMs=${a.sinceMs}`);
const fetches = recentFetches();
if (fetches) lines.push(fetches);
return lines.length ? `\n${lines.join("\n")}` : "";
} catch {
return "";
}
}
function safeStringify(value: unknown): string {
try {
return JSON.stringify(value, null, 2) ?? String(value);
} catch {
return String(value);
}
}
let benignSuppressed = 0;
let benignLastLoggedAt = 0;
let benignLastOrigin: string | undefined;
const BENIGN_LOG_INTERVAL_MS = 5 * 60_000;
const MAX_BENIGN_ORIGIN_BYTES = 1024;
/**
* Bun raises an off-path `unhandledRejection: TypeError: null is not an object` (native-only stack)
* whenever a streaming `fetch(..., { signal })` response body is torn down by an abort before/while
* we read it — turn supersede, client disconnect, upstream RST. The daemon is never at risk (the
* failed request is already isolated), the throw has no JS source location, and call-site body
* cancellation cannot fully close the runtime-internal window. Treat this exact shape as benign:
* keep the process alive, drop the alarmist banner, and fold repeats into a rate-limited summary so
* crash.log stays readable for genuinely novel faults.
*
* Second known shape (260712): `TypeError: Invalid state: ReadableStream is locked`
* (code ERR_INVALID_STATE, native-only stack ending in onSinkClose*). When a client
* disconnects mid-SSE on the tee()'d passthrough path (responses.ts Bun#32111
* workaround), Bun's sink-close teardown tries to cancel the tee-locked source body
* and rejects off-path. Request lifecycle is already settled at that point; same
* benign handling applies. The tee path remains the DEFAULT passthrough shape;
* the gated eager relay (relay-eager.ts, #314) does not tee and may not produce
* this shape — detection stays unchanged either way.
*/
export function isBenignAbortTeardown(err: unknown): boolean {
if (!(err instanceof TypeError)) return false;
const bareNullTeardown = err.message === "null is not an object"; // bare form only (no `(evaluating …)`)
const lockedStreamTeardown = err.message === "Invalid state: ReadableStream is locked"
&& (err as { code?: unknown }).code === "ERR_INVALID_STATE";
if (!bareNullTeardown && !lockedStreamTeardown) return false;
// Native-only: no JS source frame, parenthesized or not. Hidden JSC source fields
// (sourceURL/line/column) do not decide this: Bun can attach them to errors raised from
// its own builtin frames, and the benign summary still records them through diagnose().
return !hasJsSourceFrame(err.stack ?? "");
}
/**
* True when a stack line is an `at …` frame ending in `line:col` (optionally inside
* parentheses) whose location is not a Bun builtin (`native:`). Covers `at fn (/abs/x.ts:1:2)`,
* `at /abs/x.ts:1:2`, `at async fn (file:///x.ts:1:2)` and Windows drive paths.
*/
function hasJsSourceFrame(stack: string): boolean {
return stack.split(/\r?\n/).some(raw => {
const frame = raw.trim();
return frame.startsWith("at ")
&& /:\d+:\d+\)?$/.test(frame)
&& !/[(\s]native:\d+:\d+\)?$/.test(frame);
});
}
/**
* The JSC hidden throw site (`sourceURL:line:col`), when the error carries one. Best-effort:
* this runs inside the process crash handler, so an accessor that throws yields no site.
*/
function hiddenThrowSite(err: unknown): string | undefined {
try {
if (!err || typeof err !== "object") return undefined;
const e = err as Record<string, unknown>;
if (typeof e.sourceURL !== "string" || !e.sourceURL) return undefined;
const site = `${e.sourceURL}:${String(e.line ?? e.originalLine ?? "")}:${String(e.column ?? e.originalColumn ?? "")}`;
return truncateRetainedUtf8(site, MAX_BENIGN_ORIGIN_BYTES);
} catch {
return undefined;
}
}
function record(kind: string, err: unknown, promise?: unknown): void {
if (kind === "unhandledRejection" && isBenignAbortTeardown(err)) {
benignSuppressed++;
const now = Date.now();
// A throw site JSC recorded on hidden fields is new information when it differs from
// the last one logged, so it is written even inside the fold window; repeats still fold.
const origin = hiddenThrowSite(err);
const novelOrigin = origin !== undefined && origin !== benignLastOrigin;
if (!novelOrigin && now - benignLastLoggedAt < BENIGN_LOG_INTERVAL_MS) return; // fold repeats silently
benignLastLoggedAt = now;
if (origin !== undefined) benignLastOrigin = origin;
const summary = `\n[${new Date(now).toISOString()}] benign-abort-teardown x${benignSuppressed}`
+ ` (Bun fetch-body abort; proxy unaffected)${diagnose(err)}${diagnosePromise(promise)}${breadcrumb()}\n`;
benignSuppressed = 0;
try { appendFileSync(crashLogPath(), summary); } catch { /* logging must never throw */ }
return; // no stderr banner — this is expected noise, not a crash
}
const line = formatCrashEntry(kind, err, promise);
// Always surface to stderr so foreground `ocx start` users still see it,
// then persist for later diagnosis.
console.error(`⚠️ ${kind} (proxy stayed up; logged to crash.log)`);
console.error(line.trimStart());
try {
appendFileSync(crashLogPath(), line);
} catch {
/* logging must never throw */
}
}
interface FetchTrace { url: string; at: number; origin: string; settled: boolean; rejected?: string }
const FETCH_RING_MAX = 12;
const MAX_DIAGNOSTIC_VALUE_BYTES = 8 * 1024;
const fetchRing: FetchTrace[] = [];
let fetchInstrumented = false;
let fetchRingBytes = 0;
function fetchTraceBytes(trace: FetchTrace): number {
return retainedUtf8Bytes(trace.url) + retainedUtf8Bytes(trace.origin) + retainedUtf8Bytes(trace.rejected ?? "");
}
function removeOldestFetchTrace(): number {
const trace = fetchRing.shift();
if (!trace) return 0;
const bytes = fetchTraceBytes(trace);
fetchRingBytes -= bytes;
return bytes;
}
function setFetchTraceRejected(trace: FetchTrace, value: string): void {
const retained = fetchRing.includes(trace);
if (retained) fetchRingBytes -= retainedUtf8Bytes(trace.rejected ?? "");
trace.rejected = truncateRetainedUtf8(value, MAX_DIAGNOSTIC_VALUE_BYTES);
if (retained) fetchRingBytes += retainedUtf8Bytes(trace.rejected);
}
function appendFetchTrace(url: string, origin: string): FetchTrace {
const trace: FetchTrace = {
url: truncateRetainedUtf8(redactUrlForLog(url), MAX_DIAGNOSTIC_VALUE_BYTES),
at: Date.now(),
origin: truncateRetainedUtf8(origin, MAX_DIAGNOSTIC_VALUE_BYTES),
settled: false,
};
fetchRing.push(trace);
fetchRingBytes += fetchTraceBytes(trace);
while (fetchRing.length > FETCH_RING_MAX) removeOldestFetchTrace();
return trace;
}
/**
* The recurring native-only rejection carries no source location, and every JS `await fetch(...)`
* is already try/caught — so the offending promise is created INSIDE Bun's fetch and rejects off the
* awaited path. Wrap global fetch to record each call's origin (a JS stack captured at call time) and
* whether it later rejected. crash-guard then dumps the still-pending / recently-rejected fetches so
* the next fault names the exact call site Bun lost.
*/
function instrumentFetch(): void {
if (fetchInstrumented) return;
const g = globalThis as { fetch?: typeof fetch };
const original = g.fetch;
if (typeof original !== "function") return;
fetchInstrumented = true;
g.fetch = function instrumentedFetch(this: unknown, ...args: Parameters<typeof fetch>): ReturnType<typeof fetch> {
let url = "";
try {
const input = args[0];
url = typeof input === "string" ? input : input instanceof URL ? input.href : (input as Request)?.url ?? "";
} catch { /* best-effort */ }
const origin = (new Error().stack ?? "").split("\n").slice(2, 5).map(l => l.trim()).join(" <- ");
const trace = appendFetchTrace(url, origin);
enforceAppOwnedMemoryBudget();
let p: ReturnType<typeof fetch>;
try {
p = original.apply(this, args);
} catch (e) {
trace.settled = true;
setFetchTraceRejected(trace, redactDiagnosticText(e instanceof Error ? `${e.name}: ${e.message}` : String(e)));
throw e;
}
return p.then(
r => { trace.settled = true; return r; },
e => { trace.settled = true; setFetchTraceRejected(trace, redactDiagnosticText(e instanceof Error ? `${e.name}: ${e.message}` : String(e))); throw e; },
);
} as typeof fetch;
}
export function crashRingMetrics(): { entries: number; bytes: number; oldestAt: number | null } {
return { entries: fetchRing.length, bytes: fetchRingBytes, oldestAt: fetchRing[0]?.at ?? null };
}
export function evictOldestCrashTraceForBudget(): number {
return removeOldestFetchTrace();
}
export function appendCrashTraceForTests(url: string, origin: string, rejected?: string): void {
const trace = appendFetchTrace(url, origin);
if (rejected !== undefined) {
trace.settled = true;
setFetchTraceRejected(trace, rejected);
}
}
export function crashRingEntriesForTests(): readonly Readonly<FetchTrace>[] {
return fetchRing.map(trace => ({ ...trace }));
}
export function resetCrashRingForTests(): void {
fetchRing.length = 0;
fetchRingBytes = 0;
}
export function recordCrashForTests(kind: string, err: unknown): void {
record(kind, err);
}
export function resetBenignFoldForTests(): void {
benignSuppressed = 0;
benignLastLoggedAt = 0;
benignLastOrigin = undefined;
}
/** Render the recent fetch ring (pending first) for the crash breadcrumb. */
function recentFetches(): string {
try {
if (fetchRing.length === 0) return "";
const now = Date.now();
const rows = fetchRing.slice(-6).map(f => {
const state = !f.settled ? "PENDING" : f.rejected ? `REJECTED(${f.rejected})` : "ok";
return ` [${state}] ${f.url} ageMs=${now - f.at}${!f.settled ? ` origin=${f.origin}` : ""}`;
});
return ` fetches:\n${rows.join("\n")}`;
} catch {
return "";
}
}
function redactDiagnosticText(value: string): string {
return redactSecretString(value);
}
/**
* Register global handlers that keep the proxy alive and capture full stacks.
* Idempotent: safe to call more than once.
*/
export function installCrashGuards(): void {
if (installed) return;
installed = true;
instrumentFetch();
process.on("unhandledRejection", (reason, promise) => {
record("unhandledRejection", reason, promise);
});
process.on("uncaughtException", err => {
record("uncaughtException", err);
});
}