@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
475 lines (422 loc) • 23.6 kB
text/typescript
import { createHmac, randomBytes } from "node:crypto";
import { chmodSync, closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs";
import { join } from "node:path";
import { isMissingPathError } from "../config/atomic-write";
import { linkDir } from "./paths";
/**
* Durable activity holds for Remote Link WebSockets (design T4 section 5).
*
* A link socket that closes while busy, for any reason except revocation, leaves a hold: the HMAC of
* its own thread id, the hashed result keys it was waiting for, one running mark per turn that was
* running, and whether its results were untracked. A hold counts as link activity, so the Home's own
* automatic and activity-gated Child restarts withhold while it exists. It is a liveness aid against
* that automation, not an authorization boundary against the Child's own processes.
*
* Crash fence: before the first link upgrade of a process is admitted, ws-run-state.json is written
* durably as "running"; a graceful stop writes the holds, then "clean". A start that finds "running"
* or an unreadable fence turns every link into an unknown hold, because work may have been in flight
* when the process died. Nothing here reads or writes a file until a fence exists or is armed.
*/
export type UnknownReason = "crash" | "unreadable" | "overflow" | "write-failed" | "key-rotated" | "key-unavailable";
/** Why a hold keeps a link in flight (the operator's view, D8). */
export type HoldReason = "pending-results" | "running-turn" | "running-unknown" | "untracked" | "no-thread";
/** One hold as the operator sees it: reasons and counts only, no hash, key or id. */
export interface HoldSummary { since: number; reasons: HoldReason[]; pendingResults: number; runningTurns: number; runningUnknown: number }
export interface LinkHoldsSummary { holds: HoldSummary[]; unknown: UnknownReason | null }
export type ClearHoldsResult = { ok: true; cleared: LinkHoldsSummary | null } | { ok: false; reason: "write-failed" | "links-unreadable" };
export interface HoldInput {
/** HMAC of the socket's own thread id; null when it had none (operator only). */
threadHash: string | null;
/** Hashed result keys the socket was waiting for (CodexWebSocketActivity). */
pendingKeys: readonly string[];
/** Running turns whose cut request could not be fingerprinted: unknown, operator only. */
runningMarks: number;
/** Running turns whose cut request was fingerprinted after expansion; each is released on its own. */
marks?: readonly FingerprintMark[];
untracked: boolean;
}
/** The cut request of a running turn: its expanded-input prefix length, HMAC, and turn reference. */
export interface FingerprintMark { n: number; inputHmac: string; turnRef: string }
/** A stored mark: its own random id, so a release removes exactly the marks a request saw, never an identical later one. */
export interface StoredMark extends FingerprintMark { id: string }
interface Hold {
threadHash: string | null;
pendingKeys: string[];
runningMarks: number;
marks: StoredMark[];
untracked: boolean;
createdAt: number;
/** In memory only: creation order in this process, so a request settles only holds older than itself. */
seq: number;
}
/** Canonical JSON: object keys sorted, no whitespace; throws for anything JSON cannot represent. */
function canonical(value: unknown): string {
const seen = new Set<unknown>();
const sort = (input: unknown): unknown => {
if (input === null || typeof input === "string" || typeof input === "boolean") return input;
if (typeof input === "number") { if (!Number.isFinite(input)) throw new Error("non-finite number"); return input; }
if (typeof input !== "object") throw new Error("not representable");
if (seen.has(input)) throw new Error("cycle");
seen.add(input);
const out = Array.isArray(input) ? input.map(sort)
: Object.fromEntries(Object.keys(input as Record<string, unknown>).sort()
.filter(key => (input as Record<string, unknown>)[key] !== undefined)
.map(key => [key, sort((input as Record<string, unknown>)[key])]));
seen.delete(input);
return out;
};
return JSON.stringify(sort(value));
}
const isMark = (value: unknown): value is StoredMark => {
const mark = value as Record<string, unknown> | null;
return !!mark && Number.isSafeInteger(mark.n) && (mark.n as number) > 0 && typeof mark.inputHmac === "string" && typeof mark.turnRef === "string"
&& typeof mark.id === "string" && mark.id.length > 0;
};
interface LinkHolds { holds: Hold[]; unknown: UnknownReason | null }
interface HoldsFile { version: 1; keyId: string | null; links: Record<string, LinkHolds> }
export interface ActivityHoldsDeps {
/** Directory of the files; defaults to <configDir>/link. */
dir?: string;
/** Link ids in links.json, read when crash evidence must become unknown holds; null when unreadable. */
linkIds: () => readonly string[] | null;
now?: () => number;
/**
* Holds per link and in total before a link becomes unknown. Required: the owner of every link
* WebSocket cap is src/server/index/link-websocket-limits.ts, and src/link may not import it.
*/
limits: { holdsPerLink: number; holdsTotal: number };
/** Test seam: inject a failure or a crash at each durability point of a write. */
writeHooks?: HoldWriteHooks;
}
export interface HoldWriteHooks {
/** Test seam: runs after the temporary file is opened, before its content is written. */
afterOpen?: (temp: string) => void;
beforeRename?: (temp: string, target: string) => void;
afterRename?: (target: string) => void;
/** Test seam: runs after the rename, where the directory fsync follows on Linux and macOS (every platform). */
syncDir?: (dir: string) => void;
}
const FENCE = "ws-run-state.json";
const HOLDS = "activity-holds.json";
const KEY = "activity-ref-key";
function isLinkHolds(value: unknown): value is LinkHolds {
if (!value || typeof value !== "object") return false;
const entry = value as Record<string, unknown>;
const reasons = [null, "crash", "unreadable", "overflow", "write-failed", "key-rotated", "key-unavailable"];
return Array.isArray(entry.holds) && reasons.includes(entry.unknown as string | null) && entry.holds.every(hold => {
const h = hold as Record<string, unknown>;
return h && (h.threadHash === null || typeof h.threadHash === "string") && Array.isArray(h.pendingKeys)
&& h.pendingKeys.every(key => typeof key === "string") && Number.isSafeInteger(h.runningMarks) && (h.runningMarks as number) >= 0
&& typeof h.untracked === "boolean" && typeof h.createdAt === "number"
&& (h.marks === undefined || (Array.isArray(h.marks) && h.marks.every(isMark)));
});
}
export function createActivityHolds(deps: ActivityHoldsDeps) {
const dir = deps.dir ?? linkDir();
const now = deps.now ?? Date.now;
const links = new Map<string, LinkHolds>();
let armed = false;
let key: Buffer | null = null;
// Set when crash evidence could not be tied to the links (links.json unreadable): every link,
// known or not, then counts as unknown, and the fence is left running for the next start.
let allUnknown: UnknownReason | null = null;
let holdSeq = 0;
const hooks = deps.writeHooks ?? {};
/**
* Strict durable replace: a private temp file, content fsync, rename, then a directory fsync whose
* failure is reported, because a fence or a hold that is not durable must not be treated as one.
* Windows has no directory descriptor to sync; NTFS journals the rename itself.
*/
const durableWrite = (name: string, content: string): void => {
mkdirSync(dir, { recursive: true, mode: 0o700 });
const target = join(dir, name);
const temp = target + "." + process.pid + "." + randomBytes(6).toString("hex") + ".tmp";
const descriptor = openSync(temp, "wx", 0o600);
try {
try { hooks.afterOpen?.(temp); writeSync(descriptor, content); fsyncSync(descriptor); } finally { closeSync(descriptor); }
} catch (error) {
// A failure before the rename must not leave the temporary file behind.
try { unlinkSync(temp); } catch { /* reported through the original error */ }
throw error;
}
try {
hooks.beforeRename?.(temp, target);
renameSync(temp, target);
} catch (error) {
try { unlinkSync(temp); } catch { /* reported through the original error */ }
throw error;
}
hooks.afterRename?.(target);
if (process.platform !== "win32") chmodSync(target, 0o600);
// The seam sits at the post-rename durability point on every platform, so the callers' handling
// of a rename that landed but is reported as not durable runs under test on Windows too.
hooks.syncDir?.(dir);
if (process.platform !== "win32") {
const directory = openSync(dir, "r");
try { fsyncSync(directory); } finally { closeSync(directory); }
}
};
const write = (name: string, value: unknown): boolean => {
try { durableWrite(name, JSON.stringify(value) + "\n"); return true; } catch { return false; }
};
const readRaw = (name: string): { missing: true } | { missing: false; text: string | null } => {
try { return { missing: false, text: readFileSync(join(dir, name), "utf8") }; }
catch (error) { return isMissingPathError(error) ? { missing: true } : { missing: false, text: null }; }
};
/** The private reference key; created on first use. Its id travels in the holds file. */
const refKey = (): Buffer => {
if (key) return key;
const existing = readRaw(KEY);
if (!existing.missing && existing.text && /^[a-f0-9]{64}$/.test(existing.text.trim())) {
key = Buffer.from(existing.text.trim(), "hex");
return key;
}
const fresh = randomBytes(32);
durableWrite(KEY, fresh.toString("hex") + "\n");
key = fresh;
return key;
};
const keyId = (): string => createHmac("sha256", refKey()).update("opencodex-activity-key-id").digest("hex").slice(0, 16);
const entry = (linkId: string): LinkHolds => {
let value = links.get(linkId);
if (!value) { value = { holds: [], unknown: null }; links.set(linkId, value); }
return value;
};
/** Marks every link unknown; false when the links could not be listed (allUnknown covers them). */
const markAll = (reason: UnknownReason): boolean => {
let ids: readonly string[] | null = null;
try { ids = deps.linkIds(); } catch { ids = null; }
if (!ids) { allUnknown ??= reason; return false; }
for (const linkId of ids) entry(linkId).unknown ??= reason;
return true;
};
const total = (): number => [...links.values()].reduce((sum, value) => sum + value.holds.length, 0);
/** Writes every link's holds; `without` leaves one link out (the operator's clear writes before it forgets). */
const persist = (without?: string): boolean => {
let id: string | null = null;
try { id = key ? keyId() : null; } catch { return false; }
const out: HoldsFile = { version: 1, keyId: id, links: {} };
for (const [linkId, value] of links) {
if (linkId !== without && (value.holds.length || value.unknown)) out.links[linkId] = { ...value, holds: value.holds.map(({ seq: _seq, ...hold }) => hold as Hold) };
}
return write(HOLDS, out);
};
/** A failed hold write keeps the holds in memory and makes the link unknown (fail closed). */
const persistOrMark = (linkId: string): void => {
if (!persist()) entry(linkId).unknown ??= "write-failed";
};
const finished = (hold: Hold): boolean =>
hold.threadHash !== null && !hold.untracked && hold.runningMarks === 0 && hold.marks.length === 0 && hold.pendingKeys.length === 0;
return {
/** At Home start. Reads only when a fence file exists. */
start(): void {
const fence = readRaw(FENCE);
if (fence.missing) return;
let state: unknown = null;
try { state = fence.text === null ? null : (JSON.parse(fence.text) as { state?: unknown }).state; } catch { state = null; }
const held = readRaw(HOLDS);
let file: HoldsFile | null = null;
if (!held.missing) {
try {
const parsed = JSON.parse(held.text ?? "") as HoldsFile;
if (parsed?.version === 1 && parsed.links && typeof parsed.links === "object"
&& Object.values(parsed.links).every(isLinkHolds)) file = parsed;
} catch { file = null; }
if (!file) markAll("unreadable");
}
let keyUnavailable = false;
if (file && file.keyId !== null) {
try { keyId(); } catch { keyUnavailable = true; }
}
if (file && keyUnavailable) {
// Hashes cannot be checked or made: keyed holds and every link stay unknown, nothing is cleaned.
for (const [linkId, value] of Object.entries(file.links)) {
links.set(linkId, { holds: value.holds.filter(hold => hold.threadHash === null)
.map(hold => ({ ...hold, pendingKeys: [...hold.pendingKeys], marks: [...(hold.marks ?? [])], seq: ++holdSeq })), unknown: value.unknown ?? "key-unavailable" });
}
markAll("key-unavailable");
if (state !== "clean") markAll("crash");
return;
}
if (file) {
const rotated = file.keyId !== null && file.keyId !== keyId();
for (const [linkId, value] of Object.entries(file.links)) {
const kept = rotated ? value.holds.filter(hold => hold.threadHash === null) : value.holds;
links.set(linkId, { holds: kept.map(hold => ({ ...hold, pendingKeys: [...hold.pendingKeys], marks: [...(hold.marks ?? [])], seq: ++holdSeq })),
unknown: value.unknown ?? (rotated && kept.length < value.holds.length ? "key-rotated" : null) });
}
}
if (state !== "clean") {
// The crash evidence becomes holds; only once that is durable, for links that could be listed,
// may the fence say clean. Otherwise it stays running and the next start does this again.
if (markAll("crash") && persist()) write(FENCE, { version: 1, state: "clean" });
}
},
/** Before the first link upgrade of this process. False refuses every link upgrade. */
arm(): boolean {
if (armed) return true;
armed = write(FENCE, { version: 1, state: "running" });
return armed;
},
get armed(): boolean { return armed; },
threadHash(threadId: string): string {
return createHmac("sha256", refKey()).update("opencodex-link-thread-v1\0" + threadId).digest("hex");
},
add(linkId: string, input: HoldInput): void {
const value = entry(linkId);
if (value.holds.length >= deps.limits.holdsPerLink || total() >= deps.limits.holdsTotal) {
value.unknown ??= "overflow";
} else {
// Fingerprints need the thread to be matched by; without it they are operator-only, like unknown marks.
const marks = input.threadHash === null ? [] : (input.marks ?? []).map(mark => ({ n: mark.n, inputHmac: mark.inputHmac, turnRef: mark.turnRef, id: randomBytes(12).toString("hex") }));
const unknown = input.runningMarks + (input.threadHash === null ? input.marks?.length ?? 0 : 0);
value.holds.push({ threadHash: input.threadHash, pendingKeys: [...input.pendingKeys], runningMarks: unknown, marks, untracked: input.untracked, createdAt: now(), seq: ++holdSeq });
}
persistOrMark(linkId);
},
/** The current hold sequence: a request records it when it begins and passes it to observe. */
sequence(): number {
return holdSeq;
},
/**
* A later request from the same link and own thread returned these results. Only holds that existed
* when the request began (seq at most beganAt) are settled: a hold created later, by another socket,
* may wait for a new call that reuses the same id, and keeps it.
*/
observe(linkId: string, threadHash: string | null, resultKeys: readonly string[], beganAt: number): void {
const value = links.get(linkId);
if (!value || threadHash === null || resultKeys.length === 0) return;
const returned = new Set(resultKeys);
let changed = false;
for (const hold of value.holds) {
if (hold.threadHash !== threadHash || hold.seq > beganAt) continue;
const left = hold.pendingKeys.filter(keyHash => !returned.has(keyHash));
if (left.length !== hold.pendingKeys.length) { hold.pendingKeys = left; changed = true; }
}
if (!changed) return;
value.holds = value.holds.filter(hold => !finished(hold));
persistOrMark(linkId);
},
/** HMAC of a turn's expanded input items (canonical JSON). Throws when they cannot be canonicalized. */
inputHmac(items: readonly unknown[]): string {
return createHmac("sha256", refKey()).update("opencodex-link-input-v1\0" + canonical(items)).digest("hex");
},
/** HMAC of Codex's own turn id (client_metadata.turn_id). */
turnRef(turnId: string): string {
return createHmac("sha256", refKey()).update("opencodex-link-turn-v1\0" + turnId).digest("hex");
},
/**
* Request-log refs (design 9.1): a domain of their own per kind, so they never join with hold hashes or marks,
* and the link id inside, so one id gives unrelated refs on two links (the metrics join rows of one link only).
*/
logRef(kind: "thread" | "turn", linkId: string, id: string): string {
return createHmac("sha256", refKey()).update("opencodex-link-log-" + kind + "-v2\0" + linkId + "\0" + id).digest("hex");
},
/** Whether any hold on this link has a fingerprinted running mark: callers skip hashing otherwise. */
hasMarks(linkId: string): boolean {
return links.get(linkId)?.holds.some(hold => hold.marks.length > 0) ?? false;
},
/** Fingerprinted marks a request of this own thread and turn could release. */
markCandidates(linkId: string, threadHash: string, turnRef: string): StoredMark[] {
return (links.get(linkId)?.holds ?? []).filter(hold => hold.threadHash === threadHash)
.flatMap(hold => hold.marks.filter(mark => mark.turnRef === turnRef));
},
/** Rule (d): an upstream-completed inference request of the same thread and turn re-sent these, by id. */
releaseMarks(linkId: string, threadHash: string, released: readonly StoredMark[]): void {
const value = links.get(linkId);
if (!value || released.length === 0) return;
const ids = new Set(released.map(mark => mark.id));
let changed = false;
for (const hold of value.holds) {
if (hold.threadHash !== threadHash) continue;
const left = hold.marks.filter(mark => !ids.has(mark.id));
if (left.length !== hold.marks.length) { hold.marks = left; changed = true; }
}
if (!changed) return;
value.holds = value.holds.filter(hold => !finished(hold));
persistOrMark(linkId);
},
/** Running marks on a link: fingerprinted and unknown (operator only). */
markCounts(linkId: string): { fingerprinted: number; unknown: number } {
const holds = links.get(linkId)?.holds ?? [];
return { fingerprinted: holds.reduce((sum, hold) => sum + hold.marks.length, 0), unknown: holds.reduce((sum, hold) => sum + hold.runningMarks, 0) };
},
/** Whether a request on this link could release anything: callers skip parsing otherwise. */
releasable(linkId: string): boolean {
return links.get(linkId)?.holds.some(hold => hold.threadHash !== null && hold.pendingKeys.length > 0) ?? false;
},
/** The operator's view of one link (D8); null when nothing holds it. */
summary(linkId: string): LinkHoldsSummary | null {
const value = links.get(linkId);
const unknown = value?.unknown ?? allUnknown;
if (!value?.holds.length && !unknown) return null;
return {
holds: (value?.holds ?? []).map(hold => {
const reasons: HoldReason[] = [];
if (hold.pendingKeys.length) reasons.push("pending-results");
if (hold.marks.length) reasons.push("running-turn");
if (hold.runningMarks) reasons.push("running-unknown");
if (hold.untracked) reasons.push("untracked");
if (hold.threadHash === null) reasons.push("no-thread");
return { since: hold.createdAt, reasons, pendingResults: hold.pendingKeys.length, runningTurns: hold.marks.length, runningUnknown: hold.runningMarks };
}),
unknown,
};
},
/**
* The operator's override (D8): forgets every hold and the unknown state of one link, durably. The
* file is written without the link first; memory changes only once every write succeeded. An
* all-links unknown state (links unreadable at start) is first written per link, which needs the
* links to be listable; until then nothing is cleared.
*
* A failed write may already have replaced the file (a directory sync fails after the rename), so
* the fence carries the uncertainty: in a process that has not armed it, the clear sets it running
* first. The durable holds write is the commit point: if it fails, the fence stays running (the
* next start keeps every link unknown) and nothing is cleared; once it succeeded, the clear is
* committed and the fence is set clean. That clean write may fail before its rename (running: every
* link unknown at the next start, fail closed) or after it (clean: the committed clear); neither
* contradicts the answer. An armed process keeps the fence running until its graceful stop.
*/
clear(linkId: string): ClearHoldsResult {
const cleared = this.summary(linkId);
if (!cleared) return { ok: true, cleared: null };
if (allUnknown) {
if (!markAll(allUnknown)) return { ok: false, reason: "links-unreadable" };
allUnknown = null;
}
const fenced = !armed;
if (fenced && !write(FENCE, { version: 1, state: "running" })) return { ok: false, reason: "write-failed" };
if (!persist(linkId)) {
// Best effort: put the link back on disk. The fence is running whatever this write does.
persist();
return { ok: false, reason: "write-failed" };
}
links.delete(linkId);
// Committed. A failed clean write leaves the fence running at worst, which only widens the next start's unknown set.
if (fenced) write(FENCE, { version: 1, state: "clean" });
return { ok: true, cleared };
},
removeLink(linkId: string): void {
if (links.delete(linkId)) persist();
},
/** Activity the restart gates see: one per hold, one more for an unknown link. */
count(linkId: string): number {
const value = links.get(linkId);
return (value?.holds.length ?? 0) + (value?.unknown || allUnknown ? 1 : 0);
},
describe(linkId: string): { holds: number; unknown: UnknownReason | null } {
const value = links.get(linkId);
return { holds: value?.holds.length ?? 0, unknown: value?.unknown ?? allUnknown };
},
/** Graceful stop, after the link sockets closed: holds first, then clean. */
stop(): boolean {
if (!armed) return true;
// Unknown state that exists only in memory (links unreadable at start) is written per link once
// the links can be listed; until then the fence is never cleaned.
if (allUnknown && markAll(allUnknown)) allUnknown = null;
if (!persist() || allUnknown) return false;
return write(FENCE, { version: 1, state: "clean" });
},
};
}
export type ActivityHolds = ReturnType<typeof createActivityHolds>;