@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
495 lines (457 loc) • 25.9 kB
JavaScript
// Plain ESM with no Bun or TypeScript dependency; its one caller is the Home's startup step,
// src/link/child-binary-cache-start.ts. Types: child-binary-cache-fill.d.mts.
import { createHash, randomBytes } from "node:crypto";
import { closeSync, fsyncSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { detectInstallFromPath } from "../update/install-detection.mjs";
/**
* Fills $OPENCODEX_HOME/ocx-child-builds/<version>/ when an updated Home of that version starts (lane
* O2 of the disposable-machines design), so a packaged Home can plant ocx-child on a bare machine
* after an update by any installer.
*
* Trust is the update's own: the expected digests come from CHILD_RELEASE_DIGESTS_PATH inside the
* running package tree, which its installer verified, and each binary, fetched from the GitHub
* release of exactly that version, is cached only when its SHA-256 equals that digest. Nothing is
* fetched without that file, from a source checkout, on Windows, or without evidence of an upgrade:
* ocx-child-builds/.versions/ holds one marker per version, "<version>.filled" when its fill completed
* and "<version>.baseline" when it ran and fetched nothing (a first start, or a package without
* digests, silently, so the first release that ships them sees an upgrade); markers are only added,
* so the newest can never go down, and only a running version newer than it fills. A first start (no record: a fresh install) records
* its version and fetches nothing, a restart of the same version fetches nothing, and a record or
* cache directory for a newer version means a downgrade. A listed build that cannot be verified loses its manifest line,
* so it can never stay trusted under an older manifest; the fill deletes no build file (unlisted, the reader
* refuses it); the record stays behind until every listed build is verified, so the next start retries.
* A digest file that is there but unreadable or unusable is not an absent one: nothing is written and
* the record stays, so a repaired package still fills. Overlapping fills cannot lose a build: the
* manifest is rebuilt from the builds verified on disk when it is written, and a completed record
* whose manifest no longer covers the package's digests is repaired on the next start by re-listing
* the builds still verified on disk; a same-version start never downloads. ocx-child-builds/.fill.lock (reclaimed once its
* process is gone or after an hour) keeps concurrent starts from downloading twice; it is not the guarantee.
*
* The directory is OpenCodex's own (uninstall removes it); an operator's hand-placed builds live in
* cache/ocx-child/<version>/, which the reader checks first and this module never reads or writes.
*
* Nobody else can substitute a build or forge a record: each of ocx-child-builds, .versions and the version
* directory must be a real directory (never a symlink; created 0700 when missing) owned by this user
* and writable by nobody else, and every directory above them must be owned by this user or root and
* writable by nobody else unless sticky. Every write re-checks the target directory's identity, so a
* directory swapped in during a download receives nothing. An existing build counts only when it
* passes the reader's file checks as well as the digest. Windows has no mode bits to establish that
* privacy, so there the fill does nothing and builds are placed by hand.
*/
/** Digests of the release's ocx-child builds, relative to the package root ("<sha256> ocx-child-<triple>[.exe]"). */
export const CHILD_RELEASE_DIGESTS_PATH = "native/ocx-child/SHA256SUMS";
export const CHILD_RELEASE_REPOSITORY = "lidge-jun/opencodex";
/** The build file names child-binary-source.ts plants from (childArtifactFilename of every target). */
export const CHILD_CACHE_ARTIFACT_FILES = Object.freeze([
"ocx-child-darwin-arm64", "ocx-child-darwin-x64", "ocx-child-linux-x64", "ocx-child-linux-arm64", "ocx-child-windows-x64.exe",
]);
const MAX_CHILD_BINARY_BYTES = 32 * 1024 * 1024;
const MAX_MANIFEST_BYTES = 256 * 1024;
const SHA256_HEX = /^[0-9a-f]{64}$/;
const VERSIONS_DIR = ".versions";
const POSIX = process.platform !== "win32";
const SEMVER = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/;
export function childReleaseAssetUrl(version, file) {
return "https://github.com/" + CHILD_RELEASE_REPOSITORY + "/releases/download/v" + encodeURIComponent(version) + "/" + encodeURIComponent(file);
}
/** The fill's own directory under OPENCODEX_HOME, listed as owned once created (never cache/, which is the operator's). */
export const CHILD_FILL_DIR = "ocx-child-builds";
export function childBinaryCacheDir(opencodexHome, version) {
return join(opencodexHome, CHILD_FILL_DIR, version);
}
function parseVersion(value) {
if (typeof value !== "string" || value.length > 96) return null;
const match = SEMVER.exec(value);
if (!match) return null;
return { core: [BigInt(match[1]), BigInt(match[2]), BigInt(match[3])], pre: match[4] === undefined ? [] : match[4].split(".") };
}
/** SemVer precedence (build metadata ignored): -1 older, 0 equal, 1 newer. */
export function compareReleaseVersions(left, right) {
const a = parseVersion(left), b = parseVersion(right);
if (!a || !b) return null;
for (let i = 0; i < 3; i++) if (a.core[i] !== b.core[i]) return a.core[i] > b.core[i] ? 1 : -1;
if (a.pre.length === 0 || b.pre.length === 0) return a.pre.length === b.pre.length ? 0 : a.pre.length === 0 ? 1 : -1;
for (let i = 0; i < Math.max(a.pre.length, b.pre.length); i++) {
const x = a.pre[i], y = b.pre[i];
if (x === y) continue;
if (x === undefined) return -1;
if (y === undefined) return 1;
const xn = /^\d+$/.test(x), yn = /^\d+$/.test(y);
if (xn && yn) return BigInt(x) > BigInt(y) ? 1 : -1;
if (xn !== yn) return xn ? -1 : 1;
return x > y ? 1 : -1;
}
return 0;
}
/** "<sha256> <file>" lines, the format child-binary-source.ts reads (parseSha256Sums). */
function parseManifest(text) {
const entries = new Map();
for (const raw of text.split(/\r?\n/)) {
const line = raw.trim();
if (!line || line.startsWith("#")) continue;
const match = /^([0-9A-Fa-f]{64})\s+\*?([^\s/\\]+)$/.exec(line);
if (match) entries.set(match[2], match[1].toLowerCase());
}
return entries;
}
function readRegular(path, maxBytes) {
let st;
try { st = lstatSync(path); } catch (error) { if (error?.code === "ENOENT") return null; throw error; }
if (!st.isFile() || st.isSymbolicLink() || st.size > maxBytes) throw new Error("not a regular file: " + path);
return readFileSync(path);
}
/** null: no digest file (every release so far); "invalid": there but unreadable or unusable; else the digests. */
function readDigests(packageRoot) {
let bytes;
try { bytes = readRegular(join(packageRoot, CHILD_RELEASE_DIGESTS_PATH), MAX_MANIFEST_BYTES); } catch { return "invalid"; }
if (bytes === null) return null;
const known = new Set(CHILD_CACHE_ARTIFACT_FILES);
const digests = new Map([...parseManifest(bytes.toString("utf8"))].filter(([file, sha]) => known.has(file) && SHA256_HEX.test(sha)));
return digests.size > 0 ? digests : "invalid";
}
function currentUid() {
return typeof process.getuid === "function" ? process.getuid() : null;
}
/** Every directory above the cache: owned by this user or root, writable by nobody else unless sticky (/tmp). */
function assertTrustedAncestors(home) {
if (!POSIX) return;
let dir = realpathSync(home);
for (;;) {
const st = lstatSync(dir);
const owner = st.uid === currentUid() || st.uid === 0;
const writers = (st.mode & 0o022) === 0 || (st.mode & 0o1000) !== 0;
if (!st.isDirectory() || !owner || !writers) throw new Error("unsafe cache directory: " + dir);
const parent = dirname(dir);
if (parent === dir) return;
dir = parent;
}
}
/** A real directory (never a symlink) private to this user, created 0700 when missing; returns its identity. */
function realDirectory(path) {
let st;
try { st = lstatSync(path); } catch (error) {
if (error?.code !== "ENOENT") throw error;
mkdirSync(path, { mode: 0o700 });
st = lstatSync(path);
}
if (!st.isDirectory() || st.isSymbolicLink()) throw new Error("unsafe cache directory: " + path);
if (POSIX && (st.uid !== currentUid() || (st.mode & 0o022) !== 0)) throw new Error("unsafe cache directory: " + path);
return { dev: st.dev, ino: st.ino };
}
function sameDirectory(path, identity) {
try {
const st = lstatSync(path);
return st.isDirectory() && !st.isSymbolicLink() && st.dev === identity.dev && st.ino === identity.ino;
} catch { return false; }
}
function sha256Hex(bytes) {
return createHash("sha256").update(bytes).digest("hex");
}
/** Exclusive private temp beside the target, then rename: a symlink at the name is replaced, never followed. */
function publish(dir, identity, name, bytes) {
if (!sameDirectory(dir, identity)) throw new Error("cache directory changed");
const temp = join(dir, ".ocx-child-cache." + process.pid + "." + randomBytes(6).toString("hex") + ".tmp");
const fd = openSync(temp, "wx", 0o600);
try {
// writeFileSync on a descriptor writes every byte or throws; a short or failed write never reaches the name.
try { writeFileSync(fd, bytes); fsyncSync(fd); } finally { closeSync(fd); }
if (!sameDirectory(dir, identity)) throw new Error("cache directory changed");
renameSync(temp, join(dir, name));
} catch (error) {
try { unlinkSync(temp); } catch { /* already gone */ }
throw error;
}
}
function unpublish(dir, identity, name) {
if (!sameDirectory(dir, identity)) throw new Error("cache directory changed");
try { unlinkSync(join(dir, name)); } catch (error) { if (error?.code !== "ENOENT") throw error; }
}
/**
* The record is one empty marker per version in ocx-child-builds/.versions/: "<version>.filled" when
* that version's fill completed, "<version>.baseline" when it ran and fetched nothing (a first start,
* or a package without digests). Markers are only ever added, so the newest version recorded can never
* go down, whichever fill writes last. null: no marker (or an unreadable directory), no evidence.
*/
function readVersionRecord(root) {
let names;
try {
const st = lstatSync(join(root, VERSIONS_DIR));
if (!st.isDirectory() || st.isSymbolicLink()) return null;
// Another user able to add or remove markers could suppress fills or fake an upgrade.
if (POSIX && (st.uid !== currentUid() || (st.mode & 0o022) !== 0)) names = null;
else names = readdirSync(join(root, VERSIONS_DIR));
} catch { return null; }
if (names === null) throw new Error("unsafe cache directory: " + join(root, VERSIONS_DIR));
let newest = null;
for (const name of names) {
const match = /^(.+)\.(baseline|filled)$/.exec(name);
if (!match || compareReleaseVersions(match[1], match[1]) === null) continue;
const order = newest === null ? 1 : compareReleaseVersions(match[1], newest.version);
if (order === 1) newest = { version: match[1], filled: match[2] === "filled" };
else if (order === 0 && match[2] === "filled") newest.filled = true;
}
return newest;
}
/** The release builds verified on disk right now: what a manifest may list, whoever wrote them. */
function verifiedOnDisk(dir, digests) {
const verified = new Map();
for (const [file, expected] of digests) {
const bytes = readCachedBuild(join(dir, file));
if (bytes && sha256Hex(bytes) === expected) verified.set(file, expected);
}
return verified;
}
function readCacheManifest(dir) {
return parseManifest((readRegular(join(dir, "SHA256SUMS"), MAX_MANIFEST_BYTES) ?? Buffer.alloc(0)).toString("utf8"));
}
/** A completed fill's manifest still lists every build the package vouches for. */
function manifestCovers(dir, digests) {
let listed;
try { listed = readCacheManifest(dir); } catch { return false; }
return [...digests].every(([file, sha]) => listed.get(file) === sha);
}
/** An existing build the reader would accept: a non-empty regular file, never a symlink, writable only by this user. */
function readCachedBuild(path) {
let st;
try { st = lstatSync(path); } catch { return null; }
if (!st.isFile() || st.isSymbolicLink() || st.size <= 0 || st.size > MAX_CHILD_BINARY_BYTES) return null;
if (POSIX && (st.uid !== currentUid() || (st.mode & 0o022) !== 0)) return null;
try { return readFileSync(path); } catch { return null; }
}
async function download(url, fetchImpl, timeoutMs) {
let response;
try { response = await fetchImpl(url, { redirect: "follow", signal: AbortSignal.timeout(timeoutMs) }); } catch { return null; }
if (!response.ok || !response.body) return null;
if (Number(response.headers.get("content-length") ?? "0") > MAX_CHILD_BINARY_BYTES) {
try { await response.body.cancel(); } catch { /* already closed */ }
return null;
}
const chunks = [];
let size = 0;
try {
for await (const chunk of response.body) {
size += chunk.byteLength;
if (size > MAX_CHILD_BINARY_BYTES) return null;
chunks.push(chunk);
}
} catch { return null; }
return Buffer.concat(chunks, size);
}
/** A cache directory for a version newer than this one: this Home was downgraded. */
function newerVersionCached(home, version) {
let names;
try { names = readdirSync(join(home, CHILD_FILL_DIR)); } catch { return false; }
return names.some(name => compareReleaseVersions(name, version) === 1);
}
/** The package tree this module was loaded from: <root>/src/link/child-binary-cache-fill.mjs. */
export function runningPackageRoot() {
return fileURLToPath(new URL("../../", import.meta.url));
}
/**
* Keeps Home processes sharing one OPENCODEX_HOME from downloading the same builds at once. It is an
* optimisation, never the guarantee: two fills can still overlap after a stale lock is reclaimed, so
* the manifest is rebuilt from what is verified on disk when it is written, and a completed record
* whose manifest no longer covers the package's digests is repaired on the next start.
*/
const FILL_LOCK = ".fill.lock";
/** A fill takes at most a few minutes (five builds, 20 s each); an older lock is a crash's leftover. */
const STALE_LOCK_MS = 60 * 60 * 1000;
function processAlive(pid) {
try { process.kill(pid, 0); return true; } catch (error) { return error?.code === "EPERM"; }
}
function readLock(path) {
try {
const st = lstatSync(path);
if (!st.isFile() || st.isSymbolicLink() || st.size > 256) return null;
const token = readFileSync(path, "utf8").trim();
const pid = Number(token.split(".")[0]);
return Number.isSafeInteger(pid) && pid > 0 ? { token, pid, mtimeMs: st.mtimeMs } : null;
} catch { return null; }
}
function removeQuietly(path) {
try { unlinkSync(path); } catch { /* already gone */ }
}
/**
* The lock file appears whole or not at all (linked from a private temp), names "<pid>.<nonce>", and
* is reclaimed only when its process is gone or it is older than STALE_LOCK_MS. null: another fill runs.
*/
function acquireFillLock(root, rootIdentity, now) {
const path = join(root, FILL_LOCK);
const token = process.pid + "." + randomBytes(8).toString("hex");
for (let attempt = 0; attempt < 2; attempt++) {
if (!sameDirectory(root, rootIdentity)) throw new Error("cache directory changed");
const temp = join(root, ".ocx-child-cache." + token + ".lock.tmp");
const fd = openSync(temp, "wx", 0o600);
try {
// As in publish: the whole record or nothing, and the temp never outlives a failed write or fsync.
const record = Buffer.from(token + "\n");
try {
if (writeSync(fd, record) !== record.length) throw new Error("short write to the cache fill lock");
fsyncSync(fd);
} finally { closeSync(fd); }
try { linkSync(temp, path); return { path, token }; }
catch (error) { if (error?.code !== "EEXIST") throw error; }
} finally { removeQuietly(temp); }
const holder = readLock(path);
if (holder && processAlive(holder.pid) && now() - holder.mtimeMs < STALE_LOCK_MS) return null;
// Move the leftover aside, and put it back if a live fill took the lock in the meantime.
const aside = join(root, ".ocx-child-cache." + token + ".stale.tmp");
try { renameSync(path, aside); } catch { continue; }
if (readLock(aside)?.token !== holder?.token) {
try { linkSync(aside, path); } catch { /* a third fill holds it now */ }
removeQuietly(aside);
return null;
}
removeQuietly(aside);
}
return null;
}
function assertFillLockHeld(lock) {
if (readLock(lock.path)?.token !== lock.token) throw new Error("cache fill lock lost");
}
function releaseFillLock(lock) {
if (lock && readLock(lock.path)?.token === lock.token) removeQuietly(lock.path);
}
/** A real directory is there (a symlink or another error counts as present: the write path refuses it). */
function directoryPresent(path) {
try { return lstatSync(path).isDirectory(); } catch (error) { return error?.code !== "ENOENT"; }
}
/**
* What the record alone decides, before any lock or write; null when there is work to do. A start
* that writes nothing (a same-version restart, a downgrade, a package without digests whose version
* is recorded) takes no lock and touches no file.
*/
function decideFromRecord(recorded, version, digests, dir) {
if (!digests) return recorded !== null && compareReleaseVersions(version, recorded.version) !== 1 ? { kind: "skipped", reason: "no_digests" } : null;
if (recorded === null) return null;
const order = compareReleaseVersions(version, recorded.version);
if (order === -1) return { kind: "skipped", reason: "downgrade" };
// A same-version start fetches nothing. It re-lists the builds still verified on disk only when a
// completed fill's manifest lost lines (fills that overlapped); a version directory removed by hand
// stays removed.
if (order === 0 && (!recorded.filled || !directoryPresent(dir) || manifestCovers(dir, digests))) return { kind: "skipped", reason: "not_an_upgrade" };
return null;
}
export async function fillChildBinaryCacheForVersion(options) {
// The caller names the Home (the startup step passes getConfigDir(), the Home the reader resolves too); no
// environment fallback, so the fill and the reader can never resolve two different Homes.
const home = options.opencodexHome;
if (typeof home !== "string" || home.length === 0) throw new Error("fillChildBinaryCacheForVersion: opencodexHome is required");
if (compareReleaseVersions(options.version, options.version) === null) return { kind: "skipped", reason: "invalid_version" };
if ((options.platform ?? process.platform) === "win32") return { kind: "skipped", reason: "unsupported_platform" };
const digests = readDigests(options.packageRoot);
// A digest file that is there but unusable is not "no digests": the record stays, so a repair still fills.
if (digests === "invalid") return { kind: "skipped", reason: "invalid_digests" };
const root = join(home, CHILD_FILL_DIR);
const dir = childBinaryCacheDir(home, options.version);
let rootIdentity, lock = null, recorded;
try {
assertTrustedAncestors(home);
if (newerVersionCached(home, options.version)) return { kind: "skipped", reason: "downgrade" };
// After the Home directory itself passed the ancestor checks and before the fill's directory is
// created: the startup step claims it for uninstall.
const refused = options.claimCache?.() ?? null;
if (refused) return { kind: "skipped", reason: refused };
rootIdentity = realDirectory(root);
const early = decideFromRecord(readVersionRecord(root), options.version, digests, dir);
if (early) return early;
lock = acquireFillLock(root, rootIdentity, options.now ?? Date.now);
if (!lock) return { kind: "skipped", reason: "busy" };
recorded = readVersionRecord(root);
} catch (error) {
releaseFillLock(lock);
// Without digests only the baseline record was at stake: stay silent, as before this lane.
if (!digests) return { kind: "skipped", reason: "no_digests" };
throw error;
}
try {
// Decided again under the lock: another start may have recorded meanwhile.
const settled = decideFromRecord(recorded, options.version, digests, dir);
if (settled) return settled;
const record = kind => {
assertFillLockHeld(lock);
const versions = join(root, VERSIONS_DIR);
publish(versions, realDirectory(versions), options.version + "." + kind, Buffer.alloc(0));
};
if (!digests) {
// The baseline a later digest-bearing release needs to recognise its start as an upgrade.
try { record("baseline"); } catch { /* best effort */ }
return { kind: "skipped", reason: "no_digests" };
}
if (recorded === null) { record("baseline"); return { kind: "skipped", reason: "first_start" }; }
const sameVersion = compareReleaseVersions(options.version, recorded.version) === 0;
const identity = realDirectory(dir);
const fetchImpl = options.fetchImpl ?? fetch;
const timeoutMs = options.timeoutMs ?? 60_000;
const fetched = [];
const failed = [];
for (const [file, expected] of [...digests].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
const present = readCachedBuild(join(dir, file));
if (present && sha256Hex(present) === expected) continue;
if (sameVersion) continue;
const bytes = await download(childReleaseAssetUrl(options.version, file), fetchImpl, timeoutMs);
if (bytes && sha256Hex(bytes) === expected) {
publish(dir, identity, file, bytes);
fetched.push(file);
} else {
failed.push(file);
}
}
assertFillLockHeld(lock);
// Rebuilt from the disk as it is now, so an overlapping fill's verified builds stay listed and a
// build the release does not vouch for never stays under an older line (the fill deletes no build file;
// unlisted, the reader refuses it). Lines for targets the release does not list are left as they
// are: the fill manages only the builds its digests name.
const verified = verifiedOnDisk(dir, digests);
const manifest = new Map([...readCacheManifest(dir)].filter(([file]) => !digests.has(file)));
for (const [file, sha] of verified) manifest.set(file, sha);
if (manifest.size > 0) publish(dir, identity, "SHA256SUMS", Buffer.from([...manifest].map(([file, sha]) => sha + " " + file + "\n").join("")));
else unpublish(dir, identity, "SHA256SUMS");
// An incomplete fill leaves the older record, so the next start of this version retries.
if (verified.size === digests.size) record("filled");
// Counted from what is verified on disk now, never from what was written.
const cached = [...verified.keys()].sort();
return { kind: "filled", cached, fetched: fetched.filter(file => verified.has(file)), failed: failed.filter(file => !verified.has(file)) };
} finally {
releaseFillLock(lock);
}
}
/** The running package's version, from its own package.json; null when unreadable. */
export function runningPackageVersion(packageRoot = runningPackageRoot()) {
try {
const parsed = JSON.parse(readRegular(join(packageRoot, "package.json"), MAX_MANIFEST_BYTES)?.toString("utf8") ?? "null");
return typeof parsed?.version === "string" ? parsed.version : null;
} catch { return null; }
}
/**
* Startup step (src/server/index.ts): fill the cache for the running version in the background and
* say what happened in one line. Never throws and never delays the server.
*/
export async function fillChildBinaryCacheOnStart(options = {}, out = { log: console.log, warn: console.warn }) {
let outcome;
// The documented opt-out (structure/remote-link.md, the Remote Link guide): nothing read or written.
if (process.env.OCX_DISABLE_CHILD_CACHE_FILL === "1") return { kind: "skipped", reason: "disabled" };
const packageRoot = options.packageRoot ?? runningPackageRoot();
const onDisk = runningPackageVersion(packageRoot);
const version = options.version ?? onDisk ?? "";
try {
// A source checkout (a developer's server, the test suite) never fetches a release.
if (detectInstallFromPath(packageRoot) === "source") return { kind: "skipped", reason: "source_checkout" };
// A package.json that is not the running code's version (an in-place install still extracting)
// records nothing: its baseline could suppress that version's upgrade fill.
if (options.bundledVersion !== undefined && onDisk !== options.bundledVersion) return { kind: "skipped", reason: "version_mismatch" };
outcome = await fillChildBinaryCacheForVersion({ timeoutMs: 20_000, ...options, packageRoot, version });
} catch {
out.warn("[link] the ocx-child builds of this release could not be cached; planting on a bare machine needs a build supplied by hand.");
return null;
}
if (outcome.kind === "filled") {
const fetched = outcome.fetched.length;
if (fetched > 0) out.log("[link] cached " + fetched + " verified ocx-child build" + (fetched === 1 ? "" : "s") + " for v" + version + ".");
if (outcome.failed.length > 0) out.warn("[link] " + outcome.failed.length + " ocx-child build" + (outcome.failed.length === 1 ? "" : "s") + " of this release could not be fetched or did not match the package digests; only verified builds are planted.");
}
if (outcome.kind === "skipped" && outcome.reason === "invalid_digests") out.warn("[link] this package's ocx-child digest file could not be read; no ocx-child build was cached.");
return outcome;
}