@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
933 lines (877 loc) • 34 kB
text/typescript
import {
existsSync,
mkdirSync,
readdirSync,
readFileSync,
renameSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { basename, join } from "node:path";
import { randomUUID } from "node:crypto";
import { Database } from "bun:sqlite";
import { resolveCodexHomeDir } from "../../codex/home";
import { readThreadFieldsFromRollout } from "../../codex/history-provider";
import { discoverRuntimeDbPaths, insertRowsConflictIgnore, isSqlRowArray, mapDbError, probeStateDbWritable, tableColumnNames, tableExists, withWritableDb } from "./db";
import type { ReconcileErr, SqlRow } from "./db";
import { ARCHIVED_SESSIONS_DIR, TRASH_DIR, TRASH_EPOCH_DIR, ZST_SUFFIX, isExistError, isRolloutFileName, isSafeArchivedPhysicalRel, renameNoReplace, toForwardSlash } from "./paths";
import { RESTORE_PENDING_FILE, readRestorePending, writeRestorePending } from "./pending";
import type { RestorePendingSections, RestorePendingState } from "./pending";
import { SATELLITE_BACKUP_FILE, beginSatelliteWriteLocks, commitSatelliteLock, readSatelliteBackupFile, remapSatelliteBackupPaths, restoreConsolidateGlobalJob, rollbackAllSatelliteLocks } from "./satellite";
import type { SatelliteBackup, SatelliteWriteLocks } from "./satellite";
import { absFromRel, removeEmptyTrashRoot } from "./staging";
import type { StagedFile } from "./staging";
import type { CleanupManifestEntry, CleanupMode, RestoreErrorCode, RestoreResult, TrashEntrySummary } from "./types";
interface TrashManifest {
quarantinedAt?: number;
mode?: CleanupMode;
entries?: CleanupManifestEntry[];
}
/**
* Parse a trash `manifest.json` atomically.
*
* Any missing `entries` array, or any malformed entry / `physicalRelPaths` value /
* required field, rejects the **entire** manifest (returns null). Individual bad
* entries are never filtered out so a partial parse cannot silently drop evidence.
*/
function parseTrashManifest(raw: string): TrashManifest | null {
try {
const parsed = JSON.parse(raw) as unknown;
if (!parsed || typeof parsed !== "object") return null;
const o = parsed as Record<string, unknown>;
if (!Array.isArray(o.entries)) return null;
const entries: CleanupManifestEntry[] = [];
for (const e of o.entries) {
if (!e || typeof e !== "object" || Array.isArray(e)) return null;
const entry = e as Record<string, unknown>;
if (typeof entry.relPath !== "string" || entry.relPath.length === 0) return null;
if (typeof entry.bytes !== "number" || !Number.isFinite(entry.bytes)) return null;
if (typeof entry.mtimeMs !== "number" || !Number.isFinite(entry.mtimeMs)) return null;
if (!Array.isArray(entry.physicalRelPaths) || entry.physicalRelPaths.length === 0) return null;
const physical: string[] = [];
for (const p of entry.physicalRelPaths) {
// Do not strip bad elements — one malformed path invalidates the whole manifest.
if (typeof p !== "string" || p.length === 0) return null;
physical.push(p);
}
if ("threadId" in entry && typeof entry.threadId !== "string") return null;
if ("rolloutPath" in entry && typeof entry.rolloutPath !== "string") return null;
if (
"archived" in entry
&& entry.archived !== null
&& typeof entry.archived !== "number"
) {
return null;
}
entries.push({
relPath: entry.relPath,
bytes: entry.bytes,
mtimeMs: entry.mtimeMs,
physicalRelPaths: physical,
...(typeof entry.threadId === "string" ? { threadId: entry.threadId } : {}),
...(typeof entry.rolloutPath === "string" ? { rolloutPath: entry.rolloutPath } : {}),
...(entry.archived === null || typeof entry.archived === "number"
? { archived: entry.archived as number | null }
: {}),
});
}
const out: TrashManifest = { entries };
if (typeof o.quarantinedAt === "number" && Number.isFinite(o.quarantinedAt)) {
out.quarantinedAt = o.quarantinedAt;
}
if (o.mode === "quarantine" || o.mode === "permanent") out.mode = o.mode;
return out;
} catch {
return null;
}
}
/**
* Validate a trash entry id as a single `.trash/<epoch>` segment under CODEX_HOME.
* Returns the absolute stage directory, or null when the id is unsafe / missing.
*/
export function resolveTrashStageDir(
trashId: string,
codexHome: string,
): { ok: true; stageDir: string; id: string } | { ok: false; error: RestoreErrorCode } {
const normalized = toForwardSlash(trashId.trim()).replace(/\/+$/, "");
if (!normalized.startsWith(`${TRASH_DIR}/`)) return { ok: false, error: "invalid_trash" };
const rest = normalized.slice(TRASH_DIR.length + 1);
if (!rest || rest.includes("/") || rest.includes("\\") || rest.includes("..")) {
return { ok: false, error: "invalid_trash" };
}
if (!TRASH_EPOCH_DIR.test(rest)) return { ok: false, error: "invalid_trash" };
let stageDir: string;
try {
stageDir = absFromRel(codexHome, `${TRASH_DIR}/${rest}`);
} catch {
return { ok: false, error: "invalid_trash" };
}
if (!existsSync(stageDir)) return { ok: false, error: "missing_trash" };
try {
if (!statSync(stageDir).isDirectory()) return { ok: false, error: "invalid_trash" };
} catch {
return { ok: false, error: "missing_trash" };
}
return { ok: true, stageDir, id: `${TRASH_DIR}/${rest}` };
}
function sumTrashEntryBytes(stageDir: string, manifest: TrashManifest | null): {
fileCount: number;
bytes: number;
} {
let fileCount = 0;
let bytes = 0;
let names: string[] = [];
try {
names = readdirSync(stageDir);
} catch {
return { fileCount: 0, bytes: 0 };
}
for (const name of names) {
if (
name === "manifest.json"
|| name === SATELLITE_BACKUP_FILE
|| name === RESTORE_PENDING_FILE
) {
continue;
}
if (!isRolloutFileName(name)) continue;
try {
const st = statSync(join(stageDir, name));
if (!st.isFile()) continue;
fileCount += 1;
bytes += st.size;
} catch { /* */ }
}
// Prefer live FS counts; fall back to manifest totals when the stage is empty of rollouts.
if (fileCount === 0 && manifest?.entries?.length) {
fileCount = manifest.entries.reduce((n, e) => n + Math.max(1, e.physicalRelPaths.length), 0);
bytes = manifest.entries.reduce((n, e) => n + (e.bytes || 0), 0);
}
return { fileCount, bytes };
}
/** List quarantine entries under `CODEX_HOME/.trash/` (relative ids only). */
export function listTrashEntries(
codexHome: string = resolveCodexHomeDir(),
): TrashEntrySummary[] {
const trashRoot = join(codexHome, TRASH_DIR);
let names: string[] = [];
try {
names = readdirSync(trashRoot);
} catch {
return [];
}
const out: TrashEntrySummary[] = [];
for (const name of names) {
if (!TRASH_EPOCH_DIR.test(name)) continue;
const stageDir = join(trashRoot, name);
try {
if (!statSync(stageDir).isDirectory()) continue;
} catch {
continue;
}
let manifest: TrashManifest | null = null;
try {
manifest = parseTrashManifest(readFileSync(join(stageDir, "manifest.json"), "utf8"));
} catch {
manifest = null;
}
const { fileCount, bytes } = sumTrashEntryBytes(stageDir, manifest);
// Skip empty collision placeholders left behind without a manifest or rollouts.
if (fileCount === 0 && !manifest?.entries?.length) {
try {
if (!existsSync(join(stageDir, "manifest.json"))) continue;
} catch {
continue;
}
}
out.push({
id: `${TRASH_DIR}/${name}`,
epoch: name,
fileCount,
bytes,
...(manifest?.quarantinedAt !== undefined ? { quarantinedAt: manifest.quarantinedAt } : {}),
...(manifest?.mode ? { mode: manifest.mode } : {}),
});
}
out.sort((a, b) => {
const aq = a.quarantinedAt ?? (Number(a.epoch.split("-")[0]) || 0);
const bq = b.quarantinedAt ?? (Number(b.epoch.split("-")[0]) || 0);
return bq - aq || b.epoch.localeCompare(a.epoch);
});
return out;
}
/** True when a snapshotted thread row covers every NOT NULL column on the live schema. */
function threadSnapshotCoversRequiredColumns(row: SqlRow, requiredCols: string[]): boolean {
for (const col of requiredCols) {
if (!(col in row) || row[col] === undefined) return false;
}
return true;
}
function requiredThreadColumnNames(db: Database): string[] {
if (!tableExists(db, "threads")) return [];
const rows = db.query<{ name: string; notnull: number }, []>(
`PRAGMA table_info("threads")`,
).all();
return rows.filter(r => r.notnull === 1).map(r => r.name);
}
/**
* Build a production-shaped thread row for schemas that predate full satellite snapshots.
* Prefer `readThreadFieldsFromRollout` (canonical history/session_meta path); fall back to
* the sparse manifest fields only when the live schema does not require model/source/message.
*/
function reconstructThreadRowFromRollout(
entry: CleanupManifestEntry,
rolloutAbsPath: string,
allowedCols: Set<string>,
requiredCols: string[],
): SqlRow | null {
if (typeof entry.threadId !== "string" || typeof entry.rolloutPath !== "string") return null;
const fields = readThreadFieldsFromRollout(rolloutAbsPath);
const row: SqlRow = {
id: entry.threadId,
rollout_path: entry.rolloutPath,
};
if (fields) {
// Prefer manifest thread id (binding) but keep rollout-derived listing fields.
if (allowedCols.has("model_provider")) row.model_provider = fields.modelProvider;
if (allowedCols.has("source")) row.source = fields.source;
if (allowedCols.has("first_user_message")) row.first_user_message = fields.firstUserMessage;
if (allowedCols.has("has_user_event")) row.has_user_event = fields.hasUserEvent;
if (allowedCols.has("cwd") && fields.cwd !== undefined) row.cwd = fields.cwd;
if (allowedCols.has("history_mode") && fields.historyMode !== undefined) {
row.history_mode = fields.historyMode;
}
if (allowedCols.has("cli_version") && fields.cliVersion !== undefined) {
row.cli_version = fields.cliVersion;
}
}
if (allowedCols.has("archived")) {
row.archived = entry.archived ?? 1;
}
if (allowedCols.has("archived_at")) {
row.archived_at = null;
}
// Fill remaining NOT NULL columns with safe empties when the rollout lacked them
// (e.g. fixture rollouts without a user turn still need first_user_message = '').
for (const col of requiredCols) {
if (row[col] !== undefined) continue;
if (col === "id" || col === "rollout_path") continue;
if (col === "model_provider") row[col] = "openai";
else if (col === "source") row[col] = "cli";
else if (col === "first_user_message") row[col] = "";
else if (col === "has_user_event") row[col] = 0;
else if (col === "archived") row[col] = entry.archived ?? 1;
else return null; // unknown required column we cannot invent
}
// If the schema requires listing fields, refuse when the rollout was unreadable.
const needsSessionMeta = requiredCols.some(
c => c === "model_provider" || c === "source" || c === "first_user_message",
);
if (needsSessionMeta && !fields) return null;
return row;
}
function restoreThreadsFromManifest(
stateDbPath: string | null,
entries: CleanupManifestEntry[],
backup: SatelliteBackup | null,
busyTimeoutMs: number,
codexHome: string,
): { ok: true } | ReconcileErr {
const manifestThreadIds = entries
.map(e => e.threadId)
.filter((id): id is string => typeof id === "string");
const backupThreadIds = backup?.threadIds ?? [];
const needsThreads = manifestThreadIds.length > 0
|| backupThreadIds.length > 0
|| Boolean(backup?.threads?.length);
if (needsThreads && (!stateDbPath || !existsSync(stateDbPath))) {
return { ok: false, error: "db_reconcile_failed" };
}
if (!stateDbPath || !existsSync(stateDbPath)) {
return { ok: true };
}
const result = withWritableDb(stateDbPath, busyTimeoutMs, db => {
if (!tableExists(db, "threads")) throw new Error("missing_threads_table");
const requiredCols = requiredThreadColumnNames(db);
const allowedCols = tableColumnNames(db, "threads");
const snapshotThreads = backup?.threads && isSqlRowArray(backup.threads)
? backup.threads
: [];
const completeSnapshots = snapshotThreads.filter(row =>
threadSnapshotCoversRequiredColumns(row, requiredCols),
);
const coveredIds = new Set(
completeSnapshots
.map(r => r.id)
.filter((id): id is string => typeof id === "string"),
);
// Legacy Phase-2 quarantine (no / incomplete satellite thread snapshots): reconstruct
// every required column from the restored rollout via the history-provider session path.
const toReconstruct = entries.filter(
e => typeof e.threadId === "string"
&& typeof e.rolloutPath === "string"
&& !coveredIds.has(e.threadId!),
);
const reconstructed: SqlRow[] = [];
for (const entry of toReconstruct) {
let abs: string | undefined;
try {
abs = absFromRel(codexHome, entry.rolloutPath!);
} catch {
abs = undefined;
}
// Legacy compressed-only quarantine: manifest rolloutPath is often the logical
// `.jsonl` name while the only restored physical file is `.jsonl.zst`.
if (!abs || !existsSync(abs)) {
for (const rel of entry.physicalRelPaths) {
try {
const candidate = absFromRel(codexHome, rel);
if (existsSync(candidate)) {
abs = candidate;
break;
}
} catch {
/* try next physical path */
}
}
}
if (!abs) throw new Error("missing_rollout_for_thread");
// Prefer a plain .jsonl sibling when present; otherwise readThreadFieldsFromRollout
// decompresses a lone .jsonl.zst in memory (bounded) for legacy quarantine restores.
if (abs.endsWith(ZST_SUFFIX)) {
const plain = abs.slice(0, -".zst".length);
if (existsSync(plain)) abs = plain;
}
const row = reconstructThreadRowFromRollout(entry, abs, allowedCols, requiredCols);
if (!row) throw new Error("thread_reconstruct_failed");
reconstructed.push(row);
}
if (completeSnapshots.length > 0) {
insertRowsConflictIgnore(db, "threads", completeSnapshots);
}
if (reconstructed.length > 0) {
insertRowsConflictIgnore(db, "threads", reconstructed);
}
if (backup?.dynamicTools && isSqlRowArray(backup.dynamicTools) && tableExists(db, "thread_dynamic_tools")) {
insertRowsConflictIgnore(db, "thread_dynamic_tools", backup.dynamicTools);
}
if (backup?.spawnEdges && isSqlRowArray(backup.spawnEdges) && tableExists(db, "thread_spawn_edges")) {
insertRowsConflictIgnore(db, "thread_spawn_edges", backup.spawnEdges);
}
});
if (!result.ok) return result;
return { ok: true };
}
/** Test-only failure injection for restore atomicity regressions. */
export interface RestoreTestHooks {
/** After state threads/dependents commit, before satellite commits. */
failAfterStateCommit?: boolean;
/** After the first satellite DB commit (logs → memories → goals). */
failAfterFirstSatelliteCommit?: boolean;
/** When the leftover staged-rollout completeness gate runs. */
failAtLeftoverStageGate?: boolean;
/** Fail the initial restore-pending.json write (before any file moves). */
failInitialPendingWrite?: boolean;
/** Fail a later pending update after the temp is written but before rename. */
failPendingWriteBeforeRename?: boolean;
/** Crash immediately after file moves (marker already durable). */
failAfterFileMoves?: boolean;
/**
* After this many successful rollout moves in the current attempt, throw.
* Exercises mid-loop failure with some dests placed and others still staged.
*/
failAfterMoveCount?: number;
/** Fail renaming the completed stage to a non-listable tombstone dir. */
failStageTombstoneRename?: boolean;
/** After tombstone rename, skip best-effort tombstone delete (orphan is OK). */
failTombstoneDelete?: boolean;
/**
* Test-only: spin-wait this many ms after rollout file moves, before DB
* reconcile, so cleanup can race an in-flight restore.
*/
holdAfterFileMovesMs?: number;
/**
* Test-only: publish a ready file after rollout moves, then wait until the
* release file exists. This makes cross-thread race tests phase-driven.
*/
pauseAfterFileMoves?: { readyPath: string; releasePath: string };
}
/**
* Resume must not clear owed satellite work when the matching backup section is
* absent — fail closed per section instead.
*/
function failClosedSatelliteResume(
priorPending: RestorePendingState,
satelliteBackup: SatelliteBackup | null,
): RestoreErrorCode | null {
const owed = priorPending.pending;
if (!owed.logs && !owed.memories && !owed.goals) return null;
if (!satelliteBackup) return "db_reconcile_failed";
if (owed.logs && !satelliteBackup.logs) return "db_reconcile_failed";
if (owed.memories && !satelliteBackup.memories) return "db_reconcile_failed";
if (owed.goals && !satelliteBackup.goals) return "db_reconcile_failed";
return null;
}
/**
* Successful restore finalization: rename the stage to a tombstone name that
* `listTrashEntries` ignores, then delete the tombstone best-effort. A failed
* rename leaves the original stage (and all evidence) intact for retry.
*/
function finalizeRestoredStage(
stageDir: string,
codexHome: string,
hooks?: Pick<RestoreTestHooks, "failStageTombstoneRename" | "failTombstoneDelete">,
): boolean {
const trashRoot = join(codexHome, TRASH_DIR);
const epoch = basename(stageDir);
const tombstoneName = `.tombstone-${epoch}-${randomUUID()}`;
const tombstonePath = join(trashRoot, tombstoneName);
try {
if (hooks?.failStageTombstoneRename) throw new Error("test_fail_stage_tombstone_rename");
renameSync(stageDir, tombstonePath);
} catch {
return false;
}
if (!hooks?.failTombstoneDelete) {
try { rmSync(tombstonePath, { recursive: true, force: true }); } catch { /* best-effort */ }
}
return true;
}
/**
* Restore one quarantine entry: move JSONL back, re-insert threads (+ satellites
* when satellite-backup.json is present), then remove the trash directory.
*
* Late failures after files have moved never compensate metadata or restage.
* Instead they persist `restore-pending.json` (accepted dest paths + which
* state/logs/memories/goals sections still need work) atomically *before* any
* rollout move, then update it after each section so a retry can accept existing
* destinations and resume only missing metadata.
*/
export function restoreTrashEntry(
trashId: string,
options?: {
codexHome?: string;
busyTimeoutMs?: number;
_test?: RestoreTestHooks;
},
): RestoreResult {
const codexHome = options?.codexHome ?? resolveCodexHomeDir();
const busyTimeoutMs = options?.busyTimeoutMs ?? 100;
const hooks = options?._test;
const resolved = resolveTrashStageDir(trashId, codexHome);
if (!resolved.ok) {
return { ok: false, count: 0, bytes: 0, restoredPaths: [], error: resolved.error };
}
const { stageDir, id } = resolved;
let manifestRaw: string;
try {
manifestRaw = readFileSync(join(stageDir, "manifest.json"), "utf8");
} catch {
return { ok: false, trashDir: id, count: 0, bytes: 0, restoredPaths: [], error: "invalid_trash" };
}
const manifest = parseTrashManifest(manifestRaw);
if (!manifest?.entries?.length) {
return { ok: false, trashDir: id, count: 0, bytes: 0, restoredPaths: [], error: "invalid_trash" };
}
const pendingRead = readRestorePending(stageDir);
if (pendingRead.status === "invalid") {
// Malformed marker means an incomplete restore may already have moved files;
// never treat it as a fresh restore.
return { ok: false, trashDir: id, count: 0, bytes: 0, restoredPaths: [], error: "fs_failed" };
}
const priorPending = pendingRead.status === "valid" ? pendingRead.state : null;
const acceptedDest = new Set(priorPending?.acceptedDestRels ?? []);
// Partial permanent purges may leave only a subset of physical files on disk —
// trim to survivors rather than failing the whole entry for a purged twin.
// Resume also treats already-restored accepted destinations as survivors.
const entries: CleanupManifestEntry[] = [];
for (const entry of manifest.entries) {
if (!entry.physicalRelPaths.every(isSafeArchivedPhysicalRel)) {
return { ok: false, trashDir: id, count: 0, bytes: 0, restoredPaths: [], error: "invalid_trash" };
}
const surviving = entry.physicalRelPaths.filter(rel => {
if (existsSync(join(stageDir, basename(rel)))) return true;
if (!acceptedDest.has(rel)) return false;
try {
return existsSync(absFromRel(codexHome, rel));
} catch {
return false;
}
});
if (surviving.length === 0) {
return { ok: false, trashDir: id, count: 0, bytes: 0, restoredPaths: [], error: "fs_failed" };
}
entries.push({ ...entry, physicalRelPaths: surviving });
}
const paths = discoverRuntimeDbPaths(codexHome);
const backupRead = readSatelliteBackupFile(stageDir);
if (backupRead.status === "invalid") {
return {
ok: false,
trashDir: id,
count: 0,
bytes: 0,
restoredPaths: [],
error: "db_reconcile_failed",
};
}
let satelliteBackup: SatelliteBackup | null = null;
if (backupRead.status === "ok") {
const remapped = remapSatelliteBackupPaths(backupRead.backup, paths);
if (!remapped.ok) {
return {
ok: false,
trashDir: id,
count: 0,
bytes: 0,
restoredPaths: [],
error: "db_reconcile_failed",
};
}
satelliteBackup = remapped.backup;
}
if (priorPending) {
const resumeErr = failClosedSatelliteResume(priorPending, satelliteBackup);
if (resumeErr) {
return {
ok: false,
trashDir: id,
count: 0,
bytes: 0,
restoredPaths: [],
error: resumeErr,
};
}
}
const pendingSections: RestorePendingSections = {
state: priorPending ? priorPending.pending.state : true,
logs: priorPending ? priorPending.pending.logs : Boolean(satelliteBackup?.logs),
memories: priorPending ? priorPending.pending.memories : Boolean(satelliteBackup?.memories),
goals: priorPending ? priorPending.pending.goals : Boolean(satelliteBackup?.goals),
};
const needAnySatellite = pendingSections.logs || pendingSections.memories || pendingSections.goals;
if (pendingSections.state) {
const needsThreads = entries.some(e => typeof e.threadId === "string")
|| Boolean(satelliteBackup?.threadIds?.length)
|| Boolean(satelliteBackup?.threads?.length);
if (needsThreads && (!paths.state || !existsSync(paths.state))) {
return {
ok: false,
trashDir: id,
count: 0,
bytes: 0,
restoredPaths: [],
error: "db_reconcile_failed",
};
}
const probe = probeStateDbWritable(codexHome, busyTimeoutMs);
if (!probe.ok) {
return {
ok: false,
trashDir: id,
count: 0,
bytes: 0,
restoredPaths: [],
error: probe.error === "codex_busy" ? "codex_busy" : "db_reconcile_failed",
};
}
}
// Acquire only the satellite locks still needed so a busy DB for an already-
// finished section cannot block resume. Locks happen before moves on a fresh
// attempt so failure stays retryable (nothing has left the stage yet).
let satelliteLocks: SatelliteWriteLocks | undefined;
if (needAnySatellite) {
try {
satelliteLocks = beginSatelliteWriteLocks(paths, busyTimeoutMs, {
logs: pendingSections.logs,
memories: pendingSections.memories,
goals: pendingSections.goals,
});
} catch (error) {
return {
ok: false,
trashDir: id,
count: 0,
bytes: 0,
restoredPaths: [],
error: mapDbError(error) === "codex_busy" ? "codex_busy" : "db_reconcile_failed",
};
}
}
const failBeforeMoves = (error: RestoreErrorCode): RestoreResult => {
if (satelliteLocks) rollbackAllSatelliteLocks(satelliteLocks);
return { ok: false, trashDir: id, count: 0, bytes: 0, restoredPaths: [], error };
};
// Plan renames: staged basename → original archived_sessions path.
// Resume accepts destinations already restored by this incomplete attempt.
const alreadyMoved: StagedFile[] = [];
const toMove: StagedFile[] = [];
for (const entry of entries) {
for (const rel of entry.physicalRelPaths) {
const base = basename(rel);
const from = join(stageDir, base);
let to: string;
try {
to = absFromRel(codexHome, rel);
} catch {
return failBeforeMoves("invalid_trash");
}
const fromExists = existsSync(from);
const toExists = existsSync(to);
if (toExists && acceptedDest.has(rel) && !fromExists) {
alreadyMoved.push({ from, to, relPath: rel });
continue;
}
if (toExists) {
return failBeforeMoves("dest_exists");
}
if (!fromExists) {
return failBeforeMoves("fs_failed");
}
toMove.push({ from, to, relPath: rel });
}
}
const planned = [...alreadyMoved, ...toMove];
const restoredPaths = [...new Set(entries.map(e => e.relPath))];
const bytes = entries.reduce((sum, e) => sum + (e.bytes || 0), 0);
const partialCounts = { count: restoredPaths.length, bytes, restoredPaths };
let pendingWriteCount = 0;
const persistPending = (): void => {
pendingWriteCount += 1;
const isInitial = pendingWriteCount === 1;
writeRestorePending(
stageDir,
{
version: 1,
filesRestored: true,
acceptedDestRels: planned.map(m => m.relPath),
pending: { ...pendingSections },
},
{
failWrite: Boolean(isInitial && hooks?.failInitialPendingWrite),
failBeforeRename: Boolean(!isInitial && hooks?.failPendingWriteBeforeRename),
},
);
};
// Durable resume marker before any rollout leaves the stage. Crash after a
// later move can still accept destinations from this marker.
try {
persistPending();
} catch {
return failBeforeMoves("fs_failed");
}
const newlyMoved: StagedFile[] = [];
try {
mkdirSync(join(codexHome, ARCHIVED_SESSIONS_DIR), { recursive: true });
for (const item of toMove) {
// Atomic no-replace (.trash ↔ archived_sessions). Mid-loop failure keeps
// already-placed dests and the durable planned acceptedDestRels marker.
renameNoReplace(item.from, item.to);
newlyMoved.push(item);
if (
hooks?.failAfterMoveCount !== undefined
&& newlyMoved.length >= hooks.failAfterMoveCount
) {
throw new Error("test_fail_after_move_count");
}
}
} catch (error) {
// Marker was written before any move. Never reverse successful renames or
// drop/narrow acceptedDestRels — resume must accept placed dests and finish
// the remaining staged files.
if (satelliteLocks) rollbackAllSatelliteLocks(satelliteLocks);
const placed = [...alreadyMoved, ...newlyMoved];
const placedPhysical = new Set(placed.map(m => m.relPath));
const partialEntries = entries.filter(e =>
e.physicalRelPaths.every(rel => placedPhysical.has(rel)),
);
const midMoveRestored = [...new Set(partialEntries.map(e => e.relPath))];
return {
ok: false,
trashDir: id,
count: midMoveRestored.length,
bytes: partialEntries.reduce((sum, e) => sum + (e.bytes || 0), 0),
restoredPaths: midMoveRestored,
error: isExistError(error) ? "dest_exists" : "fs_failed",
};
}
const moved = [...alreadyMoved, ...newlyMoved];
/**
* Never compensate DBs or restage files after moves. Keep restored files,
* persist which sections remain, and return accurate partial counts.
*/
const abortAfterMoves = (error: RestoreErrorCode): RestoreResult => {
if (satelliteLocks) {
rollbackAllSatelliteLocks(satelliteLocks);
satelliteLocks = undefined;
}
try {
persistPending();
} catch {
/* best-effort — files already restored; prior atomic marker remains */
}
return { ok: false, trashDir: id, ...partialCounts, error };
};
if (hooks?.pauseAfterFileMoves) {
writeFileSync(hooks.pauseAfterFileMoves.readyPath, "ready\n");
while (!existsSync(hooks.pauseAfterFileMoves.releasePath)) Bun.sleepSync(10);
}
if (hooks?.holdAfterFileMovesMs !== undefined) {
const holdMs = Math.max(0, Math.floor(hooks.holdAfterFileMovesMs));
if (holdMs > 0) {
const deadline = Date.now() + holdMs;
while (Date.now() < deadline) { /* test-only spin wait */ }
}
}
if (hooks?.failAfterFileMoves) {
return abortAfterMoves("fs_failed");
}
if (pendingSections.state) {
const threadsRestored = restoreThreadsFromManifest(
paths.state,
entries,
satelliteBackup,
busyTimeoutMs,
codexHome,
);
if (!threadsRestored.ok) {
return abortAfterMoves(
threadsRestored.error === "codex_busy" ? "codex_busy" : "db_reconcile_failed",
);
}
pendingSections.state = false;
try {
persistPending();
} catch {
return abortAfterMoves("fs_failed");
}
}
if (hooks?.failAfterStateCommit) {
return abortAfterMoves("db_reconcile_failed");
}
if (satelliteLocks && satelliteBackup) {
const locks = satelliteLocks;
try {
// Commit one satellite DB at a time; uncommitted txs roll back via
// rollbackAllSatelliteLocks. Completed sections are cleared in pending.
if (pendingSections.logs && satelliteBackup.logs) {
if (!locks.logs) throw new Error("missing_logs_lock");
if (!tableExists(locks.logs.db, "logs")) throw new Error("missing_logs_table");
insertRowsConflictIgnore(locks.logs.db, "logs", satelliteBackup.logs.rows);
commitSatelliteLock(locks.logs);
locks.logs = undefined;
pendingSections.logs = false;
persistPending();
if (hooks?.failAfterFirstSatelliteCommit) {
throw new Error("test_fail_after_first_satellite");
}
}
if (pendingSections.memories && satelliteBackup.memories) {
if (!locks.memories) throw new Error("missing_memories_lock");
const mem = satelliteBackup.memories;
if (!tableExists(locks.memories.db, "stage1_outputs")) {
throw new Error("missing_stage1_outputs_table");
}
insertRowsConflictIgnore(locks.memories.db, "stage1_outputs", mem.stage1);
if (tableExists(locks.memories.db, "jobs")) {
insertRowsConflictIgnore(locks.memories.db, "jobs", mem.stage1Jobs);
if (mem.consolidateTouched) {
restoreConsolidateGlobalJob(
locks.memories.db,
mem.consolidateJob,
mem.consolidatePostImage,
);
}
}
commitSatelliteLock(locks.memories);
locks.memories = undefined;
pendingSections.memories = false;
persistPending();
if (hooks?.failAfterFirstSatelliteCommit && !satelliteBackup.logs) {
throw new Error("test_fail_after_first_satellite");
}
}
if (pendingSections.goals && satelliteBackup.goals) {
if (!locks.goals) throw new Error("missing_goals_lock");
const g = satelliteBackup.goals;
if (!tableExists(locks.goals.db, "thread_goals")) {
throw new Error("missing_thread_goals_table");
}
insertRowsConflictIgnore(locks.goals.db, "thread_goals", g.goals);
if (tableExists(locks.goals.db, "thread_goal_continuation_deferrals")) {
insertRowsConflictIgnore(
locks.goals.db,
"thread_goal_continuation_deferrals",
g.deferrals,
);
}
commitSatelliteLock(locks.goals);
locks.goals = undefined;
pendingSections.goals = false;
persistPending();
if (
hooks?.failAfterFirstSatelliteCommit
&& !satelliteBackup.logs
&& !satelliteBackup.memories
) {
throw new Error("test_fail_after_first_satellite");
}
}
// Close any locks acquired for DBs that had no pending work / backup rows.
rollbackAllSatelliteLocks(locks);
satelliteLocks = undefined;
} catch (error) {
return abortAfterMoves(
mapDbError(error) === "codex_busy" ? "codex_busy" : "db_reconcile_failed",
);
}
}
if (
pendingSections.state
|| pendingSections.logs
|| pendingSections.memories
|| pendingSections.goals
) {
return abortAfterMoves("db_reconcile_failed");
}
// Completeness gate: every planned file must sit at its restored path, and the stage
// must hold no leftover rollout files, before we destroy the quarantine evidence.
for (const item of moved) {
if (!existsSync(item.to) || existsSync(item.from)) {
return abortAfterMoves("fs_failed");
}
}
try {
if (hooks?.failAtLeftoverStageGate) {
return abortAfterMoves("fs_failed");
}
for (const name of readdirSync(stageDir)) {
if (
name === "manifest.json"
|| name === SATELLITE_BACKUP_FILE
|| name === RESTORE_PENDING_FILE
) {
continue;
}
if (!isRolloutFileName(name)) continue;
return abortAfterMoves("fs_failed");
}
} catch {
return abortAfterMoves("fs_failed");
}
if (!finalizeRestoredStage(stageDir, codexHome, hooks)) {
return {
ok: false,
trashDir: id,
...partialCounts,
error: "fs_failed",
};
}
removeEmptyTrashRoot(codexHome);
return {
ok: true,
trashDir: id,
...partialCounts,
};
}