@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
556 lines (528 loc) • 21.5 kB
text/typescript
import { readConfigDiagnostics } from "../../config";
import { registerCurrentServerResourceCleanup } from "../../lib/server-resource-ownership";
import {
didRunOptionalShutdownHooks,
registerOptionalShutdownHook,
} from "../../lib/optional-shutdown-hooks";
import { queryLabStatus } from "../query";
import { rebuildLabProjection } from "../projection/rebuild";
import { planLabAutomationRuns } from "./planner";
import {
loadLabAutomationState,
mutateLabAutomationState,
} from "./persistence";
import { loadLabAutomationConfig } from "./config-persistence";
import {
rollBudgetWindow,
runBudgetRemaining,
liveRequestBudgetRemaining,
isRunBudgetExhausted,
isLiveRequestBudgetExhausted,
} from "./budgets";
import {
canReserveScheduledCooldown,
clearCooldown,
cooldownForFailure,
setCooldown,
} from "./cooldown";
import {
enqueuePlannedRuns,
selectDispatchableRuns,
transitionRun,
trimTerminalRuns,
countRunsByState,
} from "./queue";
import { dispatchLabAutomationRun, type DispatchResult } from "./dispatch";
import { recoverLabAutomationState } from "./recovery";
import type {
AutomationDispatchDeps,
LabAutomationPolicyV1,
LabAutomationRoutesV1,
LabAutomationRunRecordV1,
LabAutomationStatusV1,
} from "./types";
import { LabAutomationError } from "./types";
import { LAB_AUTOMATION_HARD_MAX } from "./constants";
import { cancelQueuedRun } from "./queue";
type SchedulerOwner = {
timer: ReturnType<typeof setInterval>;
ownerToken: symbol;
};
type DispatchOwner = {
token: symbol;
deps: AutomationDispatchDeps;
};
const schedulerTimers = new Map<string, SchedulerOwner>();
const ticksInProgress = new Set<string>();
let shutdownRequested = false;
const dispatchDepsByConfigDir = new Map<string, DispatchOwner>();
const inFlightControllers = new Map<string, AbortController>();
const cancellingRunIds = new Set<string>();
// Test seam: invoked after a manual run's queue row and per-run shutdown hook exist but
// before the post-registration sweep check, so a test can land a sweep in that gap.
let manualEnqueuePostWriteHookForTests: (() => void) | null = null;
function configKey(configDir?: string): string {
return configDir ?? "";
}
function dispatchDepsFor(configDir?: string): AutomationDispatchDeps {
return dispatchDepsByConfigDir.get(configKey(configDir))?.deps ?? {};
}
function cancellationCooldownUntil(policy: LabAutomationPolicyV1, now: number): number {
return now + Math.max(policy.failureCooldownMs, LAB_AUTOMATION_HARD_MAX.schedulerTickMs);
}
/**
* Register process-local dispatch authority for one runtime owner.
*
* The returned receipt releases only the authority that this call installed. If a same-root
* successor has already replaced it, releasing the predecessor is a no-op for the successor.
*/
export function setLabAutomationDispatchDeps(deps: AutomationDispatchDeps): () => void {
const key = configKey(deps.configDir);
if (Object.keys(deps).length === 0) {
dispatchDepsByConfigDir.delete(key);
return () => {};
}
const token = Symbol("lab-automation-runtime-owner");
dispatchDepsByConfigDir.set(key, { token, deps });
const existingScheduler = schedulerTimers.get(key);
if (existingScheduler) existingScheduler.ownerToken = token;
let released = false;
let detachServerCleanup = () => {};
let detachShutdownHook = () => {};
const release = () => {
if (released) return;
released = true;
detachServerCleanup();
detachShutdownHook();
const current = dispatchDepsByConfigDir.get(key);
if (current?.token !== token) return;
dispatchDepsByConfigDir.delete(key);
const scheduler = schedulerTimers.get(key);
if (scheduler?.ownerToken === token) {
clearInterval(scheduler.timer);
schedulerTimers.delete(key);
}
};
detachServerCleanup = registerCurrentServerResourceCleanup(release);
// Shutdown teardown is registered here, at activation, so `server/lifecycle.ts` never has
// to import Lab in order to stop it. Scoped to this configDir, unlike the previous
// unscoped call from the shutdown path.
detachShutdownHook = registerOptionalShutdownHook(`lab-automation:${key}`, () => {
requestLabAutomationShutdown();
stopLabAutomationScheduler(deps.configDir);
});
return release;
}
export function isLabAutomationSchedulerRunning(configDir?: string): boolean {
return schedulerTimers.has(configKey(configDir));
}
export function requestLabAutomationShutdown(): void {
shutdownRequested = true;
for (const controller of inFlightControllers.values()) {
controller.abort(new Error("cancelled"));
}
}
export function buildLabAutomationStatus(configDir?: string): LabAutomationStatusV1 {
const { policy, routes } = loadLabAutomationConfig(configDir);
const state = loadLabAutomationState(configDir);
const now = Date.now();
const hourAgo = now - LAB_AUTOMATION_HARD_MAX.budgetWindowMs;
const completedLastHour = state.runs.filter((row) => row.state === "completed" && (row.completedAt ?? 0) >= hourAgo).length;
const blockedLastHour = state.runs.filter((row) => row.state === "blocked" && (row.completedAt ?? 0) >= hourAgo).length;
return {
policy,
routes,
counters: {
queued: countRunsByState(state, "queued"),
running: countRunsByState(state, "running"),
completedLastHour,
blockedLastHour,
remainingRunBudget: runBudgetRemaining(policy, state, now),
remainingLiveRequestBudget: liveRequestBudgetRemaining(policy, state, now),
},
schedulerRunning: isLabAutomationSchedulerRunning(configDir),
};
}
function scheduledEligibilityCode(
policy: LabAutomationPolicyV1,
routes: LabAutomationRoutesV1,
run: LabAutomationRunRecordV1,
): string | null {
if (run.trigger !== "scheduled") return null;
if (!policy.enabled) return "automation_disabled";
switch (run.evidenceLayer) {
case "protocol_conformance":
return policy.layers.protocolConformance ? null : "layer_disabled";
case "live_route_compatibility": {
if (!policy.layers.liveRouteCompatibility) return "layer_disabled";
if (!run.providerName || !run.modelId) return "route_ineligible";
const enrolled = routes.routes.some((route) =>
route.providerName === run.providerName && route.modelId === run.modelId
);
return enrolled ? null : "route_ineligible";
}
case "task_effectiveness":
if (!policy.layers.taskEffectiveness) return "layer_disabled";
return policy.taskEffectivenessBackgroundEnabled ? null : "task_background_disabled";
}
}
function reconcileQueuedState(
state: ReturnType<typeof loadLabAutomationState>,
policy: LabAutomationPolicyV1,
routes: LabAutomationRoutesV1,
now: number,
): ReturnType<typeof loadLabAutomationState> {
let next = state;
for (const run of state.runs) {
if (run.state !== "queued" || run.trigger !== "scheduled") continue;
const code = scheduledEligibilityCode(policy, routes, run);
if (code) next = transitionRun(next, run.runId, "cancelled", now, code);
}
return next;
}
/** Apply current policy/route enrollment to already queued scheduled work without executing it. */
export function reconcileLabAutomationQueue(configDir?: string): void {
const { policy, routes } = loadLabAutomationConfig(configDir);
const now = Date.now();
mutateLabAutomationState(configDir, (state) => ({
state: trimTerminalRuns(reconcileQueuedState(state, policy, routes, now), now),
value: undefined,
}));
}
function loadPlannerConfig(configDir?: string): import("../../types").OcxConfig | undefined {
const runtimeDeps = dispatchDepsFor(configDir);
try {
return runtimeDeps.loadConfig?.() ?? readConfigDiagnostics().config;
} catch {
return undefined;
}
}
/**
* Scheduled planning is projection-backed. Rebuild once when the disposable SQLite projection
* is missing/incompatible; if replay/rebuild cannot establish a readable projection, fail closed
* instead of treating every scenario as missing and generating provider traffic.
*/
function ensureAutomationProjection(configDir?: string): boolean {
const status = queryLabStatus(configDir);
if (status.projectionAvailable) return true;
try {
rebuildLabProjection(configDir);
} catch {
return false;
}
return queryLabStatus(configDir).projectionAvailable;
}
function claimNextRun(
policy: LabAutomationPolicyV1,
routes: LabAutomationRoutesV1,
planned: ReturnType<typeof planLabAutomationRuns>,
enqueuePlan: boolean,
configDir: string | undefined,
now: number,
manualRunId?: string,
): LabAutomationRunRecordV1 | null {
return mutateLabAutomationState(configDir, (loaded) => {
let state = rollBudgetWindow(loaded, now);
if (policy.enabled && manualRunId === undefined && enqueuePlan) {
state = enqueuePlannedRuns(state, planned, "scheduled", now);
}
state = reconcileQueuedState(state, policy, routes, now);
if (isRunBudgetExhausted(policy, state, now)) {
return { state: trimTerminalRuns(state, now), value: null };
}
const predicate = (run: LabAutomationRunRecordV1): boolean => {
if (manualRunId !== undefined) return run.runId === manualRunId && run.trigger === "manual";
return policy.enabled && run.trigger === "scheduled";
};
const dispatchable = selectDispatchableRuns(policy, state, now, (run) => {
if (!predicate(run)) return false;
if (run.trigger === "scheduled" && !canReserveScheduledCooldown(state, run.runKey, now)) {
return false;
}
if (run.evidenceLayer === "live_route_compatibility" && isLiveRequestBudgetExhausted(policy, state, now)) {
return false;
}
return scheduledEligibilityCode(policy, routes, run) === null;
});
const selected = dispatchable[0];
if (!selected) return { state: trimTerminalRuns(state, now), value: null };
state = transitionRun(state, selected.runId, "running", now);
const running = state.runs.find((run) => run.runId === selected.runId) ?? null;
return { state: trimTerminalRuns(state, now), value: running };
});
}
function finalizeRun(
run: LabAutomationRunRecordV1,
policy: LabAutomationPolicyV1,
configDir: string | undefined,
result: DispatchResult | null,
error: unknown,
cancelled: boolean,
): void {
const completedAt = Date.now();
mutateLabAutomationState(configDir, (state) => {
const current = state.runs.find((row) => row.runId === run.runId);
if (!current || current.state !== "running") {
return { state: trimTerminalRuns(state, completedAt), value: undefined };
}
let next = state;
if (cancelled || (error instanceof LabAutomationError && error.code === "cancelled")) {
next = transitionRun(next, run.runId, "cancelled", completedAt, "cancelled");
if (run.trigger === "scheduled") {
next = setCooldown(
next,
run.runKey,
cancellationCooldownUntil(policy, completedAt),
completedAt,
);
}
return { state: trimTerminalRuns(next, completedAt), value: undefined };
}
if (result) {
next = transitionRun(next, run.runId, result.terminalState, completedAt, result.terminalCode);
if (result.terminalState === "completed") {
next = clearCooldown(next, run.runKey);
} else {
next = setCooldown(
next,
run.runKey,
cooldownForFailure(policy, result.cooldownCode, completedAt),
completedAt,
);
}
return { state: trimTerminalRuns(next, completedAt), value: undefined };
}
const terminalCode = error instanceof LabAutomationError ? error.code : "dispatch_failure";
next = transitionRun(next, run.runId, "failed", completedAt, terminalCode);
next = setCooldown(
next,
run.runKey,
cooldownForFailure(policy, "harness_failure", completedAt),
completedAt,
);
return { state: trimTerminalRuns(next, completedAt), value: undefined };
});
}
async function runDispatchBatch(
configDir?: string,
options: { manualRunId?: string; abortSignal?: AbortSignal } = {},
): Promise<void> {
if (shutdownRequested) return;
const initialConfig = loadLabAutomationConfig(configDir);
const initialPolicy = initialConfig.policy;
const initialRoutes = initialConfig.routes;
if (initialPolicy.enabled && options.manualRunId === undefined && !ensureAutomationProjection(configDir)) {
reconcileLabAutomationQueue(configDir);
return;
}
const config = loadPlannerConfig(configDir);
const snapshot = loadLabAutomationState(configDir);
const planned = initialPolicy.enabled && options.manualRunId === undefined
? planLabAutomationRuns({
policy: initialPolicy,
routes: initialRoutes,
state: snapshot,
now: Date.now(),
config,
configDir,
})
: [];
let enqueuePlan = true;
// Dispatch is intentionally sequential inside one tick. The concurrency fields still bound
// atomic selection against already-running work and future parallel workers/processes.
const maxDispatches = options.manualRunId ? 1 : Math.max(1, initialPolicy.maxConcurrentRuns);
for (let dispatched = 0; dispatched < maxDispatches; dispatched += 1) {
if (shutdownRequested) break;
const { policy, routes } = loadLabAutomationConfig(configDir);
const run = claimNextRun(
policy,
routes,
planned,
enqueuePlan,
configDir,
Date.now(),
options.manualRunId,
);
enqueuePlan = false;
if (!run) break;
const controller = new AbortController();
const abortFromCaller = () => controller.abort(options.abortSignal?.reason ?? new Error("cancelled"));
if (options.abortSignal?.aborted) abortFromCaller();
else options.abortSignal?.addEventListener("abort", abortFromCaller, { once: true });
inFlightControllers.set(run.runId, controller);
let result: DispatchResult | null = null;
let error: unknown;
try {
const runtimeDeps = dispatchDepsFor(configDir);
const deps: AutomationDispatchDeps = {
...runtimeDeps,
configDir,
loadConfig: runtimeDeps.loadConfig ?? (() => readConfigDiagnostics().config),
abortSignal: controller.signal,
enforceRunIdentity: true,
};
result = await dispatchLabAutomationRun(run, deps);
} catch (caught) {
error = caught;
} finally {
options.abortSignal?.removeEventListener("abort", abortFromCaller);
const cancelled = cancellingRunIds.has(run.runId) || controller.signal.aborted;
finalizeRun(run, policy, configDir, result, error, cancelled);
inFlightControllers.delete(run.runId);
cancellingRunIds.delete(run.runId);
}
if (options.manualRunId !== undefined) break;
}
}
/** Single bounded scheduler tick — plan, enqueue, dispatch within concurrency limits. */
export async function runLabAutomationTick(configDir?: string): Promise<void> {
const key = configKey(configDir);
if (ticksInProgress.has(key)) return;
ticksInProgress.add(key);
try {
await runDispatchBatch(configDir);
} finally {
ticksInProgress.delete(key);
}
}
export function startLabAutomationScheduler(configDir?: string): void {
const key = configKey(configDir);
// Same late-request race as enqueueManualLabRun: a policy PUT or CLI enable can resume
// after the shutdown sweep already ran. Starting here would reset the latch and leave a
// live interval dispatching Lab work outside the completed sweep.
if (didRunOptionalShutdownHooks()) {
requestLabAutomationShutdown();
return;
}
const currentOwner = dispatchDepsByConfigDir.get(key)?.token;
const existing = schedulerTimers.get(key);
if (existing) {
if (currentOwner) existing.ownerToken = currentOwner;
return;
}
// The scheduler owns a live interval, so its teardown must be registered here rather
// than only in setLabAutomationDispatchDeps: the management API and the CLI can start a
// scheduler without ever installing dispatch deps (lab-automation-routes.ts
// applySchedulerPolicy, cli/lab.ts), and core no longer imports this module to stop it.
// Without this registration such a scheduler survives drainAndShutdown.
registerOptionalShutdownHook(`lab-automation-scheduler:${key}`, () => {
requestLabAutomationShutdown();
stopLabAutomationScheduler(configDir);
});
// The sweep may have run in the gap between the entry check and this registration; it
// snapshots the registry once, so a hook that landed afterwards is orphaned. Keep the
// latch set and never start the timer — the registration stays live so a repeat sweep
// still tears this scheduler down.
if (didRunOptionalShutdownHooks()) {
requestLabAutomationShutdown();
return;
}
shutdownRequested = false;
const { policy, routes } = loadLabAutomationConfig(configDir);
const now = Date.now();
mutateLabAutomationState(configDir, (state) => {
let next = recoverLabAutomationState(policy, state, now);
next = reconcileQueuedState(next, policy, routes, now);
return { state: trimTerminalRuns(next, now), value: undefined };
});
const timer = setInterval(() => {
void runLabAutomationTick(configDir);
}, LAB_AUTOMATION_HARD_MAX.schedulerTickMs);
timer.unref?.();
schedulerTimers.set(key, {
timer,
ownerToken: currentOwner ?? Symbol("lab-automation-manual-scheduler"),
});
}
/** Stop periodic scheduling only. Process shutdown is a separate explicit signal. */
export function stopLabAutomationScheduler(configDir?: string): void {
if (configDir !== undefined) {
const key = configKey(configDir);
const scheduler = schedulerTimers.get(key);
if (scheduler) clearInterval(scheduler.timer);
schedulerTimers.delete(key);
return;
}
for (const scheduler of schedulerTimers.values()) clearInterval(scheduler.timer);
schedulerTimers.clear();
}
/** Test-only reset of scheduler globals between isolated automation tests. */
export function resetLabAutomationSchedulerStateForTests(): void {
stopLabAutomationScheduler();
shutdownRequested = false;
ticksInProgress.clear();
dispatchDepsByConfigDir.clear();
inFlightControllers.clear();
cancellingRunIds.clear();
manualEnqueuePostWriteHookForTests = null;
}
/** Test-only seam for the gap between a manual run's queue write and its post-write sweep check. */
export function setLabAutomationManualEnqueuePostWriteHookForTests(hook: (() => void) | null): void {
manualEnqueuePostWriteHookForTests = hook;
}
export async function enqueueManualLabRun(
planned: import("./types").PlannedLabRunV1,
configDir?: string,
abortSignal?: AbortSignal,
): Promise<LabAutomationRunRecordV1 | null> {
// The management route is not covered by the data-plane drain gate, so a request accepted
// before listener teardown can reach this point after the shutdown sweep already ran. A
// hook registered then is never invoked; run its teardown inline instead of letting the
// dispatch escape shutdown.
if (didRunOptionalShutdownHooks()) {
requestLabAutomationShutdown();
return null;
}
const now = Date.now();
const created = mutateLabAutomationState(configDir, (state) => {
const next = enqueuePlannedRuns(state, [planned], "manual", now);
const run = next.runs.find((row) =>
row.runKey === planned.runKey && row.trigger === "manual" && row.state === "queued"
) ?? null;
return { state: next, value: run };
});
if (!created) return null;
// Manual execution can run without activation or a scheduler, so it must own a shutdown
// hook for the lifetime of its dispatch instead of relying on either of those paths.
const detachShutdownHook = registerOptionalShutdownHook(
`lab-automation-manual:${created.runId}`,
requestLabAutomationShutdown,
);
// The sweep may have run in the gap between the entry check and this registration; it
// snapshots the registry once, so a hook that landed afterwards is orphaned.
manualEnqueuePostWriteHookForTests?.();
if (didRunOptionalShutdownHooks()) {
requestLabAutomationShutdown();
detachShutdownHook();
// The queue row was already written; leaving it `queued` would persist a manual run no
// scheduler tick ever picks up while the route reports success. Cancel it and fail.
cancelLabAutomationRun(created.runId, configDir);
return null;
}
try {
// Manual execution is independent of automation enablement/layer toggles.
await runDispatchBatch(configDir, { manualRunId: created.runId, abortSignal });
} finally {
detachShutdownHook();
}
return loadLabAutomationState(configDir).runs.find((row) => row.runId === created.runId) ?? created;
}
export function cancelLabAutomationRun(runId: string, configDir?: string): boolean {
const controller = inFlightControllers.get(runId);
if (controller) {
cancellingRunIds.add(runId);
controller.abort(new Error("cancelled"));
return true;
}
const policy = loadLabAutomationConfig(configDir).policy;
const now = Date.now();
return mutateLabAutomationState(configDir, (state) => {
const run = state.runs.find((row) => row.runId === runId);
if (!run || run.state !== "queued") return { state, value: false };
let next = cancelQueuedRun(state, runId, now);
if (run.trigger === "scheduled") {
next = setCooldown(next, run.runKey, cancellationCooldownUntil(policy, now), now);
}
return { state: next, value: true };
});
}