@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
242 lines (227 loc) • 11.5 kB
text/typescript
/**
* Health-unresponsiveness recovery policy for the opt-in service health sentinel (#6671).
*
* Every supervisor this repository installs restarts the proxy only when it exits: the Windows
* Task Scheduler batch loop, launchd `KeepAlive`/`SuccessfulExit=false`, systemd
* `Restart=on-failure` and the desktop app's `on_exit`. A proxy that stays alive with its port
* listening while its event loop answers nothing is invisible to all of them. The sentinel turns
* that state into an exit, and this module is the whole of its judgement: pure, clock-injected,
* and free of I/O so every false-positive guard is testable on any platform.
*
* The guards, in the order they apply:
* - nothing counts until the target has answered once as itself and the startup grace is over;
* the grace is elapsed time, so setting the clock back neither restarts nor extends it;
* - only a probe that got no complete answer by its deadline counts. Any answer proves the event
* loop ran, and a refused or failed connection means the listener is closing, not hung;
* - an answer from another pid means the port is no longer the target's: stand down for good;
* - a probe that started far later than the cadence allows, that outlived its own deadline by
* far more than jitter, or during which the sentinel's own event loop ran late (sleep, or the
* sentinel itself paused or starved), discards the streak, because the timeout may belong to
* that pause rather than to the target;
* - once a probe times out the next ones follow back to back, so a run of timeouts is one
* continuous unresponsive stretch rather than separate stalls that line up with the cadence;
* - recovery needs both a run of consecutive timeouts and a minimum unhealthy span, which doubles
* for each recovery in the last hour, and a cap on recoveries per hour stands the sentinel down:
* a proxy that hangs again right after every restart is not fixed by killing it, and each kill
* loses the work it held. Nothing is ever replayed.
*/
export type HealthProbeOutcome =
/** A complete status line and headers arrived before the deadline. */
| { kind: "answered"; pid: number | null; startedAt: number; endedAt: number }
/**
* No complete answer by the deadline, whether or not the connection was accepted.
* `sentinelLagMs` is how late the sentinel's own event loop ran during the probe: a sentinel
* that was itself paused or starved cannot tell a silent target from its own silence.
*/
| { kind: "timeout"; startedAt: number; endedAt: number; sentinelLagMs?: number }
/** The connection was refused: nothing is listening. */
| { kind: "refused"; startedAt: number; endedAt: number }
/** Any other failure. */
| { kind: "error"; startedAt: number; endedAt: number };
export type HealthSentinelTiming = {
/** Time between the end of one answered probe and the start of the next. */
intervalMs: number;
/**
* Time between a timed-out probe and the next. Short on purpose: back-to-back probes make a run
* of timeouts mean one continuous unresponsive stretch, so recurring shorter stalls that happen
* to line up with the cadence cannot add up to a recovery.
*/
failureIntervalMs: number;
/** A probe without a complete answer by this deadline is a timeout. */
probeTimeoutMs: number;
/** Counted from the sentinel's start; nothing counts before it ends. */
startupGraceMs: number;
/** Consecutive timeouts needed before a recovery. */
failureThreshold: number;
/** Minimum span from the first failing probe's start to the last one's end, before backoff. */
minUnhealthyMs: number;
/** Slack over the cadence (and over a probe's deadline) before a pause voids the streak. */
suspendSlackMs: number;
/** A timeout during which the sentinel's own loop ran later than this is not evidence. */
lagToleranceMs: number;
/** Window over which earlier recoveries raise the backoff and count toward the cap. */
recoveryWindowMs: number;
/** Recoveries within the window after which the sentinel stands down. */
maxRecoveriesPerWindow: number;
/** Ceiling on the backoff multiplier. */
maxBackoffMultiplier: number;
/**
* Oldest the final failing observation may be when the kill is issued. Lease, history and log
* work runs between the decision and the kill; evidence older than this is discarded.
*/
maxEvidenceAgeMs: number;
};
export const DEFAULT_HEALTH_SENTINEL_TIMING: Readonly<HealthSentinelTiming> = Object.freeze({
intervalMs: 30_000,
failureIntervalMs: 1_000,
probeTimeoutMs: 20_000,
startupGraceMs: 120_000,
failureThreshold: 3,
minUnhealthyMs: 60_000,
suspendSlackMs: 5_000,
lagToleranceMs: 2_000,
recoveryWindowMs: 60 * 60_000,
maxRecoveriesPerWindow: 3,
maxBackoffMultiplier: 8,
maxEvidenceAgeMs: 5_000,
});
export type HealthSentinelDecision =
| { action: "continue"; note?: string }
| { action: "recover"; reason: string; consecutiveTimeouts: number; unhealthyMs: number }
/** Stop acting for this target; the reason is logged once. */
| { action: "stand_down"; reason: string };
export type HealthSentinelPolicyState = {
readonly armed: boolean;
readonly consecutiveTimeouts: number;
readonly firstFailureStartedAt: number | null;
readonly lastObservedAt: number | null;
readonly stoodDown: string | null;
};
export type HealthSentinelPolicy = {
observe(outcome: HealthProbeOutcome): HealthSentinelDecision;
state(): HealthSentinelPolicyState;
/** Minimum unhealthy span currently required, after backoff. */
requiredUnhealthyMs(): number;
/** How long to wait before the next probe: short while a failure streak is open. */
nextDelayMs(): number;
};
/**
* Recoveries recorded inside the window ending at `now`. An entry dated after `now` counts as
* recent: it means the wall clock was set back after it was written, and dropping it would let a
* clock rollback (or a restart after one) lift the hourly cap. It keeps counting until the clock
* passes it by a full window, which errs toward not killing.
*/
export function recentRecoveryCount(history: readonly number[], now: number, windowMs: number): number {
return history.filter(at => Number.isFinite(at) && now - at < windowMs).length;
}
/** The unhealthy span required after `recent` recoveries: doubled per recovery, capped. */
export function requiredUnhealthySpan(timing: HealthSentinelTiming, recent: number): number {
const multiplier = Math.min(timing.maxBackoffMultiplier, 2 ** Math.max(0, recent));
return timing.minUnhealthyMs * multiplier;
}
export function createHealthSentinelPolicy(input: {
targetPid: number;
startedAt: number;
/** Epoch-ms of earlier recoveries for this installation, newest last. */
recoveryHistory: readonly number[];
timing?: Partial<HealthSentinelTiming>;
}): HealthSentinelPolicy {
const timing: HealthSentinelTiming = { ...DEFAULT_HEALTH_SENTINEL_TIMING, ...input.timing };
let armed = false;
let streak = 0;
let firstFailureStartedAt: number | null = null;
let lastObservedAt: number | null = null;
let stoodDown: string | null = null;
let required = timing.minUnhealthyMs;
// Startup grace is measured as elapsed time, accumulated from forward clock movement only, so a
// wall clock set back after (or during) the grace cannot reopen it and hold off recovery for as
// long as the clock was moved.
let graceElapsedMs = 0;
let graceClock = input.startedAt;
const noteGraceProgress = (at: number) => {
if (at > graceClock) graceElapsedMs += at - graceClock;
graceClock = at;
};
const resetStreak = () => {
streak = 0;
firstFailureStartedAt = null;
};
const standDown = (reason: string): HealthSentinelDecision => {
stoodDown = reason;
resetStreak();
return { action: "stand_down", reason };
};
const observe = (outcome: HealthProbeOutcome): HealthSentinelDecision => {
if (stoodDown !== null) return { action: "stand_down", reason: stoodDown };
noteGraceProgress(outcome.startedAt);
const previous = lastObservedAt;
lastObservedAt = outcome.endedAt;
let note: string | undefined;
if (previous !== null && streak > 0 && outcome.startedAt - previous > timing.failureIntervalMs + timing.suspendSlackMs) {
// The sentinel overslept its cadence (a suspended machine, or the sentinel itself starved),
// so the timeouts counted so far may belong to that pause rather than to the target.
resetStreak();
note = "observation gap; failure streak discarded";
}
switch (outcome.kind) {
case "answered": {
if (outcome.pid !== null && outcome.pid !== input.targetPid) {
return standDown("another process answers the health endpoint");
}
if (outcome.pid === input.targetPid) armed = true;
resetStreak();
return { action: "continue", note };
}
case "refused":
case "error":
resetStreak();
return { action: "continue", note };
case "timeout": {
if (!armed) return { action: "continue", note: note ?? "not armed: the target has not answered yet" };
if (graceElapsedMs < timing.startupGraceMs) {
return { action: "continue", note: note ?? "startup grace" };
}
if (
outcome.endedAt - outcome.startedAt > timing.probeTimeoutMs + timing.suspendSlackMs
|| outcome.endedAt < outcome.startedAt
|| (previous !== null && outcome.startedAt < previous)
) {
// The probe outlived its own deadline by far more than scheduling jitter, or the wall
// clock ran backwards: its timing spanned a pause or a clock step, so this timeout says
// nothing about the target.
resetStreak();
return { action: "continue", note: "probe spanned a pause; failure streak discarded" };
}
if ((outcome.sentinelLagMs ?? 0) > timing.lagToleranceMs) {
// The sentinel itself stalled during the probe (paused, starved, or the machine slept),
// so the deadline may have expired while nobody was listening for the answer.
resetStreak();
return { action: "continue", note: "probe spanned a pause; failure streak discarded" };
}
streak += 1;
if (firstFailureStartedAt === null) firstFailureStartedAt = outcome.startedAt;
const unhealthyMs = outcome.endedAt - firstFailureStartedAt;
const recent = recentRecoveryCount(input.recoveryHistory, outcome.endedAt, timing.recoveryWindowMs);
required = requiredUnhealthySpan(timing, recent);
if (streak < timing.failureThreshold || unhealthyMs < required) return { action: "continue", note };
if (recent >= timing.maxRecoveriesPerWindow) {
return standDown(`${recent} recoveries in the last ${Math.round(timing.recoveryWindowMs / 60_000)} minutes; not recovering again`);
}
const decision: HealthSentinelDecision = {
action: "recover",
reason: `no health answer for ${streak} consecutive probes over ${Math.round(unhealthyMs / 1000)}s`,
consecutiveTimeouts: streak,
unhealthyMs,
};
resetStreak();
return decision;
}
}
};
return {
observe,
state: () => ({ armed, consecutiveTimeouts: streak, firstFailureStartedAt, lastObservedAt, stoodDown }),
requiredUnhealthyMs: () => required,
nextDelayMs: () => (streak > 0 ? timing.failureIntervalMs : timing.intervalMs),
};
}