@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
614 lines (569 loc) • 26.1 kB
text/typescript
/**
* Half-open recovery for a held account, and the pool-wide retry/probe budget
* that sits above it (#4546, wp3 follow-up).
*
* The transient hold (#4616) keeps a thread's binding while its account serves a
* 5xx streak, and detours requests to a healthy sibling. What it cannot answer is
* whether the held account is actually back: a soft-avoided account receives no
* traffic, so the two-success clearing rule can only fire through the "held"
* fallback, which hands the failing account back to every pinned thread at once.
* This module is the bounded trial that closes that gap -- a single-holder probe
* lease keyed on the health domain (the quota-cooldown domain already has its own
* lease in src/codex/routing.ts and is a different thing).
*
* Three rules the lease enforces:
*
* - While an account is held, exactly one in-flight probe may test it. Every
* other request keeps the remembered detour, so a failed probe costs the
* caller nothing -- the detour's identity is never dropped to run the trial.
* - The lease has a deadline and is released on success, failure, or expiry. A
* response that arrives after its lease was lost is STALE: it must not
* overwrite a newer binding or a newer failure state, so every lease carries
* the generation it was issued under and a settle that fails the fence
* mutates nothing.
* - When every candidate is held the caller gets a typed "binding remembered,
* dispatch withheld" outcome -- not a send to an account already known to be
* failing.
*
* The pool-wide limiter exists because per-request send budgets do not prevent
* a retry storm: thousands of requests each staying inside their own allowance
* still compose into an unbounded rate against an already-failing upstream.
* Recovery dispatches (retries and probes, never the initial send of a new
* request) are admitted only while they stay under a ratio of observed initial
* sends in a sliding window -- the standard overload-guidance shape.
*/
/** How long a granted probe may be in flight before its lease is forfeit. */
export const TRANSIENT_PROBE_LEASE_MS = 30_000;
/**
* Minimum spacing between probes of the same held account. Without it every
* request that follows a settled probe becomes the next probe, which is the
* same storm the single-holder rule exists to bound -- just serialized.
*/
export const TRANSIENT_PROBE_INTERVAL_MS = 15_000;
/**
* Grace kept on top of an entry's pacing and lease deadlines before it may be forgotten.
* Inside it a late settle can still answer "expired" rather than "stale", which is the
* distinction the settle contract exists to report.
*/
const PROBE_STATE_RETENTION_MS = 60_000;
/**
* Hard ceiling on remembered accounts. Pacing state is per account id, and account ids churn
* with configuration: without a ceiling a long-lived proxy accumulates one entry per id it
* ever probed. Above the ceiling the entries whose pacing lapses soonest are dropped, which
* at worst lets one dormant account be probed earlier than its interval; an entry holding a
* LIVE lease is never dropped, because that would hand out a second concurrent probe and
* break the single-holder rule the lease exists to enforce.
*/
export const MAX_TRANSIENT_PROBE_STATES = 1_024;
/** Below this the map is too small to be worth scanning on a grant. */
const PROBE_STATE_SWEEP_THRESHOLD = 64;
/**
* Eviction target once the ceiling is reached. Clearing a block at a time keeps the ordering
* pass off the common grant path: it runs once per block of new accounts instead of once per
* grant forever after the first time the ceiling is touched.
*/
const PROBE_STATE_EVICTION_LOW_WATER = Math.floor(MAX_TRANSIENT_PROBE_STATES * 0.9);
export interface TransientProbeLease {
readonly accountId: string;
readonly leaseId: string;
/** Epoch the lease was issued under; a settle must match the CURRENT epoch. */
readonly generation: number;
readonly expiresAt: number;
}
export type TransientProbeOutcome = "recovered" | "failed";
/**
* What a settle did to the lease.
*
* - `applied`: the probe still held the lease inside its deadline; the caller
* may act on the outcome (clear the hold, or record the fresh failure).
* - `stale`: the lease was already lost -- expired and re-issued, or invalidated
* by newer authoritative state. The result is dropped; nothing is overwritten.
* - `expired`: the probe finished after its own deadline. The lease is dead
* either way; this answer exists so the caller can tell "lost a race" from
* "ran long".
*/
export type TransientProbeSettle = "applied" | "stale" | "expired";
interface AccountProbeState {
/** Bumped on every lease grant and every external invalidation. */
generation: number;
leaseId?: string;
leaseExpiresAt?: number;
lastProbeAt?: number;
/**
* Moment this account's pacing interval lapses, recorded at grant time from the interval
* that grant actually used. Kept alongside `lastProbeAt` so cleanup honours a caller's
* longer interval instead of assuming the default.
*/
pacedUntil?: number;
lastOutcome?: TransientProbeOutcome;
}
const probeStates = new Map<string, AccountProbeState>();
let probeLeaseSeq = 0;
function probeStateFor(accountId: string): AccountProbeState {
let state = probeStates.get(accountId);
if (!state) {
state = { generation: 0 };
probeStates.set(accountId, state);
}
return state;
}
function liveLease(state: AccountProbeState, now: number): boolean {
return state.leaseId !== undefined && state.leaseExpiresAt !== undefined && state.leaseExpiresAt > now;
}
/**
* Moment an entry stops carrying anything a future decision can read: its pacing interval and
* any unsettled lease deadline, plus the grace above.
*/
function probeStateRetiresAt(state: AccountProbeState): number {
return Math.max(state.pacedUntil ?? 0, state.leaseExpiresAt ?? 0) + PROBE_STATE_RETENTION_MS;
}
/**
* Bound the remembered accounts. Called on the one path that can grow the map -- a grant is
* the only insertion -- so the ceiling holds without a timer.
*
* The first pass drops only entries that can no longer change an answer: no live lease, the
* pacing interval lapsed, and the grace elapsed. Re-creating such an entry later yields the
* same decisions it would have produced, and a late settle against it still cannot be applied
* because lease ids are issued from a monotonic counter and never repeat.
*/
function sweepProbeStates(now: number): void {
if (probeStates.size <= PROBE_STATE_SWEEP_THRESHOLD) return;
for (const [accountId, state] of probeStates) {
if (liveLease(state, now)) continue;
if (now >= probeStateRetiresAt(state)) probeStates.delete(accountId);
}
if (probeStates.size <= MAX_TRANSIENT_PROBE_STATES) return;
// Still over the ceiling with nothing retired: churn is faster than the retention window.
// Evict in retirement order so the entries closest to meaningless go first, and never one
// holding a live lease.
const evictable = Array.from(probeStates)
.filter(([, state]) => !liveLease(state, now))
.sort((a, b) => probeStateRetiresAt(a[1]) - probeStateRetiresAt(b[1]));
let excess = probeStates.size - PROBE_STATE_EVICTION_LOW_WATER;
for (const [accountId] of evictable) {
if (excess <= 0) break;
probeStates.delete(accountId);
excess -= 1;
}
}
/**
* Grant the single in-flight probe for a held account, or null when another
* probe is already out or the pacing interval has not elapsed. The grant bumps
* the epoch, so a result from any earlier lease is stale the moment it lands.
*/
export function tryAcquireTransientProbe(
accountId: string,
now = Date.now(),
options?: { leaseMs?: number; minIntervalMs?: number },
): TransientProbeLease | null {
const state = probeStateFor(accountId);
if (liveLease(state, now)) return null;
const interval = options?.minIntervalMs ?? TRANSIENT_PROBE_INTERVAL_MS;
if (state.lastProbeAt !== undefined && now - state.lastProbeAt < interval) return null;
const leaseMs = options?.leaseMs ?? TRANSIENT_PROBE_LEASE_MS;
const leaseId = `tprobe-${(probeLeaseSeq += 1).toString(36)}`;
const expiresAt = now + Math.max(1, leaseMs);
state.generation += 1;
state.leaseId = leaseId;
state.leaseExpiresAt = expiresAt;
state.lastProbeAt = now;
state.pacedUntil = now + Math.max(0, interval);
// After the grant, not before it: the entry this call just wrote holds a live lease and is
// therefore the one entry the sweep may never touch, so the ceiling is a real ceiling
// rather than "the ceiling plus whatever was inserted after the scan".
sweepProbeStates(now);
return {
accountId,
leaseId,
generation: state.generation,
expiresAt,
};
}
/** Side-effect-free mirror of {@link tryAcquireTransientProbe} eligibility. */
export function canAcquireTransientProbe(
accountId: string,
now = Date.now(),
options?: { minIntervalMs?: number },
): boolean {
const state = probeStates.get(accountId);
if (!state) return true;
if (liveLease(state, now)) return false;
const interval = options?.minIntervalMs ?? TRANSIENT_PROBE_INTERVAL_MS;
return state.lastProbeAt === undefined || now - state.lastProbeAt >= interval;
}
/**
* Report a probe's outcome. Only the current lease holder inside its deadline
* applies: anything else is a late answer from a probe that already lost, and
* dropping it is what keeps it from overwriting a newer binding or a newer
* failure state. An applied settle clears the lease so the next probe is paced
* by the interval, not by the expiry.
*/
export function settleTransientProbe(
lease: TransientProbeLease,
outcome: TransientProbeOutcome,
now = Date.now(),
): TransientProbeSettle {
const state = probeStates.get(lease.accountId);
if (!state || state.leaseId !== lease.leaseId || state.generation !== lease.generation) {
return "stale";
}
// `>=`, matching liveLease: at exactly the deadline the lease is already gone, so applying
// the outcome there would let a probe act on a lease the grant path would refuse to
// recognise -- two answers to the same instant.
if (now >= lease.expiresAt) return "expired";
state.leaseId = undefined;
state.leaseExpiresAt = undefined;
state.lastOutcome = outcome;
return "applied";
}
/**
* Hand a lease back with no outcome -- the probe never reached upstream, so
* there is nothing to record. Only the holder may release; a stale lease is
* already dead and needs no cleanup.
*/
export function releaseTransientProbe(lease: TransientProbeLease): void {
const state = probeStates.get(lease.accountId);
if (!state || state.leaseId !== lease.leaseId || state.generation !== lease.generation) return;
state.leaseId = undefined;
state.leaseExpiresAt = undefined;
}
/**
* Fence the epoch against newer authoritative state. A fresh failure recorded
* through the ordinary outcome path, or a binding that moved on, must not be
* overwritten by a probe result that was issued before it -- bumping the epoch
* makes every outstanding lease stale without waiting for its deadline.
*/
export function invalidateTransientProbe(accountId: string): void {
const state = probeStates.get(accountId);
if (!state) return;
state.generation += 1;
state.leaseId = undefined;
state.leaseExpiresAt = undefined;
}
export interface TransientProbeDiagnostics {
readonly held: boolean;
readonly generation: number;
readonly leaseId?: string;
readonly leaseExpiresAt?: number;
readonly lastProbeAt?: number;
readonly lastOutcome?: TransientProbeOutcome;
}
/** Current lease state for one account, for diagnostics. Never mutates. */
export function transientProbeDiagnostics(accountId: string, now = Date.now()): TransientProbeDiagnostics {
const state = probeStates.get(accountId);
if (!state) return { held: false, generation: 0 };
return {
held: liveLease(state, now),
generation: state.generation,
...(state.leaseId !== undefined ? { leaseId: state.leaseId, leaseExpiresAt: state.leaseExpiresAt } : {}),
...(state.lastProbeAt !== undefined ? { lastProbeAt: state.lastProbeAt } : {}),
...(state.lastOutcome !== undefined ? { lastOutcome: state.lastOutcome } : {}),
};
}
/** Test seam: lease state is module-global and must not leak between cases. */
export function clearTransientProbeLeasesForTests(): void {
probeStates.clear();
}
/**
* How many accounts currently carry probe state. Diagnostic, and the assertion surface for
* the {@link MAX_TRANSIENT_PROBE_STATES} bound.
*/
export function transientProbeStateCount(): number {
return probeStates.size;
}
/**
* What a request may do while its bound account is held.
*
* - `probe`: this caller holds the lease and may send ONE trial to the held
* account.
* - `detour`: a probe is already out (or was refused); keep the remembered
* detour. The detour's identity survives the whole probing window -- a failed
* trial must not cost the caller its working route.
* - `withheld`: every candidate is held. The binding is remembered and dispatch
* is refused; `retryAt` is the earliest moment a probe could next go out.
* Sending anyway here is exactly the "must not send, sends anyway" defect the
* hold was added to close.
*/
export type HeldAccountDispatch =
| { kind: "probe"; lease: TransientProbeLease }
| { kind: "detour"; accountId: string }
| { kind: "withheld"; boundAccountId: string; detourAccountId?: string; retryAt: number };
/**
* Decide what a request bound to a held account may do this turn. The probe is
* tried first -- somebody has to find out whether the account is back, and the
* lease guarantees it is exactly one somebody. Everyone else keeps the detour,
* and a caller with no detour left is told to wait rather than sent at an
* account already known to be failing.
*/
export function resolveHeldAccountDispatch(input: {
boundAccountId: string;
detourAccountId?: string;
now?: number;
leaseMs?: number;
minProbeIntervalMs?: number;
backpressure?: PoolBackpressureLimiter;
}): HeldAccountDispatch {
const now = input.now ?? Date.now();
const limiter = input.backpressure ?? sharedPoolBackpressure();
// The lease check runs before the budget charge: a probe another holder already has out is
// not a dispatch, and charging the pool for it would shrink the recovery budget by phantom
// sends. Between the check and the grant there is no await, so eligibility cannot change.
if (
canAcquireTransientProbe(input.boundAccountId, now, {
...(input.minProbeIntervalMs !== undefined ? { minIntervalMs: input.minProbeIntervalMs } : {}),
})
&& limiter.tryPermitProbeDispatch(now)
) {
const lease = tryAcquireTransientProbe(input.boundAccountId, now, {
...(input.leaseMs !== undefined ? { leaseMs: input.leaseMs } : {}),
...(input.minProbeIntervalMs !== undefined ? { minIntervalMs: input.minProbeIntervalMs } : {}),
});
if (lease) return { kind: "probe", lease };
}
if (input.detourAccountId !== undefined && input.detourAccountId !== input.boundAccountId) {
return { kind: "detour", accountId: input.detourAccountId };
}
return {
kind: "withheld",
boundAccountId: input.boundAccountId,
...(input.detourAccountId !== undefined ? { detourAccountId: input.detourAccountId } : {}),
// Both bounds, not just the probe pacing. A request refused by the RATIO has no probe state
// of its own yet, so `nextProbeAt` answered `now` and the refusal told the caller to try
// again immediately -- a withheld dispatch that busy-loops is the same load as the dispatch
// it refused. The limiter is the only thing that knows when its window moves.
retryAt: Math.max(
nextProbeAt(input.boundAccountId, now, input.minProbeIntervalMs),
limiter.nextRecoveryAt(now),
),
};
}
/** Earliest moment a probe of this account could next be granted. */
function nextProbeAt(accountId: string, now: number, minIntervalMs?: number): number {
const state = probeStates.get(accountId);
if (!state) return now;
const interval = minIntervalMs ?? TRANSIENT_PROBE_INTERVAL_MS;
const paced = state.lastProbeAt !== undefined ? state.lastProbeAt + interval : now;
const leased = liveLease(state, now) ? state.leaseExpiresAt! : now;
return Math.max(paced, leased);
}
/* ------------------------------------------------------------------ */
/* Pool-wide recovery backpressure */
/* ------------------------------------------------------------------ */
export interface PoolBackpressurePolicy {
/** Sliding window the ratio is measured over. */
readonly windowMs: number;
/**
* Recovery dispatches (retries + probes) admitted per observed initial send.
* 0.2 is the standard overload-guidance budget: at most one recovery send for
* every five new requests.
*/
readonly maxRetryRatio: number;
/**
* Floor under the ratio so a quiet pool can still recover: with almost no
* traffic a strict ratio admits nothing, which would wedge every held
* account behind a probe that can never run.
*/
readonly minRecoveryAllowance: number;
}
export const DEFAULT_POOL_BACKPRESSURE_POLICY: PoolBackpressurePolicy = {
windowMs: 10_000,
maxRetryRatio: 0.2,
minRecoveryAllowance: 3,
};
export interface PoolBackpressureState {
readonly windowMs: number;
readonly initialSends: number;
readonly recoveryDispatches: number;
/** Dispatches admitted under the current window's allowance. */
readonly allowance: number;
/** Lifetime refusals, including windows already rotated out. */
readonly refusedTotal: number;
readonly ratioLimit: number;
}
export interface PoolBackpressureLimiter {
/** A new request's FIRST send. Always recorded, never refused. */
recordInitialSend(now?: number): void;
/** Admit one retry dispatch, or refuse when the window's ratio is spent. */
tryPermitRetryDispatch(now?: number): boolean;
/** Admit one probe dispatch under the same shared recovery budget. */
tryPermitProbeDispatch(now?: number): boolean;
/**
* Earliest moment this limiter could admit another recovery dispatch.
*
* A refusal has to hand back a time, or the caller has nothing to wait on and busy-loops
* against a pool that is already failing -- which is the load this limiter exists to remove.
* `now` when the allowance is not spent; otherwise the moment the oldest bucket still inside
* the window falls out of it, which is strictly in the future and is a real change point
* rather than a guess.
*/
nextRecoveryAt(now?: number): number;
state(now?: number): PoolBackpressureState;
}
const BACKPRESSURE_BUCKETS = 10;
/**
* Ratio limiter over a bucketed sliding window. Buckets give a sliding answer
* without keeping per-event state: the window is the sum of the buckets whose
* span falls inside it, and grant/refuse decisions read that sum.
*/
export function createPoolBackpressureLimiter(
policy: PoolBackpressurePolicy = DEFAULT_POOL_BACKPRESSURE_POLICY,
): PoolBackpressureLimiter {
const bucketMs = Math.max(1, Math.floor(policy.windowMs / BACKPRESSURE_BUCKETS));
const buckets: Array<{ start: number; initials: number; recoveries: number }> = [];
let refusedTotal = 0;
function bucketFor(now: number): { start: number; initials: number; recoveries: number } {
const start = Math.floor(now / bucketMs) * bucketMs;
const last = buckets[buckets.length - 1];
if (last && last.start === start) return last;
while (buckets.length > 0 && buckets[0]!.start <= start - policy.windowMs) buckets.shift();
const bucket = { start, initials: 0, recoveries: 0 };
buckets.push(bucket);
return bucket;
}
function totals(now: number): { initials: number; recoveries: number } {
let initials = 0;
let recoveries = 0;
for (const bucket of buckets) {
if (bucket.start <= now - policy.windowMs) continue;
initials += bucket.initials;
recoveries += bucket.recoveries;
}
return { initials, recoveries };
}
function allowanceFor(initials: number): number {
return Math.max(policy.minRecoveryAllowance, Math.floor(initials * policy.maxRetryRatio));
}
function tryPermit(now: number): boolean {
const bucket = bucketFor(now);
const { initials, recoveries } = totals(now);
if (recoveries + 1 > allowanceFor(initials)) {
refusedTotal += 1;
return false;
}
bucket.recoveries += 1;
return true;
}
function nextRecoveryAt(now: number): number {
const { initials, recoveries } = totals(now);
if (recoveries + 1 <= allowanceFor(initials)) return now;
// The window has to move before another recovery fits. The earliest that can happen is the
// moment the oldest bucket still inside it leaves, and every such bucket started after
// `now - windowMs`, so the answer is always strictly in the future.
for (const bucket of buckets) {
if (bucket.start <= now - policy.windowMs) continue;
return bucket.start + policy.windowMs;
}
return now + policy.windowMs;
}
return {
recordInitialSend(now = Date.now()): void {
bucketFor(now).initials += 1;
},
tryPermitRetryDispatch(now = Date.now()): boolean {
return tryPermit(now);
},
tryPermitProbeDispatch(now = Date.now()): boolean {
return tryPermit(now);
},
nextRecoveryAt(now = Date.now()): number {
return nextRecoveryAt(now);
},
state(now = Date.now()): PoolBackpressureState {
const { initials, recoveries } = totals(now);
return {
windowMs: policy.windowMs,
initialSends: initials,
recoveryDispatches: recoveries,
allowance: allowanceFor(initials),
refusedTotal,
ratioLimit: policy.maxRetryRatio,
};
},
};
}
let sharedLimiter: PoolBackpressureLimiter | undefined;
/**
* The process-wide limiter every recovery dispatch shares. A per-request
* limiter cannot see the storm, which is the entire reason this layer exists.
*/
export function sharedPoolBackpressure(): PoolBackpressureLimiter {
sharedLimiter ??= createPoolBackpressureLimiter();
return sharedLimiter;
}
/**
* Point the shared limiter at a different policy. The ceiling is deliberately
* configurable here and not yet plumbed into OcxConfig -- the wiring lane owns
* that seam; this is the knob it turns.
*/
export function configureSharedPoolBackpressure(policy: PoolBackpressurePolicy): void {
sharedLimiter = createPoolBackpressureLimiter(policy);
}
/** Test seam: the shared limiter is module-global. */
export function resetSharedPoolBackpressureForTests(): void {
sharedLimiter = undefined;
}
/**
* Forget every account's probe pacing AND the shared recovery window.
*
* Called when the pool's routing state is reset wholesale -- a roster change, a config reload,
* an account removal. Both halves describe a pool that no longer exists: pacing is keyed on
* account ids that may be gone, and the window's buckets count sends made by a roster that
* changed underneath them. Keeping either across such a reset lets one context's recovery
* decisions govern the next one, which is also how it leaks between test files.
*
* This is the production reset. The two `ForTests` seams above stay separate because a test
* frequently wants exactly one half of it.
*/
export function clearPoolRecoveryState(): void {
probeStates.clear();
sharedLimiter = undefined;
}
/**
* What one physical send IS, as far as the recovery window is concerned.
*
* The window measures recovery traffic against observed demand, so it needs the distinction
* made where the send happens -- and the transport wrapper cannot make it. That layer sees a
* URL and an init; whether this is a conversation's first attempt, its third retry, or the one
* trial admitted against a held account is knowledge only the caller has. So the caller names
* it, and the classification lives here with the window rather than in the transport, which
* owns no routing policy and has an enforced import boundary saying so.
*
* - `initial`: a new request's first send. Recorded, never refused -- it is the denominator,
* and refusing it would make this a throughput cap rather than a recovery bound.
* - `retry`: a re-send of a request that already reached upstream once. Admitted only while
* recovery traffic stays under its ratio of observed demand.
* - `probe`: the half-open trial against a held account. It ALREADY paid at selection, inside
* {@link resolveHeldAccountDispatch}; charging it again would bill one send twice and shrink
* the very budget it was admitted from.
*/
export type PoolRecoveryDispatchClass = "initial" | "retry" | "probe";
export interface PoolRecoveryDispatchDecision {
readonly admitted: boolean;
/**
* Earliest moment another recovery dispatch could be admitted. `now` when the send was
* admitted; otherwise a real change point strictly in the future, so a refused caller has
* something to wait on instead of busy-looping against a pool that is already failing.
*/
readonly retryAt: number;
}
/**
* Admit one physical send against the process-wide recovery window.
*
* Per-request send budgets cannot see a storm: thousands of requests each staying inside their
* own allowance still compose into an unbounded rate against one failing upstream. This is the
* layer above them, and it is shared by construction.
*/
export function classifyPoolRecoveryDispatch(
dispatchClass: PoolRecoveryDispatchClass,
now = Date.now(),
limiter: PoolBackpressureLimiter = sharedPoolBackpressure(),
): PoolRecoveryDispatchDecision {
if (dispatchClass === "initial") {
limiter.recordInitialSend(now);
return { admitted: true, retryAt: now };
}
if (dispatchClass === "probe") return { admitted: true, retryAt: now };
return limiter.tryPermitRetryDispatch(now)
? { admitted: true, retryAt: now }
: { admitted: false, retryAt: limiter.nextRecoveryAt(now) };
}