@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
865 lines (816 loc) • 37.8 kB
text/typescript
import { errorDestinationText, providerErrorDestinationLabel } from "../lib/error-destination";
import type { OcxComboTarget, OcxConfig } from "../types";
import { getCachedProviderRoutingQuota } from "../providers/quota-routing-cache";
import type { ProviderQuota, ProviderQuotaWindow } from "../providers/quota-types";
import { sleepWithAbort } from "../lib/upstream-retry";
import {
clearComboTargetCooldownIfUnchanged,
comboTargetCooldownRevision,
comboTargetCooldownStatus,
coolComboTarget,
earliestComboCooldown,
isComboTargetInCooldown,
snapshotComboQuotaCooldowns as snapshotTargetQuotaCooldowns,
type ComboFailureCooldownScope,
} from "./failover";
import { quotaResetRemainingMs } from "./reset-window";
import { getCombo, resolveComboId, targetKey } from "./types";
import type { NormalizedComboConfig, NormalizedComboTarget } from "./types";
import {
captureConfigGeneration,
type GenerationContext,
} from "../lib/state-store-sweeper";
export interface ComboPick {
comboId: string;
target: NormalizedComboTarget;
targetIndex: number;
attempted: string[];
writerGeneration: number;
/** Cooled targets this request alone may probe, each bound to the cooldown write it reserved. */
cooldownReprobeTargets?: Record<string, number>;
}
interface SelectionState {
lastCooldownReprobeAt?: number;
activeKey?: string;
successes: number;
currentWeights: Map<string, number>;
successfulUses: Map<string, number>;
}
const selectionState = new Map<string, SelectionState>();
let lastReconciledGeneration = 0;
let liveComboTargets = new Set<string>();
function comboTargetOwnerKey(comboId: string, key: string): string {
return `${comboId}::${key}`;
}
function mayCommitComboState(comboId: string, key: string, writerGeneration: number): boolean {
return writerGeneration >= lastReconciledGeneration
|| liveComboTargets.has(comboTargetOwnerKey(comboId, key));
}
export class UnknownComboError extends Error {
constructor(readonly comboId: string) {
super(`Unknown combo: ${comboId}`);
this.name = "UnknownComboError";
}
}
declare const exclusionBrand: unique symbol;
/** Service diagnostics name configured positions and validated providers only. */
export function comboLogLabel(config: OcxConfig, comboId: string): string {
const index = Object.keys(config.combos ?? {}).indexOf(comboId);
return index < 0 ? "combo" : `combo ${index + 1}`;
}
export function comboTargetLogLabel(config: OcxConfig, comboId: string, target: OcxComboTarget): string {
const index = getCombo(config, comboId)?.targets.findIndex(item => targetKey(item) === targetKey(target)) ?? -1;
return `target ${index + 1} (${errorDestinationText(providerErrorDestinationLabel(target.provider))})`;
}
export type ComboExclusionReasons = string & { readonly [exclusionBrand]: true };
/** One target's coarse failure class in a `combo_unavailable` answer (#7965). */
export type ComboTargetUnavailableCause = "quota" | "permission" | "transient" | "unsupported" | "unavailable";
/** The combo-level class: the one class every classified target shares, `mixed` when they differ, `unknown` when none was classified. */
export type ComboUnavailableCause = ComboTargetUnavailableCause | "mixed" | "unknown";
/**
* Machine-readable fields on the public `combo_unavailable` error (#7965). Counts and closed classes
* only: no combo id, provider name, model, account or upstream text.
*/
export interface ComboUnavailableDiagnostics {
readonly cause: ComboUnavailableCause;
/** Targets configured on the combo. */
readonly targets: number;
/** Targets this request dispatched before the answer. */
readonly attempted: number;
/** How many targets fell in each class; classes no target fell in are omitted. */
readonly causes: Readonly<Partial<Record<ComboTargetUnavailableCause, number>>>;
}
/** Fold per-target classes into the combo-level diagnostics. */
export function comboUnavailableDiagnostics(
targets: number,
attempted: number,
classes: readonly ComboTargetUnavailableCause[],
): ComboUnavailableDiagnostics {
const causes: Partial<Record<ComboTargetUnavailableCause, number>> = {};
for (const value of classes) causes[value] = (causes[value] ?? 0) + 1;
const distinct = Object.keys(causes) as ComboTargetUnavailableCause[];
const cause: ComboUnavailableCause = distinct.length === 0 ? "unknown" : distinct.length === 1 ? distinct[0]! : "mixed";
return { cause, targets, attempted, causes };
}
/** Why a combo had no target to pick: the readable message suffix and its machine-readable form. */
export interface ComboExclusion {
readonly reasons: ComboExclusionReasons;
readonly diagnostics: ComboUnavailableDiagnostics;
}
export class NoAvailableComboTargetsError extends Error {
readonly code = "combo_unavailable";
constructor(readonly comboId: string, readonly exclusion?: ComboExclusion) {
super(`No available targets for combo${exclusion ? ` (${exclusion.reasons})` : ""}`);
this.name = "NoAvailableComboTargetsError";
}
}
type ComboExclusionReason = "provider missing" | "provider disabled" | "quota exhausted" | "cooldown" | "ineligible for this request";
function providerExclusionReason(config: OcxConfig, target: OcxComboTarget, now: number): ComboExclusionReason | undefined {
const provider = Object.hasOwn(config.providers, target.provider) ? config.providers[target.provider] : undefined;
if (!provider) return "provider missing";
if (provider.disabled === true) return "provider disabled";
return cachedProviderQuotaIsExhausted(getCachedProviderRoutingQuota(target.provider, provider, now), now, target.model)
? "quota exhausted"
: undefined;
}
function targetProviderIsUsable(config: OcxConfig, target: OcxComboTarget, now: number): boolean {
return providerExclusionReason(config, target, now) === undefined;
}
/**
* OCX-11: the coarse fallback trigger for a target the picker passed over for a reason other
* than a cooldown (provider missing or disabled, cached quota spent, ineligible for the request).
*/
export function comboTargetSkipTrigger(
config: OcxConfig,
target: OcxComboTarget,
now = Date.now(),
): "quota_exhausted" | "unavailable" {
return providerExclusionReason(config, target, now) === "quota exhausted" ? "quota_exhausted" : "unavailable";
}
/**
* The class a skipped target contributes to `combo_unavailable` diagnostics. A cooldown is
* classified by the HTTP status its entry kept: 429/402 is quota, 401/403 is permission, and any
* other or missing status is transient.
*/
function exclusionCause(reason: ComboExclusionReason, comboId: string, target: OcxComboTarget, now: number): ComboTargetUnavailableCause {
if (reason === "quota exhausted") return "quota";
if (reason === "ineligible for this request") return "unsupported";
if (reason !== "cooldown") return "unavailable";
const status = comboTargetCooldownStatus(comboId, target, now);
return status === 429 || status === 402 ? "quota" : status === 401 || status === 403 ? "permission" : "transient";
}
/**
* Why each target of a combo cannot be picked right now, for a zero-attempt refusal. Only the
* target index, validated provider label and a closed exclusion reason are named; the same pass
* yields the machine-readable diagnostics.
*/
/** Why the picker would pass over one target, in picker order: provider, cooldown, then request eligibility. */
function targetExclusionReason(
config: OcxConfig,
comboId: string,
target: NormalizedComboTarget,
eligible: ((target: NormalizedComboTarget) => boolean) | undefined,
now: number,
): ComboExclusionReason | undefined {
let reason = providerExclusionReason(config, target, now);
const cooldown = reason ? undefined : earliestComboCooldown(comboId, [target], now);
if (cooldown) reason = "cooldown";
if (!reason && eligible) {
try {
if (!eligible(target)) reason = "ineligible for this request";
} catch {
// An eligibility check can exhaust the translator budget; the target still was not picked.
reason = "ineligible for this request";
}
}
return reason;
}
/**
* One target's `combo_unavailable` class, from the same checks and precedence as
* {@link comboExclusion}, or `undefined` when nothing excludes it.
*/
export function comboTargetUnavailableCause(
config: OcxConfig,
comboId: string,
target: NormalizedComboTarget,
eligible?: (target: NormalizedComboTarget) => boolean,
now = Date.now(),
): ComboTargetUnavailableCause | undefined {
const reason = targetExclusionReason(config, comboId, target, eligible, now);
return reason ? exclusionCause(reason, comboId, target, now) : undefined;
}
export function comboExclusion(
config: OcxConfig,
comboId: string,
eligible?: (target: NormalizedComboTarget) => boolean,
now = Date.now(),
): ComboExclusion {
const targets = getCombo(config, comboId)?.targets ?? [];
const classes: ComboTargetUnavailableCause[] = [];
const reasons = targets.flatMap((target, index) => {
const reason = targetExclusionReason(config, comboId, target, eligible, now);
if (!reason) return [];
classes.push(exclusionCause(reason, comboId, target, now));
return [`target ${index + 1} (${errorDestinationText(providerErrorDestinationLabel(target.provider))}): ${reason}`];
}).join("; ");
return {
reasons: `${targets.length} target${targets.length === 1 ? "" : "s"}${reasons ? `; ${reasons}` : ""}` as ComboExclusionReasons,
diagnostics: comboUnavailableDiagnostics(targets.length, 0, classes),
};
}
/** The readable half of {@link comboExclusion}. */
export function comboExclusionReasons(
config: OcxConfig,
comboId: string,
eligible?: (target: NormalizedComboTarget) => boolean,
now = Date.now(),
): ComboExclusionReasons {
return comboExclusion(config, comboId, eligible, now).reasons;
}
/** Capture usable providers' quota cooldowns; request eligibility is deferred to exhaustion. */
export function snapshotComboQuotaCooldowns(
config: OcxConfig,
comboId: string,
now = Date.now(),
): Array<{ target: NormalizedComboTarget; cooldownUntil: number }> {
const targets = getCombo(config, comboId)?.targets.filter(target =>
targetProviderIsUsable(config, target, now)) ?? [];
return snapshotTargetQuotaCooldowns(comboId, targets, now);
}
function quotaWindowExhausted(percent: number | undefined, resetAt: number | undefined, now: number): boolean {
if (typeof percent !== "number" || !Number.isFinite(percent) || percent < 100) return false;
return typeof resetAt !== "number" || !Number.isFinite(resetAt) || resetAt > now;
}
// `customWindows` is a generic carrier. Most labels are provider-wide counters ("Prepaid
// credits", "Free trial", "burst", "Spark") and must gate every model. Anthropic also rides it
// for per-model family counters, where a spent Opus week says nothing about a Sonnet request.
// `routing/quota.ts` already scopes those per model when it ranks accounts
// (`anthropicFamilyWindow`); this is the same fact applied to combo target selection.
//
// The scoping keys on `window.scope`, which the PRODUCER sets only where it proved model scope
// structurally, NEVER on the label text: `quota/antigravity.ts` forwards an upstream
// `group.displayName` unchanged, so a provider-wide group named "Opus" read as per-model would
// leave a spent window unenforced and send a doomed request. Every unproven direction keeps
// gating everything, which is the behaviour before this change: no scope on the window, no
// model on the request, or a family this gateway cannot match against a model id. Failing
// closed there costs at most a diverted target.
const MODEL_FAMILY_WINDOW_LABELS = new Set(["fable", "opus", "sonnet"]);
function customWindowAppliesToModel(window: ProviderQuotaWindow, model: string | undefined): boolean {
// Exact Antigravity model readings are diagnostic only. They are not permission or
// provider-wide exhaustion evidence until a separate eligibility contract consumes them.
if (window.modelId !== undefined) return false;
if (window.scope !== "model" || model === undefined) return true;
const family = window.label.trim().toLowerCase();
if (!MODEL_FAMILY_WINDOW_LABELS.has(family)) return true;
return model.toLowerCase().includes(family);
}
export function cachedProviderQuotaIsExhausted(
quota: ProviderQuota | null,
now = Date.now(),
model?: string,
): boolean {
if (!quota) return false;
if (quotaWindowExhausted(quota.fiveHourPercent, quota.fiveHourResetAt, now)) return true;
if (quotaWindowExhausted(quota.weeklyPercent, quota.weeklyResetAt, now)) return true;
if (quotaWindowExhausted(quota.monthlyPercent, quota.monthlyResetAt, now)) return true;
if (quota.customWindows?.some(window => customWindowAppliesToModel(window, model)
&& quotaWindowExhausted(window.percent, window.resetAt, now))) return true;
if (quota.creditsUsd?.unlimited !== true
&& typeof quota.creditsUsd?.percent === "number"
&& Number.isFinite(quota.creditsUsd.percent)
&& quota.creditsUsd.percent >= 100
&& quota.creditsUsd.remaining <= 0) return true;
return false;
}
/**
* Why a catalog row is offered but cannot currently serve a request (#1711).
*
* Only one reason exists today. It is a string rather than a boolean so a later cause — a
* cooldown, a revoked key — can be told apart by a consumer that already reads the field.
*/
export type QuotaInactiveReason = "no_credit";
/**
* `"no_credit"` when every USABLE target of a catalog row has positive exhaustion evidence
* (#1711), otherwise undefined.
*
* This reuses the credential-scoped evidence and exhaustion rules used by runtime selection.
* Display-only account, model-group and service windows cannot mark a whole provider inactive.
*
* Three rules carry the correctness, all inherited rather than restated:
*
* - A target the operator has removed or disabled is not usable and is not evidence either way;
* it drops out before the vote. If nothing is left, the row is unavailable for an operator
* reason rather than a quota one, so this returns undefined.
* - The canonical ChatGPT forward provider is exempt. Native account selection owns model-scoped
* quota, and a provider-level summary cannot veto it.
* - A stale cache is NOT exhaustion. `getCachedProviderRoutingQuota` returns null past its 30-minute
* window, and a null reading ends the vote rather than counting as evidence, so an unprobed
* provider is never marked inactive.
*
* "Every" is the bar on purpose: one target that can still serve makes the row serviceable, which
* is exactly what the combo loop concludes at request time.
*/
export function quotaInactiveReason(
config: OcxConfig,
// A combo votes over its own targets, which carry a model; the single-provider case has none,
// and an absent model keeps gating on every window exactly as before.
targets: readonly { provider: string; model?: string }[],
now = Date.now(),
): QuotaInactiveReason | undefined {
const usable = targets.filter(target => {
if (!Object.hasOwn(config.providers, target.provider)) return false;
const provider = config.providers[target.provider];
return !!provider && provider.disabled !== true;
});
if (usable.length === 0) return undefined;
for (const target of usable) {
const provider = config.providers[target.provider]!;
const quota = getCachedProviderRoutingQuota(target.provider, provider, now);
if (!quota || !cachedProviderQuotaIsExhausted(quota, now, target.model)) return undefined;
}
return "no_credit";
}
function smoothWeightedIndex(
targets: NormalizedComboTarget[],
state: SelectionState,
eligible: (target: NormalizedComboTarget) => boolean,
): number {
let best = -1;
let bestScore = Number.NEGATIVE_INFINITY;
let total = 0;
for (let i = 0; i < targets.length; i++) {
const target = targets[i]!;
if (!eligible(target)) continue;
const key = targetKey(target);
const score = (state.currentWeights.get(key) ?? 0) + target.weight;
state.currentWeights.set(key, score);
total += target.weight;
if (score > bestScore) {
best = i;
bestScore = score;
}
}
if (best >= 0) {
const key = targetKey(targets[best]!);
state.currentWeights.set(key, (state.currentWeights.get(key) ?? 0) - total);
}
return best;
}
/**
* Select the eligible target whose earliest known quota reset is nearest.
*
* Only reads the last successfully cached provider-quota snapshot; it never
* triggers an upstream quota probe. When no target has fresh reset data,
* every remaining value is Infinity and configured order becomes the
* fallback. Targets with elapsed or stale reset timestamps are treated as
* unknown (Infinity).
*/
function resetWindowIndex(
config: OcxConfig,
targets: NormalizedComboTarget[],
eligible: (target: NormalizedComboTarget) => boolean,
now = Date.now(),
): number {
let selected = -1;
let smallestRemaining = Number.POSITIVE_INFINITY;
for (let index = 0; index < targets.length; index++) {
const target = targets[index]!;
if (!eligible(target)) continue;
const remaining = quotaResetRemainingMs(
getCachedProviderRoutingQuota(target.provider, config.providers[target.provider], now),
now,
// Rank by the windows that gate this target: a model-scoped window of another family
// says nothing about when this model regains capacity.
window => customWindowAppliesToModel(window, target.model),
);
// Strict comparison deliberately retains configured order for ties,
// including the no-snapshot fallback where every value is Infinity.
if (selected < 0 || remaining < smallestRemaining) {
selected = index;
smallestRemaining = remaining;
}
}
return selected;
}
/** Reserve one request-scoped recovery pass; preview and in-flight picks never call this. */
function reserveAllCooledTargets(
config: OcxConfig, comboId: string, now: number,
eligible?: (target: NormalizedComboTarget) => boolean,
): Map<string, number> | undefined {
const combo = getCombo(config, comboId);
if (!combo?.cooldownReprobe?.enabled) return;
const state = selectionState.get(comboId);
if (state?.lastCooldownReprobeAt !== undefined
&& now - state.lastCooldownReprobeAt < combo.cooldownReprobe.intervalMs) return;
const reprobeEligible = (target: NormalizedComboTarget): boolean => {
if (!targetProviderIsUsable(config, target, now)) return false;
try { return eligible?.(target) ?? true; } catch { return false; }
};
// Ordinary selection skips cooled targets without evaluating their request predicate.
// Preserve that behavior when a sibling is already available.
if (combo.targets.some(target => !isComboTargetInCooldown(comboId, target, now)
&& reprobeEligible(target))) return;
const usable = combo.targets.filter(reprobeEligible);
if (usable.length === 0) return;
// Each reserved key is bound to the cooldown entry observed here. A replacement entry —
// a provider-scoped failure's, or one this request's own failed hop wrote — is newer
// evidence and cancels that target's bypass instead of being probed through.
const reserved = new Map<string, number>();
for (const target of usable) {
const revision = comboTargetCooldownRevision(comboId, target, now);
if (revision === undefined) return;
reserved.set(targetKey(target), revision);
}
// No await between reservation and reset: concurrent requests cannot reserve another pass.
const next = state ?? { successes: 0, currentWeights: new Map<string, number>(), successfulUses: new Map<string, number>() };
next.lastCooldownReprobeAt = now;
selectionState.set(comboId, next);
console.warn(`[combo] ${comboLogLabel(config, comboId)}: reserving all-cooled targets for one request; minimum reprobe interval ${combo.cooldownReprobe.intervalMs}ms`);
return reserved;
}
export function pickComboTarget(
config: OcxConfig,
comboId: string,
options: {
exclude?: Iterable<string>;
eligible?: (target: NormalizedComboTarget) => boolean;
/** Disable recovery for picks without request context or picks already in flight. */
allowCooldownReprobe?: boolean;
/** Request-scoped cooldown bypasses, each bound to the write it was reserved against. */
cooldownReprobeTargets?: Iterable<readonly [string, number]>;
now?: number;
/** Inspect a round-robin choice without mutating its sticky/weight state. */
preview?: boolean;
} = {},
): ComboPick | null {
const writerGeneration = captureConfigGeneration();
const combo = getCombo(config, comboId);
if (!combo) throw new UnknownComboError(comboId);
const excluded = new Set(options.exclude ?? []);
const now = options.now ?? Date.now();
const cooldownReprobeTargets = new Map(options.cooldownReprobeTargets);
if (options.allowCooldownReprobe !== false && !options.preview && excluded.size === 0) {
for (const [key, revision] of reserveAllCooledTargets(config, comboId, now, options.eligible) ?? []) {
cooldownReprobeTargets.set(key, revision);
}
}
const reprobeBypass = (target: NormalizedComboTarget): boolean => {
const revision = cooldownReprobeTargets.get(targetKey(target));
return revision !== undefined && revision === comboTargetCooldownRevision(comboId, target, now);
};
const eligible = (target: NormalizedComboTarget): boolean =>
targetProviderIsUsable(config, target, now)
&& (reprobeBypass(target) || !isComboTargetInCooldown(comboId, target, now))
&& !excluded.has(targetKey(target))
&& (options.eligible?.(target) ?? true);
let targetIndex = -1;
if (combo.strategy === "round-robin") {
let state = selectionState.get(comboId);
if (!state) {
state = { successes: 0, currentWeights: new Map(), successfulUses: new Map() };
if (!options.preview) selectionState.set(comboId, state);
} else if (options.preview) {
state = {
...state,
currentWeights: new Map(state.currentWeights),
successfulUses: new Map(state.successfulUses),
};
}
if (state.activeKey) {
targetIndex = combo.targets.findIndex(target => targetKey(target) === state.activeKey && eligible(target));
if (targetIndex < 0) {
delete state.activeKey;
state.successes = 0;
}
}
if (targetIndex < 0) {
targetIndex = smoothWeightedIndex(combo.targets, state, eligible);
if (targetIndex >= 0) {
state.activeKey = targetKey(combo.targets[targetIndex]!);
state.successes = 0;
}
}
} else if (combo.strategy === "random") {
// Weighted random selection happens independently for every request.
const eligibleTargets = combo.targets
.map((target, index) => ({ target, index }))
.filter(({ target }) => eligible(target));
if (eligibleTargets.length > 0) {
const totalWeight = eligibleTargets.reduce((sum, entry) => sum + entry.target.weight, 0);
let random = Math.random() * totalWeight;
for (const entry of eligibleTargets) {
random -= entry.target.weight;
if (random <= 0) {
targetIndex = entry.index;
break;
}
}
if (targetIndex < 0) targetIndex = eligibleTargets[eligibleTargets.length - 1]!.index;
}
} else if (combo.strategy === "least-used") {
let state = selectionState.get(comboId);
if (!state) {
state = { successes: 0, currentWeights: new Map(), successfulUses: new Map() };
selectionState.set(comboId, state);
}
let fewestUses = Number.POSITIVE_INFINITY;
for (let index = 0; index < combo.targets.length; index++) {
const target = combo.targets[index]!;
if (!eligible(target)) continue;
const uses = state.successfulUses.get(targetKey(target)) ?? 0;
if (targetIndex < 0 || uses < fewestUses) {
targetIndex = index;
fewestUses = uses;
}
}
} else if (combo.strategy === "reset-window") {
targetIndex = resetWindowIndex(config, combo.targets, eligible, now);
} else {
targetIndex = combo.targets.findIndex(eligible);
}
if (targetIndex < 0) return null;
const target = combo.targets[targetIndex]!;
return {
comboId,
target,
targetIndex,
attempted: [...excluded, targetKey(target)],
writerGeneration,
...(cooldownReprobeTargets.size > 0 ? { cooldownReprobeTargets: Object.fromEntries(cooldownReprobeTargets) } : {}),
};
}
export function noteComboSuccess(
comboId: string,
combo: NormalizedComboConfig,
target: NormalizedComboTarget,
writerGeneration = captureConfigGeneration(),
): void {
const key = targetKey(target);
if (!mayCommitComboState(comboId, key, writerGeneration)) return;
if (combo.strategy === "least-used") {
let state = selectionState.get(comboId);
if (!state) {
state = { successes: 0, currentWeights: new Map(), successfulUses: new Map() };
selectionState.set(comboId, state);
}
state.successfulUses.set(key, (state.successfulUses.get(key) ?? 0) + 1);
return;
}
if (combo.strategy !== "round-robin") return;
const state = selectionState.get(comboId);
if (!state || state.activeKey !== key) return;
state.successes += 1;
if (state.successes >= combo.stickyLimit) {
delete state.activeKey;
state.successes = 0;
}
}
/**
* A request that proved a reserved target healthy lifts exactly the cooldown entry it
* probed; a replacement entry written while the request was in flight stays. Targets
* the pass reserved but never dispatched keep their cooldowns.
*/
export function settleComboReprobeSuccess(pick: ComboPick): void {
const revision = pick.cooldownReprobeTargets?.[targetKey(pick.target)];
if (revision === undefined) return;
clearComboTargetCooldownIfUnchanged(pick.comboId, pick.target, revision);
}
export function noteComboFailure(
comboId: string,
target: OcxComboTarget,
writerGeneration = captureConfigGeneration(),
): void {
if (!mayCommitComboState(comboId, targetKey(target), writerGeneration)) return;
const state = selectionState.get(comboId);
if (state?.activeKey === targetKey(target)) {
delete state.activeKey;
state.successes = 0;
}
}
export function advanceComboAfterFailure(
config: OcxConfig,
pick: ComboPick,
options: {
retryAfter?: string | null;
resetAt?: unknown | unknown[];
now?: number;
cooldownMs?: number;
eligible?: (target: NormalizedComboTarget) => boolean;
cooldownScope?: ComboFailureCooldownScope;
/** A request-local deadline can advance the ladder without penalizing its target. */
recordFailure?: boolean;
onCooldownRecorded?: (target: Pick<OcxComboTarget, "provider" | "model">) => void;
status?: number;
code?: string | null;
message?: string;
/** Account-qualified provider label of the account this failure came from, when a pool resolved one. */
failedAccount?: string;
} = {},
): ComboPick | null {
if (options.recordFailure !== false) noteComboFailure(pick.comboId, pick.target, pick.writerGeneration);
const combo = getCombo(config, pick.comboId);
// A 429 from a pooled provider names the ACCOUNT that is spent, not the target. The pool
// records that account's own cooldown and rotates past it, so cooling the target here blacks
// out the accounts that still answer: on 2026-09-28 one Anthropic account replied
// `Retry-After: 318747` (88.5h) and anthropic/claude-opus-5-5 left selection for four hours
// while four of five accounts still returned 200.
//
// Only a pool-resolved account qualifies, and an unidentified account still cools the target --
// including the pool's OWN "all accounts are cooled" 429, which refuses locally before it picks
// one. That is what keeps this from trading a blackhole for a hammer: when nothing else is
// holding the target back, the target cooldown still does.
// A 402 credit refusal is the provider-scoped twin: the caller names an account only after the
// pool has cooled it and still holds an eligible sibling.
const accountScoped = options.failedAccount !== undefined
&& (options.status === 429 ? options.cooldownScope === "target" : options.status === 402);
// "none" records no cooldown at all: the failure described the request, not the target, so
// the target must stay immediately selectable for the next (differently shaped) request.
if (options.recordFailure !== false && options.cooldownScope !== "none" && !accountScoped) {
const cooldownTargets = options.cooldownScope === "provider" && combo
? combo.targets.filter(target => target.provider === pick.target.provider)
: [pick.target];
for (const target of cooldownTargets) {
const recorded = coolComboTarget(pick.comboId, target, {
...options,
cooldownMs: options.cooldownMs ?? combo?.cooldownMs,
writerGeneration: pick.writerGeneration,
});
if (recorded) options.onCooldownRecorded?.(target);
}
}
// #5691: under `cooldownWaitPolicy: "before-last-resort"` this final synchronous pick
// must not dispatch an emergency target while a normal one is merely cooling. Returning
// null hands the decision to `pickComboTargetWithWait`, which waits a normal target out
// inside the combo's wait budget or dispatches the last resort when none is reachable.
// Without the policy the eligible set is unchanged and the pick is exactly as before.
const defersLastResort = combo?.cooldownWaitPolicy === "before-last-resort"
&& combo.targets.some(target => !target.lastResort);
return pickComboTarget(config, pick.comboId, {
exclude: pick.attempted,
preview: options.recordFailure === false,
now: options.now,
cooldownReprobeTargets: pick.cooldownReprobeTargets
? Object.entries(pick.cooldownReprobeTargets)
: undefined,
eligible: target => (options.eligible?.(target) ?? true)
&& (!defersLastResort || !target.lastResort),
});
}
export async function pickComboTargetWithWait(
config: OcxConfig,
comboId: string,
options: {
exclude?: Iterable<string>;
eligible?: (target: NormalizedComboTarget) => boolean;
waitForCooldownMs: number;
abortSignal?: AbortSignal;
/** Disable recovery when retrying a target after its normal cooldown. */
allowCooldownReprobe?: boolean;
/** Request-scoped cooldown bypasses carried through the wait, each bound to its write. */
cooldownReprobeTargets?: Iterable<readonly [string, number]>;
now?: number;
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
},
): Promise<ComboPick | null> {
const now = options.now ?? Date.now();
const excluded = new Set(options.exclude ?? []);
const customEligible = options.eligible;
const cooldownReprobeTargets = new Map(options.cooldownReprobeTargets);
if (options.allowCooldownReprobe !== false && excluded.size === 0 && !options.abortSignal?.aborted) {
for (const [key, revision] of reserveAllCooledTargets(config, comboId, now, customEligible) ?? []) {
cooldownReprobeTargets.set(key, revision);
}
}
// Milliseconds already slept inside this call. `waitForCooldownMs` is documented as a cap
// per *selection attempt*, so a deferral wait and the ordinary wait below must share it —
// otherwise a 3s deferral followed by a 9s ordinary wait spends 12s against a 10s budget.
let spentWaitMs = 0;
// #5691: `cooldownWaitPolicy: "before-last-resort"` makes one extra attempt
// over the normal targets alone, so a *brief* cooldown on a preferred target
// waits rather than dispatching a target the operator marked emergency-only.
//
// It only ever defers. Every exit below falls through to the unchanged
// selection, which still sees the last-resort target — a policy that could
// withhold it when no normal target is reachable would turn a fallback into
// an outage, which is worse than the premature routing it prevents.
const policyCombo = getCombo(config, comboId);
const defersLastResort = policyCombo?.cooldownWaitPolicy === "before-last-resort"
&& policyCombo.targets.some(target => !target.lastResort);
if (defersLastResort && !options.abortSignal?.aborted) {
const normalOnly = (target: NormalizedComboTarget): boolean =>
!target.lastResort && (customEligible?.(target) ?? true);
const normalPick = pickComboTarget(config, comboId, {
allowCooldownReprobe: false,
cooldownReprobeTargets,
exclude: excluded,
eligible: normalOnly,
now,
});
if (normalPick) return normalPick;
const waitable = policyCombo.targets.filter(target =>
!target.lastResort
&& targetProviderIsUsable(config, target, now)
&& !excluded.has(targetKey(target))
&& isComboTargetInCooldown(comboId, target, now)
&& (customEligible?.(target) ?? true),
);
const soonest = earliestComboCooldown(comboId, waitable, now);
const normalDelay = soonest === undefined ? undefined : soonest.expiry - now;
if (normalDelay !== undefined && normalDelay <= options.waitForCooldownMs) {
console.warn(
`[combo] ${comboLogLabel(config, comboId)}: deferring last resort, waiting ${normalDelay}ms for ${comboTargetLogLabel(config, comboId, soonest!.target)}`,
);
try {
await (options.sleep ?? sleepWithAbort)(normalDelay, options.abortSignal);
} catch (error) {
if (options.abortSignal?.aborted) return null;
throw error;
}
spentWaitMs += normalDelay;
if (options.abortSignal?.aborted) return null;
// The combo can be deleted or renamed while this request sleeps.
if (!getCombo(config, comboId)) return null;
const waited = pickComboTarget(config, comboId, {
allowCooldownReprobe: false,
cooldownReprobeTargets,
exclude: excluded,
now: now + normalDelay,
eligible: normalOnly,
});
if (waited) return waited;
}
// No normal target is reachable. Fall through; the last resort is eligible.
}
// Both advance when the deferral above slept; they are identical to `now` and
// `options.waitForCooldownMs` when it did not, so the non-policy path is unchanged.
const clock = now + spentWaitMs;
const remainingWaitMs = options.waitForCooldownMs - spentWaitMs;
const pick = pickComboTarget(config, comboId, {
allowCooldownReprobe: false,
cooldownReprobeTargets,
exclude: excluded,
eligible: customEligible,
now: clock,
});
if (pick || remainingWaitMs <= 0 || options.abortSignal?.aborted) return pick;
const combo = getCombo(config, comboId);
if (!combo) throw new UnknownComboError(comboId);
const waitingTargets = combo.targets.filter(target =>
targetProviderIsUsable(config, target, clock)
&& !excluded.has(targetKey(target))
&& isComboTargetInCooldown(comboId, target, clock)
&& (customEligible?.(target) ?? true),
);
const earliest = earliestComboCooldown(comboId, waitingTargets, clock);
if (earliest === undefined) return null;
const delay = earliest.expiry - clock;
if (delay > remainingWaitMs) return null;
// The expiry computation above is the single source of truth for the wait budget.
// Its target preserves configured order for ties.
const target = earliest.target;
console.warn(
`[combo] ${comboLogLabel(config, comboId)}: all targets cooling, waiting ${delay}ms for ${comboTargetLogLabel(config, comboId, target)}`,
);
try {
await (options.sleep ?? sleepWithAbort)(delay, options.abortSignal);
} catch (error) {
if (options.abortSignal?.aborted) return null;
throw error;
}
if (options.abortSignal?.aborted) return null;
// Management updates can delete or rename the combo while this request sleeps.
if (!getCombo(config, comboId)) return null;
return pickComboTarget(config, comboId, {
exclude: excluded,
allowCooldownReprobe: false,
cooldownReprobeTargets,
now: clock + delay,
eligible: customEligible,
});
}
export function reconcileComboRotationState(context: GenerationContext): number {
if (context.generation <= lastReconciledGeneration) return 0;
let removed = 0;
for (const [comboId, state] of selectionState) {
if (!context.comboIds.has(comboId)) {
selectionState.delete(comboId);
removed += 1;
continue;
}
if (state.activeKey && !context.comboTargets.has(comboTargetOwnerKey(comboId, state.activeKey))) {
delete state.activeKey;
state.successes = 0;
removed += 1;
}
for (const key of state.currentWeights.keys()) {
if (context.comboTargets.has(comboTargetOwnerKey(comboId, key))) continue;
state.currentWeights.delete(key);
removed += 1;
}
for (const key of state.successfulUses.keys()) {
if (context.comboTargets.has(comboTargetOwnerKey(comboId, key))) continue;
state.successfulUses.delete(key);
removed += 1;
}
}
liveComboTargets = new Set(context.comboTargets);
lastReconciledGeneration = context.generation;
return removed;
}
export function clearComboSelectionState(comboId?: string): void {
if (comboId === undefined) {
selectionState.clear();
liveComboTargets.clear();
lastReconciledGeneration = 0;
return;
}
selectionState.delete(comboId);
}
export function tryPickComboModel(config: OcxConfig, modelId: string, preview = false): ComboPick | null {
const comboId = resolveComboId(config, modelId);
if (!comboId) return null;
if (!getCombo(config, comboId)) throw new UnknownComboError(comboId);
const picked = pickComboTarget(config, comboId, { preview, allowCooldownReprobe: false });
if (!picked) throw new NoAvailableComboTargetsError(comboId, comboExclusion(config, comboId));
return picked;
}