@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
606 lines (574 loc) • 24.9 kB
text/typescript
import type { OcxComboTarget, OcxConfig } from "../types";
import { getCachedProviderRoutingQuota } from "../providers/quota-routing-cache";
import type { ProviderQuota, ProviderQuotaWindow } from "../providers/quota-types";
import { sleepWithAbort } from "../lib/upstream-retry";
import {
coolComboTarget,
earliestComboCooldown,
isComboTargetInCooldown,
type ComboFailureCooldownScope,
} from "./failover";
import { quotaResetRemainingMs } from "./reset-window";
import { getCombo, resolveComboId, targetKey } from "./types";
import type { NormalizedComboConfig, NormalizedComboTarget } from "./types";
import {
captureConfigGeneration,
type GenerationContext,
} from "../lib/state-store-sweeper";
export interface ComboPick {
comboId: string;
target: NormalizedComboTarget;
targetIndex: number;
attempted: string[];
writerGeneration: number;
}
interface SelectionState {
activeKey?: string;
successes: number;
currentWeights: Map<string, number>;
successfulUses: Map<string, number>;
}
const selectionState = new Map<string, SelectionState>();
let lastReconciledGeneration = 0;
let liveComboTargets = new Set<string>();
function comboTargetOwnerKey(comboId: string, key: string): string {
return `${comboId}::${key}`;
}
function mayCommitComboState(comboId: string, key: string, writerGeneration: number): boolean {
return writerGeneration >= lastReconciledGeneration
|| liveComboTargets.has(comboTargetOwnerKey(comboId, key));
}
export class UnknownComboError extends Error {
constructor(readonly comboId: string) {
super(`Unknown combo: ${comboId}`);
this.name = "UnknownComboError";
}
}
export class NoAvailableComboTargetsError extends Error {
readonly code = "combo_unavailable";
constructor(readonly comboId: string) {
super(`No available targets for combo: ${comboId}`);
this.name = "NoAvailableComboTargetsError";
}
}
function targetProviderIsUsable(config: OcxConfig, target: OcxComboTarget, now: number): boolean {
if (!Object.hasOwn(config.providers, target.provider)) return false;
const provider = config.providers[target.provider];
if (!provider || provider.disabled === true) return false;
return !cachedProviderQuotaIsExhausted(getCachedProviderRoutingQuota(target.provider, provider, now), now, target.model);
}
function quotaWindowExhausted(percent: number | undefined, resetAt: number | undefined, now: number): boolean {
if (typeof percent !== "number" || !Number.isFinite(percent) || percent < 100) return false;
return typeof resetAt !== "number" || !Number.isFinite(resetAt) || resetAt > now;
}
// `customWindows` is a generic carrier. Most labels are provider-wide counters ("Prepaid
// credits", "Free trial", "burst", "Spark") and must gate every model. Anthropic also rides it
// for per-model family counters, where a spent Opus week says nothing about a Sonnet request.
// `routing/quota.ts` already scopes those per model when it ranks accounts
// (`anthropicFamilyWindow`); this is the same fact applied to combo target selection.
//
// The scoping keys on `window.scope`, which the PRODUCER sets only where it proved model scope
// structurally, NEVER on the label text: `quota/antigravity.ts` forwards an upstream
// `group.displayName` unchanged, so a provider-wide group named "Opus" read as per-model would
// leave a spent window unenforced and send a doomed request. Every unproven direction keeps
// gating everything, which is the behaviour before this change: no scope on the window, no
// model on the request, or a family this gateway cannot match against a model id. Failing
// closed there costs at most a diverted target.
const MODEL_FAMILY_WINDOW_LABELS = new Set(["fable", "opus", "sonnet"]);
function customWindowAppliesToModel(window: ProviderQuotaWindow, model: string | undefined): boolean {
if (window.scope !== "model" || model === undefined) return true;
const family = window.label.trim().toLowerCase();
if (!MODEL_FAMILY_WINDOW_LABELS.has(family)) return true;
return model.toLowerCase().includes(family);
}
export function cachedProviderQuotaIsExhausted(
quota: ProviderQuota | null,
now = Date.now(),
model?: string,
): boolean {
if (!quota) return false;
if (quotaWindowExhausted(quota.fiveHourPercent, quota.fiveHourResetAt, now)) return true;
if (quotaWindowExhausted(quota.weeklyPercent, quota.weeklyResetAt, now)) return true;
if (quotaWindowExhausted(quota.monthlyPercent, quota.monthlyResetAt, now)) return true;
if (quota.customWindows?.some(window => customWindowAppliesToModel(window, model)
&& quotaWindowExhausted(window.percent, window.resetAt, now))) return true;
if (quota.creditsUsd?.unlimited !== true
&& typeof quota.creditsUsd?.percent === "number"
&& Number.isFinite(quota.creditsUsd.percent)
&& quota.creditsUsd.percent >= 100
&& quota.creditsUsd.remaining <= 0) return true;
return false;
}
/**
* Why a catalog row is offered but cannot currently serve a request (#1711).
*
* Only one reason exists today. It is a string rather than a boolean so a later cause — a
* cooldown, a revoked key — can be told apart by a consumer that already reads the field.
*/
export type QuotaInactiveReason = "no_credit";
/**
* `"no_credit"` when every USABLE target of a catalog row has positive exhaustion evidence
* (#1711), otherwise undefined.
*
* This reuses the credential-scoped evidence and exhaustion rules used by runtime selection.
* Display-only account, model-group and service windows cannot mark a whole provider inactive.
*
* Three rules carry the correctness, all inherited rather than restated:
*
* - A target the operator has removed or disabled is not usable and is not evidence either way;
* it drops out before the vote. If nothing is left, the row is unavailable for an operator
* reason rather than a quota one, so this returns undefined.
* - The canonical ChatGPT forward provider is exempt. Native account selection owns model-scoped
* quota, and a provider-level summary cannot veto it.
* - A stale cache is NOT exhaustion. `getCachedProviderRoutingQuota` returns null past its 30-minute
* window, and a null reading ends the vote rather than counting as evidence, so an unprobed
* provider is never marked inactive.
*
* "Every" is the bar on purpose: one target that can still serve makes the row serviceable, which
* is exactly what the combo loop concludes at request time.
*/
export function quotaInactiveReason(
config: OcxConfig,
// A combo votes over its own targets, which carry a model; the single-provider case has none,
// and an absent model keeps gating on every window exactly as before.
targets: readonly { provider: string; model?: string }[],
now = Date.now(),
): QuotaInactiveReason | undefined {
const usable = targets.filter(target => {
if (!Object.hasOwn(config.providers, target.provider)) return false;
const provider = config.providers[target.provider];
return !!provider && provider.disabled !== true;
});
if (usable.length === 0) return undefined;
for (const target of usable) {
const provider = config.providers[target.provider]!;
const quota = getCachedProviderRoutingQuota(target.provider, provider, now);
if (!quota || !cachedProviderQuotaIsExhausted(quota, now, target.model)) return undefined;
}
return "no_credit";
}
function smoothWeightedIndex(
targets: NormalizedComboTarget[],
state: SelectionState,
eligible: (target: NormalizedComboTarget) => boolean,
): number {
let best = -1;
let bestScore = Number.NEGATIVE_INFINITY;
let total = 0;
for (let i = 0; i < targets.length; i++) {
const target = targets[i]!;
if (!eligible(target)) continue;
const key = targetKey(target);
const score = (state.currentWeights.get(key) ?? 0) + target.weight;
state.currentWeights.set(key, score);
total += target.weight;
if (score > bestScore) {
best = i;
bestScore = score;
}
}
if (best >= 0) {
const key = targetKey(targets[best]!);
state.currentWeights.set(key, (state.currentWeights.get(key) ?? 0) - total);
}
return best;
}
/**
* Select the eligible target whose earliest known quota reset is nearest.
*
* Only reads the last successfully cached provider-quota snapshot; it never
* triggers an upstream quota probe. When no target has fresh reset data,
* every remaining value is Infinity and configured order becomes the
* fallback. Targets with elapsed or stale reset timestamps are treated as
* unknown (Infinity).
*/
function resetWindowIndex(
config: OcxConfig,
targets: NormalizedComboTarget[],
eligible: (target: NormalizedComboTarget) => boolean,
now = Date.now(),
): number {
let selected = -1;
let smallestRemaining = Number.POSITIVE_INFINITY;
for (let index = 0; index < targets.length; index++) {
const target = targets[index]!;
if (!eligible(target)) continue;
const remaining = quotaResetRemainingMs(
getCachedProviderRoutingQuota(target.provider, config.providers[target.provider], now),
now,
// Rank by the windows that gate this target: a model-scoped window of another family
// says nothing about when this model regains capacity.
window => customWindowAppliesToModel(window, target.model),
);
// Strict comparison deliberately retains configured order for ties,
// including the no-snapshot fallback where every value is Infinity.
if (selected < 0 || remaining < smallestRemaining) {
selected = index;
smallestRemaining = remaining;
}
}
return selected;
}
export function pickComboTarget(
config: OcxConfig,
comboId: string,
options: {
exclude?: Iterable<string>;
eligible?: (target: NormalizedComboTarget) => boolean;
now?: number;
/** Inspect a round-robin choice without mutating its sticky/weight state. */
preview?: boolean;
} = {},
): ComboPick | null {
const writerGeneration = captureConfigGeneration();
const combo = getCombo(config, comboId);
if (!combo) throw new UnknownComboError(comboId);
const excluded = new Set(options.exclude ?? []);
const now = options.now ?? Date.now();
const eligible = (target: NormalizedComboTarget): boolean =>
targetProviderIsUsable(config, target, now)
&& !isComboTargetInCooldown(comboId, target, now)
&& !excluded.has(targetKey(target))
&& (options.eligible?.(target) ?? true);
let targetIndex = -1;
if (combo.strategy === "round-robin") {
let state = selectionState.get(comboId);
if (!state) {
state = { successes: 0, currentWeights: new Map(), successfulUses: new Map() };
if (!options.preview) selectionState.set(comboId, state);
} else if (options.preview) {
state = {
...state,
currentWeights: new Map(state.currentWeights),
successfulUses: new Map(state.successfulUses),
};
}
if (state.activeKey) {
targetIndex = combo.targets.findIndex(target => targetKey(target) === state.activeKey && eligible(target));
if (targetIndex < 0) {
delete state.activeKey;
state.successes = 0;
}
}
if (targetIndex < 0) {
targetIndex = smoothWeightedIndex(combo.targets, state, eligible);
if (targetIndex >= 0) {
state.activeKey = targetKey(combo.targets[targetIndex]!);
state.successes = 0;
}
}
} else if (combo.strategy === "random") {
// Weighted random selection happens independently for every request.
const eligibleTargets = combo.targets
.map((target, index) => ({ target, index }))
.filter(({ target }) => eligible(target));
if (eligibleTargets.length > 0) {
const totalWeight = eligibleTargets.reduce((sum, entry) => sum + entry.target.weight, 0);
let random = Math.random() * totalWeight;
for (const entry of eligibleTargets) {
random -= entry.target.weight;
if (random <= 0) {
targetIndex = entry.index;
break;
}
}
if (targetIndex < 0) targetIndex = eligibleTargets[eligibleTargets.length - 1]!.index;
}
} else if (combo.strategy === "least-used") {
let state = selectionState.get(comboId);
if (!state) {
state = { successes: 0, currentWeights: new Map(), successfulUses: new Map() };
selectionState.set(comboId, state);
}
let fewestUses = Number.POSITIVE_INFINITY;
for (let index = 0; index < combo.targets.length; index++) {
const target = combo.targets[index]!;
if (!eligible(target)) continue;
const uses = state.successfulUses.get(targetKey(target)) ?? 0;
if (targetIndex < 0 || uses < fewestUses) {
targetIndex = index;
fewestUses = uses;
}
}
} else if (combo.strategy === "reset-window") {
targetIndex = resetWindowIndex(config, combo.targets, eligible, now);
} else {
targetIndex = combo.targets.findIndex(eligible);
}
if (targetIndex < 0) return null;
const target = combo.targets[targetIndex]!;
return {
comboId,
target,
targetIndex,
attempted: [...excluded, targetKey(target)],
writerGeneration,
};
}
export function noteComboSuccess(
comboId: string,
combo: NormalizedComboConfig,
target: NormalizedComboTarget,
writerGeneration = captureConfigGeneration(),
): void {
const key = targetKey(target);
if (!mayCommitComboState(comboId, key, writerGeneration)) return;
if (combo.strategy === "least-used") {
let state = selectionState.get(comboId);
if (!state) {
state = { successes: 0, currentWeights: new Map(), successfulUses: new Map() };
selectionState.set(comboId, state);
}
state.successfulUses.set(key, (state.successfulUses.get(key) ?? 0) + 1);
return;
}
if (combo.strategy !== "round-robin") return;
const state = selectionState.get(comboId);
if (!state || state.activeKey !== key) return;
state.successes += 1;
if (state.successes >= combo.stickyLimit) {
delete state.activeKey;
state.successes = 0;
}
}
export function noteComboFailure(
comboId: string,
target: OcxComboTarget,
writerGeneration = captureConfigGeneration(),
): void {
if (!mayCommitComboState(comboId, targetKey(target), writerGeneration)) return;
const state = selectionState.get(comboId);
if (state?.activeKey === targetKey(target)) {
delete state.activeKey;
state.successes = 0;
}
}
export function advanceComboAfterFailure(
config: OcxConfig,
pick: ComboPick,
options: {
retryAfter?: string | null;
resetAt?: unknown | unknown[];
now?: number;
cooldownMs?: number;
eligible?: (target: NormalizedComboTarget) => boolean;
cooldownScope?: ComboFailureCooldownScope;
onCooldownRecorded?: (target: Pick<OcxComboTarget, "provider" | "model">) => void;
status?: number;
code?: string | null;
message?: string;
/** Account-qualified provider label of the account this failure came from, when a pool resolved one. */
failedAccount?: string;
} = {},
): ComboPick | null {
noteComboFailure(pick.comboId, pick.target, pick.writerGeneration);
const combo = getCombo(config, pick.comboId);
// A 429 from a pooled provider names the ACCOUNT that is spent, not the target. The pool
// records that account's own cooldown and rotates past it, so cooling the target here blacks
// out the accounts that still answer: on 2026-09-28 one Anthropic account replied
// `Retry-After: 318747` (88.5h) and anthropic/claude-opus-5-5 left selection for four hours
// while four of five accounts still returned 200.
//
// Only a pool-resolved account qualifies, and an unidentified account still cools the target --
// including the pool's OWN "all accounts are cooled" 429, which refuses locally before it picks
// one. That is what keeps this from trading a blackhole for a hammer: when nothing else is
// holding the target back, the target cooldown still does.
const accountScoped = options.cooldownScope === "target"
&& options.status === 429
&& options.failedAccount !== undefined;
// "none" records no cooldown at all: the failure described the request, not the target, so
// the target must stay immediately selectable for the next (differently shaped) request.
if (options.cooldownScope !== "none" && !accountScoped) {
const cooldownTargets = options.cooldownScope === "provider" && combo
? combo.targets.filter(target => target.provider === pick.target.provider)
: [pick.target];
for (const target of cooldownTargets) {
const recorded = coolComboTarget(pick.comboId, target, {
...options,
cooldownMs: options.cooldownMs ?? combo?.cooldownMs,
writerGeneration: pick.writerGeneration,
});
if (recorded) options.onCooldownRecorded?.(target);
}
}
// #5691: under `cooldownWaitPolicy: "before-last-resort"` this final synchronous pick
// must not dispatch an emergency target while a normal one is merely cooling. Returning
// null hands the decision to `pickComboTargetWithWait`, which waits a normal target out
// inside the combo's wait budget or dispatches the last resort when none is reachable.
// Without the policy the eligible set is unchanged and the pick is exactly as before.
const defersLastResort = combo?.cooldownWaitPolicy === "before-last-resort"
&& combo.targets.some(target => !target.lastResort);
return pickComboTarget(config, pick.comboId, {
exclude: pick.attempted,
now: options.now,
eligible: target => !isComboTargetInCooldown(pick.comboId, target, options.now)
&& (options.eligible?.(target) ?? true)
&& (!defersLastResort || !target.lastResort),
});
}
export async function pickComboTargetWithWait(
config: OcxConfig,
comboId: string,
options: {
exclude?: Iterable<string>;
eligible?: (target: NormalizedComboTarget) => boolean;
waitForCooldownMs: number;
abortSignal?: AbortSignal;
now?: number;
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
},
): Promise<ComboPick | null> {
const now = options.now ?? Date.now();
const excluded = new Set(options.exclude ?? []);
const customEligible = options.eligible;
const eligibleAt = (target: NormalizedComboTarget, at: number): boolean =>
!isComboTargetInCooldown(comboId, target, at)
&& (customEligible?.(target) ?? true);
const eligible = (target: NormalizedComboTarget): boolean => eligibleAt(target, now);
// Milliseconds already slept inside this call. `waitForCooldownMs` is documented as a cap
// per *selection attempt*, so a deferral wait and the ordinary wait below must share it —
// otherwise a 3s deferral followed by a 9s ordinary wait spends 12s against a 10s budget.
let spentWaitMs = 0;
// #5691: `cooldownWaitPolicy: "before-last-resort"` makes one extra attempt
// over the normal targets alone, so a *brief* cooldown on a preferred target
// waits rather than dispatching a target the operator marked emergency-only.
//
// It only ever defers. Every exit below falls through to the unchanged
// selection, which still sees the last-resort target — a policy that could
// withhold it when no normal target is reachable would turn a fallback into
// an outage, which is worse than the premature routing it prevents.
const policyCombo = getCombo(config, comboId);
const defersLastResort = policyCombo?.cooldownWaitPolicy === "before-last-resort"
&& policyCombo.targets.some(target => !target.lastResort);
if (defersLastResort && !options.abortSignal?.aborted) {
const normalOnly = (target: NormalizedComboTarget): boolean =>
!target.lastResort && eligible(target);
const normalPick = pickComboTarget(config, comboId, {
exclude: excluded,
eligible: normalOnly,
now,
});
if (normalPick) return normalPick;
const waitable = policyCombo.targets.filter(target =>
!target.lastResort
&& targetProviderIsUsable(config, target, now)
&& !excluded.has(targetKey(target))
&& isComboTargetInCooldown(comboId, target, now)
&& (customEligible?.(target) ?? true),
);
const soonest = earliestComboCooldown(comboId, waitable, now);
const normalDelay = soonest === undefined ? undefined : soonest.expiry - now;
if (normalDelay !== undefined && normalDelay <= options.waitForCooldownMs) {
console.warn(
`[combo] ${comboId}: deferring last resort, waiting ${normalDelay}ms for ${targetKey(soonest!.target)}`,
);
try {
await (options.sleep ?? sleepWithAbort)(normalDelay, options.abortSignal);
} catch (error) {
if (options.abortSignal?.aborted) return null;
throw error;
}
spentWaitMs += normalDelay;
if (options.abortSignal?.aborted) return null;
// The combo can be deleted or renamed while this request sleeps.
if (!getCombo(config, comboId)) return null;
const waited = pickComboTarget(config, comboId, {
exclude: excluded,
now: now + normalDelay,
eligible: target => !target.lastResort
&& !isComboTargetInCooldown(comboId, target, now + normalDelay)
&& (customEligible?.(target) ?? true),
});
if (waited) return waited;
}
// No normal target is reachable. Fall through; the last resort is eligible.
}
// Both advance when the deferral above slept; they are identical to `now` and
// `options.waitForCooldownMs` when it did not, so the non-policy path is unchanged.
const clock = now + spentWaitMs;
const remainingWaitMs = options.waitForCooldownMs - spentWaitMs;
const pick = pickComboTarget(config, comboId, {
exclude: excluded,
eligible: target => eligibleAt(target, clock),
now: clock,
});
if (pick || remainingWaitMs <= 0 || options.abortSignal?.aborted) return pick;
const combo = getCombo(config, comboId);
if (!combo) throw new UnknownComboError(comboId);
const waitingTargets = combo.targets.filter(target =>
targetProviderIsUsable(config, target, clock)
&& !excluded.has(targetKey(target))
&& isComboTargetInCooldown(comboId, target, clock)
&& (customEligible?.(target) ?? true),
);
const earliest = earliestComboCooldown(comboId, waitingTargets, clock);
if (earliest === undefined) return null;
const delay = earliest.expiry - clock;
if (delay > remainingWaitMs) return null;
// The expiry computation above is the single source of truth for the wait budget.
// Its target preserves configured order for ties.
const target = earliest.target;
console.warn(
`[combo] ${comboId}: all targets cooling, waiting ${delay}ms for ${targetKey(target)}`,
);
try {
await (options.sleep ?? sleepWithAbort)(delay, options.abortSignal);
} catch (error) {
if (options.abortSignal?.aborted) return null;
throw error;
}
if (options.abortSignal?.aborted) return null;
// Management updates can delete or rename the combo while this request sleeps.
if (!getCombo(config, comboId)) return null;
return pickComboTarget(config, comboId, {
exclude: excluded,
now: clock + delay,
eligible: targetCandidate => eligibleAt(targetCandidate, clock + delay),
});
}
export function reconcileComboRotationState(context: GenerationContext): number {
if (context.generation <= lastReconciledGeneration) return 0;
let removed = 0;
for (const [comboId, state] of selectionState) {
if (!context.comboIds.has(comboId)) {
selectionState.delete(comboId);
removed += 1;
continue;
}
if (state.activeKey && !context.comboTargets.has(comboTargetOwnerKey(comboId, state.activeKey))) {
delete state.activeKey;
state.successes = 0;
removed += 1;
}
for (const key of state.currentWeights.keys()) {
if (context.comboTargets.has(comboTargetOwnerKey(comboId, key))) continue;
state.currentWeights.delete(key);
removed += 1;
}
for (const key of state.successfulUses.keys()) {
if (context.comboTargets.has(comboTargetOwnerKey(comboId, key))) continue;
state.successfulUses.delete(key);
removed += 1;
}
}
liveComboTargets = new Set(context.comboTargets);
lastReconciledGeneration = context.generation;
return removed;
}
export function clearComboSelectionState(comboId?: string): void {
if (comboId === undefined) {
selectionState.clear();
liveComboTargets.clear();
lastReconciledGeneration = 0;
return;
}
selectionState.delete(comboId);
}
export function tryPickComboModel(config: OcxConfig, modelId: string, preview = false): ComboPick | null {
const comboId = resolveComboId(config, modelId);
if (!comboId) return null;
if (!getCombo(config, comboId)) throw new UnknownComboError(comboId);
const picked = pickComboTarget(config, comboId, { preview });
if (!picked) throw new NoAvailableComboTargetsError(comboId);
return picked;
}