@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
1,013 lines • 53.1 kB
text/typescript
import { clientFacingModelText, type ClientFacingModelLabel } from "../lib/error-destination";
import { isBuiltinAnthropicInstanceRow } from "../providers/anthropic-instance";
/**
* Generic OAuth multi-account 429 failover (#2568).
*
* The API-key twin (`providers/key-failover.ts`) rotates by default for any key provider with a
* 2+ pool, but it returns false for `authMode === "oauth"`, and the only OAuth rotator that
* exists is Anthropic's — behind its own opt-in. So xAI, Cursor, Kimi, GitHub Copilot,
* Antigravity and Nous have no recovery path on a 429 even with several accounts logged in.
*
* Conversation ownership lives in generic-session-affinity.ts; this module owns account
* eligibility, quota-ranked preference and bounded refusal recovery. Probe leases remain
* provider-specific.
*
* NOT a home for Codex (`codex/routing.ts` owns quota scopes and probe leases) or Anthropic
* (`oauth/anthropic-routing.ts` owns affinity and a fail-closed local-cli credential rule).
* Both are excluded by `isGenericFailoverProvider`.
*/
import { credentialGeneration, getAccountSet } from "./store";
import { oauthCredentialSiblingsOf, oauthCredentialSlot } from "./credential-siblings";
import { accountInFlight } from "./kiro-account-load";
import type { ProviderAccount } from "./types";
import { getValidAccessSnapshotForAccount, type OAuthAccessSnapshot } from "./index";
import {
accountHeadroomPercent,
exhaustedCooldownMs,
hasHeadroomEvidence,
isAccountQuotaExhausted,
rankAccountsByHeadroom,
rankAccountsByResetFirst,
classifyModelFamilyForQuota,
type QuotaModelFamily,
} from "./account-quota-rank";
import {
genericPoolKey,
normalizeAccountPoolStickyLimit,
notePoolRotationSuccess,
peekRoundRobinAccount,
pickRoundRobinAccount,
seedPoolRotationAccount,
} from "./pool-kernel";
import { parseRetryAfterMs } from "../combos/failover";
import type { KiroRefusalKind } from "../adapters/kiro-refusal";
import { kiroAccountEvidence } from "../providers/kiro-usage";
import { kiroEvidenceIdentity } from "../providers/kiro-account-state-disk";
import { kiroAccountSupportsModel } from "../providers/kiro-model-catalog";
import { antigravityAccountSupportsModel } from "../providers/antigravity-account-roster";
import { ACCOUNT_QUOTA_TTL_MS } from "../providers/quota-wire";
import { sweepExpiredOnWrite } from "../lib/state-store-sweeper";
import { subscribeOAuthAccountPauseChanges } from "../lib/account-selection-events";
import type { OcxConfig, OcxProviderConfig } from "../types";
/** Cap same-request rotations so a short Retry-After cannot spin. Mirrors the Anthropic bound. */
export const GENERIC_OAUTH_MAX_FAILOVERS_PER_REQUEST = 3;
/**
* Most accounts one request may fund and rotate through, whatever the roster size. Each funded
* account gets the normal transient ladder, so this fixes the default ingress ceiling at
* 6 × TRANSIENT_RETRY_MAX_ATTEMPTS physical sends: enrolling more accounts cannot turn one request
* into a pool-wide 429 storm.
*/
export const GENERIC_OAUTH_MAX_ACCOUNTS_PER_REQUEST = 6;
const DEFAULT_COOLDOWN_MS = 60_000;
/**
* How long a presence answer may be reused before the store is consulted again.
*
* `loadAuthStore` has no cache: every call chmods the config dir and the secret, reads the whole
* file, parses it and normalizes the store (store.ts:136-151). Since presence now decides
* activation, this predicate runs on paths that have not seen a 429 at all — the streaming and
* non-streaming runTurn entry points evaluate it once per request — so an uncached check would put
* a synchronous file read in front of every request for every OAuth provider.
*
* Two seconds is short enough that a login in another window is picked up before the operator can
* switch back and send a prompt, and long enough that a burst of requests shares one read. The
* cache holds a COUNT, never a credential.
*/
const PRESENCE_CACHE_TTL_MS = 2_000;
/**
* Providers whose rotation is owned elsewhere and must not be handled here.
*
* `openai` is the Codex pool: quota scopes, probe leases and affinity semantics that this
* module deliberately does not reimplement. `anthropic` has its own pool with a fail-closed
* rule about background local-cli credential slots.
*/
const EXCLUDED_PROVIDERS = new Set(["openai", "anthropic"]);
interface AccountHealth {
cooldownUntil: number;
cooldownSource: "retry-after" | "default" | "kiro-suspension" | "auth";
identity?: string;
}
interface PresenceEntry {
eligible: number;
readAt: number;
}
/** Process-local, like the Anthropic pool's: a restart is allowed to forget a cooldown. */
const health = new Map<string, AccountHealth>();
/**
* Login slot -> recent eligible-account count. TTL-bounded; never holds credential material.
* Keyed like health below, so a pause through one row of a shared login invalidates the other's.
*/
const presence = new Map<string, PresenceEntry>();
const presenceKey = (provider: string) => oauthCredentialSlot(provider);
subscribeOAuthAccountPauseChanges(provider => presence.delete(presenceKey(provider)));
// Keyed by the login slot: rows that share one login (Kimi and Kimi (Responses)) share each
// account's health, so a cooldown earned through one row is honored by the other.
const healthKey = (provider: string, accountId: string, family?: QuotaModelFamily) => {
const slot = oauthCredentialSlot(provider);
return family ? `${slot}\u0000${accountId}\u0000${family}` : `${slot}\u0000${accountId}`;
};
export function quarantineKiroSuspendedAccount(accountId: string, generation?: string, now = Date.now()): void {
const live = getAccountSet("kiro")?.accounts.find(row => row.id === accountId);
if (!live || (generation && credentialGeneration(live.credential) !== generation)) return;
health.set(healthKey("kiro", accountId), {
cooldownUntil: now + 24 * 60 * 60_000, cooldownSource: "kiro-suspension",
identity: kiroEvidenceIdentity(live),
});
sweepExpiredOnWrite(now);
}
function isCooled(provider: string, accountId: string, now: number, family?: QuotaModelFamily): boolean {
const entry = health.get(healthKey(provider, accountId, family));
if (!entry) return false;
if (provider === "google-antigravity" && entry.cooldownSource === "auth") {
const live = getAccountSet(provider)?.accounts.find(row => row.id === accountId);
if (!live || entry.identity !== credentialGeneration(live.credential)) {
health.delete(healthKey(provider, accountId, family));
return false;
}
}
if (provider === "kiro" && entry.cooldownSource === "kiro-suspension") {
const live = getAccountSet("kiro")?.accounts.find(row => row.id === accountId);
if (!live || entry.identity !== kiroEvidenceIdentity(live)) {
health.delete(healthKey(provider, accountId, family));
return false;
}
}
if (entry.cooldownUntil <= now) {
health.delete(healthKey(provider, accountId, family));
return false;
}
return true;
}
export type KiroSkipReason = "paused" | "needs_reauth" | "suspended" | "cooldown" | "quota_exhausted";
/** Eligibility for automatic alternatives; an active singleton can still send. */
export function kiroAutoSelection(
account: ProviderAccount, now = Date.now(),
): { autoSelectable: boolean; skipReason?: KiroSkipReason } {
if (account.paused === true) return { autoSelectable: false, skipReason: "paused" };
if (account.needsReauth === true) return { autoSelectable: false, skipReason: "needs_reauth" };
const cooled = isCooled("kiro", account.id, now);
if (cooled && health.get(healthKey("kiro", account.id))?.cooldownSource === "kiro-suspension")
return { autoSelectable: false, skipReason: "suspended" };
if (kiroAccountEvidence(account, now).exhausted === true)
return { autoSelectable: false, skipReason: "quota_exhausted" };
if (cooled) return { autoSelectable: false, skipReason: "cooldown" };
return { autoSelectable: true };
}
/** True when this provider participates in generic rotation at all. */
export function isGenericFailoverProvider(providerName: string, provider: OcxProviderConfig): boolean {
return provider.authMode === "oauth" && !EXCLUDED_PROVIDERS.has(providerName)
&& !isBuiltinAnthropicInstanceRow(providerName, provider);
}
/**
* Stored accounts that could serve traffic if asked, ignoring cooldowns.
*
* Cooldowns are excluded on purpose: they are transient and per-request, while this answers the
* durable question "did the operator log in more than one account". Treating a cooled account as
* absent would switch the feature off for the rest of the cooldown, which is exactly when it is
* needed.
*/
function eligibleAccountCount(providerName: string, now: number): number {
const cached = presence.get(presenceKey(providerName));
if (cached && now >= cached.readAt && now - cached.readAt < PRESENCE_CACHE_TTL_MS) return cached.eligible;
const set = getAccountSet(providerName);
// Kiro terminal refresh marks the just-refused account needsReauth before the alternate
// selector runs. Both stored logins still express consent to recover through the survivor.
const eligible = set
? set.accounts.filter(account => account.paused !== true
&& (providerName === "kiro" || account.needsReauth !== true)).length
: 0;
presence.set(presenceKey(providerName), { eligible, readAt: now });
return eligible;
}
/**
* Presence IS consent (#2568d).
*
* `hasKeyPoolFailover` already reads a 2+ key pool as the operator asking for rotation, and a
* second OAuth login is the same statement. One account stays a strict no-op either way, so this
* only changes behaviour for someone who deliberately logged in twice.
*/
export function hasFailoverAccountQuorum(providerName: string, now = Date.now()): boolean {
return eligibleAccountCount(providerName, now) >= 2;
}
/**
* Whether REACTIVE 429 rotation is active for this provider.
*
* Presence is the only rule: two or more eligible stored accounts. The
* `oauthAccountFailover.enabled` booleans no longer suppress it.
*
* That is a deliberate narrowing of #2568d. Rotation here runs only after upstream has already
* refused the request, so the choice the old knob offered was between "retry on the second
* account you deliberately logged in" and "return a 429 while that account sits idle". The
* second is a defect, not a preference — and an operator who does not want rotation expresses
* that by not storing a second account, exactly as they do for `apiKeyPool`.
*
* The knob is not gone. It still governs {@link isProactivePreferenceEnabled}, which decides
* whether a HEALTHY request may be steered to a different account before dispatch — a real
* behavioural choice that remains refusable — and it still carries `strategy` and
* `autoSwitchThreshold`.
*/
export function isGenericOAuthFailoverEnabled(
config: OcxConfig,
providerName: string,
now = Date.now(),
): boolean {
const provider = config.providers?.[providerName];
if (!provider || !isGenericFailoverProvider(providerName, provider)) return false;
return hasFailoverAccountQuorum(providerName, now);
}
/**
* Whether the pre-dispatch account PREFERENCE may run for this provider.
*
* Unlike reactive rotation, this moves a request that upstream has not refused, so it stays
* refusable: an explicit provider value wins over the global default, and a global `false`
* turns it off only when the provider has no override. A malformed value falls through rather
* than taking a provider out of service.
*/
export function isProactivePreferenceEnabled(config: OcxConfig, providerName: string, now: number): boolean {
const provider = config.providers?.[providerName];
if (!provider || !isGenericFailoverProvider(providerName, provider)) return false;
const perProvider = provider.oauthAccountFailover?.enabled;
// Preserve the published narrow-over-broad precedence. A provider-specific true may
// opt this provider into proactive preference even when the global default is false;
// a provider-specific false refuses it even when the global setting is true.
if (typeof perProvider === "boolean") {
return perProvider && hasFailoverAccountQuorum(providerName, now);
}
return config.oauthAccountFailover?.enabled === true && hasFailoverAccountQuorum(providerName, now);
}
/** Accounts that may serve traffic right now: not cooled, not flagged for reauth. */
export function eligibleFailoverAccounts(providerName: string, now = Date.now(), family?: QuotaModelFamily): string[] {
return eligibleIdsIn(getAccountSet(providerName), providerName, now, family);
}
/** Whether an account's own model listing includes `model`; undefined when unknown. */
function accountListsModel(providerName: string, accountId: string, model: string): boolean | undefined {
if (providerName === "kiro") return kiroAccountSupportsModel(accountId, model);
if (providerName === "google-antigravity") return antigravityAccountSupportsModel(providerName, accountId, model);
return undefined;
}
/**
* Model evidence only ranks accounts already eligible and below any configured cap.
*
* Kiro and Antigravity both gate models per account, but only Antigravity's roster is a gate
* (#7500). An account whose discovered roster lists the model wins; an account with no fresh
* roster is unknown and stays usable after it; an account whose fresh roster omits the model is
* never returned, so a field of only such accounts answers EMPTY rather than falling back to them.
* Callers treat empty as "no account to move to", and the transport refuses the request through
* {@link routeAntigravityModel}. Kiro listings remain a preference only.
*/
export function preferKiroModelSupport(
providerName: string, ids: string[], model?: string | null, config?: OcxConfig,
): string[] {
if (!model) return ids;
if (providerName === "google-antigravity") {
const positive = ids.filter(id => accountListsModel(providerName, id, model) === true);
const modelMatched = positive.length ? positive : ids.filter(id => accountListsModel(providerName, id, model) !== false);
const nonExhausted = modelMatched.filter(id => !isAccountQuotaExhausted(providerName, id, model));
return nonExhausted.length ? nonExhausted : modelMatched;
}
if (providerName !== "kiro") return ids;
const cap = config?.providers?.kiro?.oauthAccountFailover?.maxConcurrentPerAccount;
const withRoom = typeof cap === "number" && Number.isInteger(cap) && cap >= 1 && cap <= 100
? ids.filter(id => accountInFlight("kiro", id) < cap) : ids;
const positive = withRoom.filter(id => kiroAccountSupportsModel(id, model) === true);
return positive.length ? positive : ids;
}
/** Where an Antigravity request may be sent, judged from per-account model discovery (#7500). */
export type AntigravityModelRoute =
| { readonly kind: "keep" }
| { readonly kind: "reroute"; readonly accountId: string }
| { readonly kind: "refuse"; readonly message: string };
const KEEP_ANTIGRAVITY_ROUTE: AntigravityModelRoute = { kind: "keep" };
/** Client-facing refusal when no usable account's discovery lists the model. Mirrored in docs-site. */
export function antigravityModelUnlistedMessage(model?: ClientFacingModelLabel): string {
return `No usable Google Antigravity account lists the model "${clientFacingModelText(model)}" in its model discovery. `
+ "Choose a model your accounts list, or add an account that has access to it.";
}
/** Client-facing refusal when the chosen account omits the model but a sibling may serve it. */
export function antigravitySelectedAccountUnlistedMessage(model?: ClientFacingModelLabel): string {
return `The selected Google Antigravity account does not list the model "${clientFacingModelText(model)}" in its model discovery. `
+ "Select an account that lists it, or enable oauthAccountFailover so OpenCodex can choose one.";
}
/**
* Fail-closed model routing for Antigravity (#7500).
*
* `plannedAccountId` is the account the request would otherwise be sent with. It is kept unless
* its own fresh roster omits the model: absence of a roster is unknown, never a denial, because
* rosters live in memory, are refreshed for usable accounts during discovery, and expire
* after six hours, so a restart must not take every account out of service.
*
* A planned account with a proven omission is replaced by a usable sibling (not paused, not
* flagged for reauth) that lists the model, then by one whose roster is unknown; siblings out
* of cooldown win over cooled ones within each tier. Moving a request before upstream refused it
* is a proactive choice, so a replacement needs {@link isProactivePreferenceEnabled}, exactly as
* Kiro's known-refusal exclusion does. Otherwise, or when no sibling qualifies, the request is
* refused locally instead of being sent to an account known not to serve the model.
*/
export function routeAntigravityModel(
config: OcxConfig,
providerName: string,
plannedAccountId: string | null | undefined,
model: string | null | undefined,
now = Date.now(),
clientModel?: ClientFacingModelLabel,
): AntigravityModelRoute {
if (providerName !== "google-antigravity" || !model || !plannedAccountId
|| accountListsModel(providerName, plannedAccountId, model) !== false) return KEEP_ANTIGRAVITY_ROUTE;
const serving = (getAccountSet(providerName)?.accounts ?? [])
.filter(account => account.id !== plannedAccountId && account.paused !== true && account.needsReauth !== true)
.map(account => account.id)
.filter(id => accountListsModel(providerName, id, model) !== false);
if (serving.length === 0) return { kind: "refuse", message: antigravityModelUnlistedMessage(clientModel) };
if (!isProactivePreferenceEnabled(config, providerName, now)) {
return { kind: "refuse", message: antigravitySelectedAccountUnlistedMessage(clientModel) };
}
const ready = new Set(eligibleFailoverAccounts(providerName, now, classifyModelFamilyForQuota(providerName, model)));
const rank = (id: string) => (ready.has(id) ? 0 : 2) + (accountListsModel(providerName, id, model) === true ? 0 : 1);
const tier = Math.min(...serving.map(rank));
// Same exhaustion preference as the initial pick: a listing account whose quota family is
// spent loses to one with headroom, unless every candidate in the tier is spent.
const candidates = preferKiroModelSupport(providerName, serving.filter(id => rank(id) === tier), model, config);
const best = rankAntigravityModelAccounts(config, candidates, model, now)[0]!;
return { kind: "reroute", accountId: best };
}
/** Strategy is applied only after readiness and model-support precedence. */
function rankAntigravityModelAccounts(config: OcxConfig, ids: string[], model: string, now: number): string[] {
const provider = "google-antigravity";
const rows = new Map((getAccountSet(provider)?.accounts ?? []).map(row => [row.id, row]));
const strategy = activeGenericStrategy(config, provider);
if (strategy === "round-robin") {
const poolKey = genericPoolKey(provider, classifyModelFamilyForQuota(provider, model));
const best = peekRoundRobinAccount(poolKey, ids, genericStickyLimit(config, provider));
return best ? [best, ...ids.filter(id => id !== best)] : ids;
}
if (strategy === "fill-first") return stableGenericRoster(provider).filter(id => ids.includes(id));
if (strategy === "reset-first") return rankAccountsByResetFirst(provider, ids, model, now, rows);
return rankAccountsByHeadroom(provider, ids, model, rows);
}
/** Reactive model refusal never changes cooldown or account-wide authentication state. */
export function rotateAntigravityAccountOnModelNotFound(
config: OcxConfig, failedAccountId: string, model: string, attempted: ReadonlySet<string>, now = Date.now(),
): string | null {
const candidates = preferKiroModelSupport("google-antigravity",
eligibleFailoverAccounts("google-antigravity", now, classifyModelFamilyForQuota("google-antigravity", model))
.filter(id => id !== failedAccountId && !attempted.has(id)), model, config);
const positive = candidates.filter(id => accountListsModel("google-antigravity", id, model) === true);
return rankAntigravityModelAccounts(config, positive.length ? positive : candidates, model, now)[0] ?? null;
}
/**
* Client-facing refusal when failover is on and a sibling may serve the model, but the request
* still resolved to an omitting account (the sibling could not be resolved or admitted).
*/
export function antigravityReplacementUnavailableMessage(model?: ClientFacingModelLabel): string {
return `The selected Google Antigravity account does not list the model "${clientFacingModelText(model)}" in its model discovery, `
+ "and no account that may serve it could be used for this request. Retry the request.";
}
/** The refusal for an account that already resolved, or null when it may serve the model. */
export function antigravityModelRefusal(
config: OcxConfig, providerName: string, accountId: string, model: string | null | undefined, now = Date.now(), clientModel?: ClientFacingModelLabel,
): string | null {
const verdict = routeAntigravityModel(config, providerName, accountId, model, now, clientModel);
if (verdict.kind === "keep") return null;
// A reroute verdict exists only with failover on, so advising to enable it would be wrong.
return verdict.kind === "refuse" ? verdict.message : antigravityReplacementUnavailableMessage(clientModel);
}
function kiroModelListingHasRoom(config: OcxConfig, accountId: string, model: string): boolean {
if (kiroAccountSupportsModel(accountId, model) !== true) return false;
const cap = config.providers?.kiro?.oauthAccountFailover?.maxConcurrentPerAccount;
return !(typeof cap === "number" && Number.isInteger(cap) && cap >= 1 && cap <= 100)
|| accountInFlight("kiro", accountId) < cap;
}
function eligibleIdsIn(
set: ReturnType<typeof getAccountSet>,
providerName: string,
now: number,
family?: QuotaModelFamily,
): string[] {
if (!set) return [];
return set.accounts
.filter(account => account.paused !== true
&& (providerName === "kiro" ? kiroAutoSelection(account, now).autoSelectable
: account.needsReauth !== true && !isCooled(providerName, account.id, now, family)))
.map(account => account.id);
}
/**
* Whether reactive rotation has an alternate account it could select right now.
*
* Answers from the same live roster read and the same guards `rotateGenericOAuthAccountOn429`
* applies: a roster of fewer than two accounts has nowhere to go, even when a cached quorum
* count or a stale failed id would suggest otherwise. It applies no cooldown and advances no
* rotation state; like every eligibility read, it may prune an already-expired cooldown entry.
*/
export function hasEligibleGenericOAuthFailoverTarget(
providerName: string,
failedAccountId: string,
now = Date.now(),
requestedModelId?: string | null,
): boolean {
const set = getAccountSet(providerName);
if (!set || set.accounts.length < 2) return false;
const family = classifyModelFamilyForQuota(providerName, requestedModelId);
// Same model filter rotation applies, so an Antigravity sibling whose roster omits the
// model is not advertised as a target rotation would then refuse to pick.
const siblings = eligibleIdsIn(set, providerName, now, family).filter(id => id !== failedAccountId);
return (providerName === "google-antigravity"
? preferKiroModelSupport(providerName, siblings, requestedModelId) : siblings).length > 0;
}
/** Select a live Antigravity sibling using activation captured before the refused send. */
export function rotateAntigravityAccountOnAuthRefusal(
poolActivated: boolean,
failedAccountId: string,
sentGeneration: string,
requestedModelId: string | null | undefined,
now = Date.now(),
): string | null {
if (!poolActivated) return null;
const set = getAccountSet("google-antigravity");
if (!set || set.accounts.length < 2) return null;
const family = classifyModelFamilyForQuota("google-antigravity", requestedModelId);
coolAntigravityAccount(set, failedAccountId, sentGeneration, family, now, {
cooldownUntil: now + DEFAULT_COOLDOWN_MS, cooldownSource: "auth", identity: sentGeneration,
});
return nextAntigravitySibling(set, failedAccountId, requestedModelId, family, now);
}
/**
* The one cooldown write both Antigravity sibling rotations (403 refusal, 5xx fault) share. It is
* fenced on the sent credential generation, so a re-login that landed mid-request is never cooled,
* and it never shortens a cooldown already running for the same account and family: a 15-second
* fault window must not cut short a 60-second refusal or a quota cooldown.
*/
function coolAntigravityAccount(
set: NonNullable<ReturnType<typeof getAccountSet>>,
failedAccountId: string,
sentGeneration: string,
family: QuotaModelFamily | undefined,
now: number,
entry: AccountHealth,
): void {
const failed = set.accounts.find(row => row.id === failedAccountId);
if (!failed || credentialGeneration(failed.credential) !== sentGeneration) return;
const key = healthKey("google-antigravity", failedAccountId, family);
if (isCooled("google-antigravity", failedAccountId, now, family)
&& (health.get(key)?.cooldownUntil ?? 0) >= entry.cooldownUntil) return;
health.set(key, entry);
presence.delete("google-antigravity");
sweepExpiredOnWrite(now);
}
/** Next live sibling after the failed account, in roster order, honouring cooldowns and model support. */
function nextAntigravitySibling(
set: NonNullable<ReturnType<typeof getAccountSet>>,
failedAccountId: string,
requestedModelId: string | null | undefined,
family: QuotaModelFamily | undefined,
now: number,
): string | null {
const eligible = new Set(preferKiroModelSupport("google-antigravity",
eligibleIdsIn(set, "google-antigravity", now, family).filter(id => id !== failedAccountId), requestedModelId));
const order = set.accounts.map(row => row.id);
const start = order.indexOf(failedAccountId);
const ring = start >= 0 ? [...order.slice(start + 1), ...order.slice(0, start)] : order;
return ring.find(id => id !== failedAccountId && eligible.has(id)) ?? null;
}
/**
* A 502/503/504 that survived the same-account retry ladder says this account's route to Google is
* failing right now, not that the credential is bad. Cool it for a short, fixed window (pool-wide
* capacity trouble then clears on its own within seconds) and name a live sibling to replay on.
* No stored verdict: nothing here outlives the process or the window.
*/
const ANTIGRAVITY_FAULT_COOLDOWN_MS = 15_000;
export function rotateAntigravityAccountOnUpstreamFault(
poolActivated: boolean,
failedAccountId: string,
sentGeneration: string,
requestedModelId: string | null | undefined,
now = Date.now(),
): string | null {
if (!poolActivated) return null;
const set = getAccountSet("google-antigravity");
if (!set || set.accounts.length < 2) return null;
const family = classifyModelFamilyForQuota("google-antigravity", requestedModelId);
coolAntigravityAccount(set, failedAccountId, sentGeneration, family, now, {
cooldownUntil: now + ANTIGRAVITY_FAULT_COOLDOWN_MS, cooldownSource: "default",
});
return nextAntigravitySibling(set, failedAccountId, requestedModelId, family, now);
}
/** Generic pool strategies the kernel can actually run. `quota` IS the pre-kernel path. */
type ActiveGenericStrategy = "round-robin" | "fill-first" | "reset-first" | "least-loaded";
/** Matches the Codex and Anthropic pools; the DTO still reports `null` for "not stored". */
const DEFAULT_GENERIC_AUTO_SWITCH_THRESHOLD = 80;
/**
* The strategy this provider's pool actually runs, or null for today's behaviour.
*
* Three different inputs answer null and they all mean the same thing to a caller: the flag is
* off, no strategy is stored, or the stored strategy is `quota` — which is precisely what the
* unflagged code already does. Collapsing them here is what keeps every call site a two-way
* branch instead of a four-way one.
*/
function activeGenericStrategy(config: OcxConfig, providerName: string): ActiveGenericStrategy | null {
// All pool strategies run behind the kernel flag; `quota` is the pre-kernel path.
// Letting `reset-first` bypass it would change 429 selection for pools that never
// opted into kernel-gated strategies.
if (config.pool?.kernel !== true) return null;
const raw = config.providers?.[providerName]?.oauthAccountFailover?.strategy;
if (providerName === "kiro" && raw === "least-loaded") return raw;
return raw === "round-robin" || raw === "fill-first" || raw === "reset-first" ? raw : null;
}
function genericStickyLimit(config: OcxConfig, providerName: string): number {
return normalizeAccountPoolStickyLimit(config.providers?.[providerName]?.oauthAccountFailover?.stickyLimit);
}
/**
* The FULL roster in a stable order, not the eligible subset.
*
* Two load-bearing reasons. The store holds accounts in LOGIN order, so two operators who added
* the same accounts in a different sequence would otherwise rotate differently; sorting makes
* the ring a property of the accounts rather than of the history. And walking the eligible
* subset instead of the full roster changes the wrap order whenever an ineligible id sits
* between two eligible ones — the bug the Codex and Anthropic copies carry a `stableAll`
* argument to avoid.
*/
function stableGenericRoster(providerName: string): string[] {
const set = getAccountSet(providerName);
if (!set) return [];
return set.accounts.map(account => account.id).sort((left, right) => left.localeCompare(right));
}
/**
* Has this account spent enough of its allowance for fill-first to move on?
*
* An unmeasured account reads as UNDER the threshold, matching the Codex pool: a threshold is a
* statement about observed usage, and treating "no observation" as "spent" would evacuate every
* quota-less provider off its active account on the very first request.
*/
function isOverAutoSwitchThreshold(providerName: string, accountId: string, threshold: number, requestedModelId?: string | null, account?: ProviderAccount): boolean {
if (threshold <= 0) return false;
const headroom = accountHeadroomPercent(providerName, accountId, requestedModelId, account);
if (headroom === null) return false;
return 100 - headroom >= threshold;
}
/**
* Fill-first: stay on the active account until it crosses its threshold, then take the next
* eligible account in the stable ring. Null means "keep the active account".
*/
function pickFillFirstGenericAccount(
config: OcxConfig,
providerName: string,
activeId: string | undefined,
now: number,
requestedModelId?: string | null,
accountRows?: ReadonlyMap<string, ProviderAccount>,
preferBelowThreshold = false,
): string | null {
const stableAll = stableGenericRoster(providerName);
if (stableAll.length < 2) return null;
const family = classifyModelFamilyForQuota(providerName, requestedModelId);
const eligible = new Set(preferKiroModelSupport(providerName,
eligibleFailoverAccounts(providerName, now, family), requestedModelId, config));
const stored = config.providers?.[providerName]?.oauthAccountFailover?.autoSwitchThreshold;
const threshold = typeof stored === "number" && Number.isInteger(stored) && stored >= 0 && stored <= 100
? stored
: DEFAULT_GENERIC_AUTO_SWITCH_THRESHOLD;
if (activeId && eligible.has(activeId) && !isOverAutoSwitchThreshold(providerName, activeId, threshold, requestedModelId, accountRows?.get(activeId))) {
return null;
}
const start = activeId ? stableAll.indexOf(activeId) : -1;
const ring = start >= 0 ? [...stableAll.slice(start + 1), ...stableAll.slice(0, start)] : stableAll;
if (preferBelowThreshold) {
const below = ring.find(id => id !== activeId && eligible.has(id)
&& !isOverAutoSwitchThreshold(providerName, id, threshold, requestedModelId, accountRows?.get(id)));
if (below) return below;
}
for (const id of ring) {
if (id !== activeId && eligible.has(id)) return id;
}
return null;
}
/**
* Advance the round-robin cursor once a dispatch has actually been admitted on this account.
*
* The early return is the whole safety story for the core path: this is reached on EVERY
* generic first dispatch, including quota pools and the fallback after a preferred account was
* dropped, so anything but round-robin must leave the cursor untouched.
*
* The live pick belongs here rather than in the proposal, and that is not stylistic.
* `peekRoundRobinAccount` never creates the pool state and `notePoolRotationSuccess` returns
* immediately when there is none, so a peek-only path would leave the ring with nothing to
* advance and round-robin would propose the same account forever. This is the same shape
* `commitAnthropicSelectionRouting` already commits with.
*/
export function noteGenericPoolSelection(
config: OcxConfig,
providerName: string,
accountId: string,
requestedModelId?: string | null,
): void {
if (activeGenericStrategy(config, providerName) !== "round-robin") return;
const family = classifyModelFamilyForQuota(providerName, requestedModelId);
const poolKey = genericPoolKey(providerName, family);
const limit = genericStickyLimit(config, providerName);
const eligible = preferKiroModelSupport(
providerName,
eligibleFailoverAccounts(providerName, Date.now(), family),
requestedModelId,
config,
);
const picked = pickRoundRobinAccount(poolKey, eligible, limit);
// The resolver may have admitted a different account than the ring proposed: a removal, a
// reauth verdict or a manual selection can land during credential resolution. Realign the
// cursor onto what actually served rather than leaving it on a road not taken.
if (picked !== accountId) seedPoolRotationAccount(poolKey, accountId);
notePoolRotationSuccess(poolKey, accountId, limit);
}
/**
* Cool the account that actually 429'd and name the next eligible one, or null.
*
* Returns the id only; the caller mints the credential so a failed refresh does not leave the
* cooldown applied to an account we then could not use. `commit: false` still cools the failed
* account but only peeks the round-robin ring, for callers that will not dispatch the answer.
*/
export function rotateGenericOAuthAccountOn429(
config: OcxConfig,
providerName: string,
failedAccountId: string,
retryAfterHeader: string | null | undefined,
now = Date.now(),
requestedModelId?: string | null,
commit = true,
): string | null {
return rotateGenericOAuthAccountOnRefusal(config, providerName, failedAccountId, "rate",
retryAfterHeader, now, requestedModelId, undefined, commit);
}
/** Kiro adds account-scoped refusal classes; all other providers retain their 429 policy. */
export function rotateGenericOAuthAccountOnRefusal(
config: OcxConfig,
providerName: string,
failedAccountId: string,
kind: KiroRefusalKind,
retryAfterHeader: string | null | undefined,
now = Date.now(),
requestedModelId?: string | null,
monthlyCooldownMs?: number,
commit = true,
): string | null {
if (kind === "other" || (providerName !== "kiro" && kind !== "rate")) return null;
if (!isGenericOAuthFailoverEnabled(config, providerName)) return null;
const set = getAccountSet(providerName);
// A single stored account has nowhere to go; rotating to itself would just replay the 429.
if (!set || set.accounts.length < 2) return null;
// `preserveServerDelay` keeps the delay the server actually stated, bounded by the parser's
// one-day ceiling, exactly as the combo path does. Truncating it locally only guarantees a
// second 429 on an account we were told to leave alone.
const parsed = kind === "rate"
? parseRetryAfterMs(retryAfterHeader, now, { preserveImmediate: true, preserveServerDelay: true })
: undefined;
// An account whose allowance is provably spent gets a reset-aligned cooldown instead of
// the default minute: retrying it every 60s until the window rolls over is pure waste.
// A Retry-After from upstream still wins — it is the server's own instruction.
const exhausted = parsed === undefined && providerName !== "kiro"
? exhaustedCooldownMs(providerName, failedAccountId, now, set.accounts.find(account => account.id === failedAccountId)) : null;
const cooldownMs = kind === "suspended" ? 24 * 60 * 60_000
: kind === "monthly_quota" ? (monthlyCooldownMs ?? ACCOUNT_QUOTA_TTL_MS)
: providerName === "kiro" ? (parsed ?? 10_000)
: (exhausted ?? parsed ?? DEFAULT_COOLDOWN_MS);
const family = classifyModelFamilyForQuota(providerName, requestedModelId);
// Suspension is recorded by the caller with the sent generation before the quorum check.
// Rewriting it here would lose its login-identity fence.
if (kind !== "suspended") health.set(healthKey(providerName, failedAccountId, providerName === "kiro" ? undefined : family), {
cooldownUntil: now + cooldownMs,
cooldownSource: parsed ? "retry-after" : "default",
});
sweepExpiredOnWrite(now);
const eligible = eligibleFailoverAccounts(providerName, now, family).filter(id => id !== failedAccountId);
if (eligible.length === 0) return null;
// A rotation means the roster in use just changed; do not answer the next activation question
// from a count read before the failure.
presence.delete(presenceKey(providerName));
// Deterministic: start after the failed account so repeated 429s walk the roster instead of
// hammering whichever id happens to sort first. The ring is built BEFORE ranking — ranking
// the store's own order would change which account a quota-less provider rotates to.
const order = set.accounts.map(account => account.id);
const start = order.indexOf(failedAccountId);
const ring = start >= 0 ? [...order.slice(start + 1), ...order.slice(0, start)] : order;
let candidates = ring.filter(id => id !== failedAccountId && eligible.includes(id));
if (candidates.length === 0) return null;
const cap = providerName === "kiro" ? config.providers?.kiro?.oauthAccountFailover?.maxConcurrentPerAccount : undefined;
if (typeof cap === "number" && Number.isInteger(cap) && cap >= 1 && cap <= 100) {
const withRoom = candidates.filter(id => accountInFlight("kiro", id) < cap);
if (withRoom.length > 0) candidates = withRoom;
}
candidates = preferKiroModelSupport(providerName, candidates, requestedModelId, config);
// The 429 path branches too. Leaving it on the quota ranking would make a configured
// strategy inert in practice the moment anything actually failed, which is the case the
// operator chose the strategy for.
const strategy = activeGenericStrategy(config, providerName);
if (strategy === "least-loaded" && isProactivePreferenceEnabled(config, "kiro", now)) {
return candidates.sort((a, b) => accountInFlight("kiro", a) - accountInFlight("kiro", b))[0] ?? null;
}
if (strategy === "round-robin") {
// PICK here, not peek: the failure already happened and this answer is the one being used,
// so the ring genuinely advances. A caller that only asks whether a sibling exists peeks.
return (commit ? pickRoundRobinAccount : peekRoundRobinAccount)(
genericPoolKey(providerName, family),
candidates,
genericStickyLimit(config, providerName),
);
}
if (strategy === "fill-first") {
// Not "keep the active account": the one that just 429'd is cooled, so fill-first takes
// the next eligible account in the stable ring rather than its usual hold.
const stableAll = stableGenericRoster(providerName);
const from = stableAll.indexOf(failedAccountId);
const walk = from >= 0 ? [...stableAll.slice(from + 1), ...stableAll.slice(0, from)] : stableAll;
for (const id of walk) {
if (id !== failedAccountId && candidates.includes(id)) return id;
}
return null;
}
if (strategy === "reset-first") {
return rankAccountsByResetFirst(providerName, candidates, requestedModelId, now,
new Map(set.accounts.map(account => [account.id, account])))[0] ?? null;
}
// With no quota evidence this returns the ring untouched, so providers without
// per-account quota keep exactly the traversal they have today.
return rankAccountsByHeadroom(providerName, candidates, requestedModelId,
new Map(set.accounts.map(account => [account.id, account])))[0] ?? null;
}
/** Known dead-account exclusion is a proactive choice with narrow-over-broad precedence. */
export function refusalAwareInitialKiroAccount(
config: OcxConfig, activeId: string, now = Date.now(), requestedModelId?: string | null,
): string | null {
if (!isProactivePreferenceEnabled(config, "kiro", now)) return null;
const set = getAccountSet("kiro");
if (!set || set.accounts.length < 2) return null;
const active = set.accounts.find(account => account.id === activeId);
if (!active) return null;
const eligibleIds = eligibleIdsIn(set, "kiro", now, classifyModelFamilyForQuota("kiro", requestedModelId));
const preferred = preferKiroModelSupport("kiro", eligibleIds, requestedModelId, config);
const activeLacks = requestedModelId && kiroAccountSupportsModel(activeId, requestedModelId) === false
&& preferred.some(id => id !== activeId && kiroModelListingHasRoom(config, id, requestedModelId));
if (!activeLacks && !isCooled("kiro", activeId, now)
&& kiroAccountEvidence(active, now).exhausted !== true) return null;
const eligible = new Set(preferred);
const start = set.accounts.findIndex(account => account.id === activeId);
for (const account of [...set.accounts.slice(start + 1), ...set.accounts.slice(0, start)]) {
if (eligible.has(account.id)) return account.id;
}
return null;
}
/**
* Full credential snapshot for a rotated account.
*
* Returns the snapshot rather than a bare bearer: Antigravity pairs an account-matched
* `projectId` with its token and Kiro carries routing metadata, so a token-only swap would mix
* one account's bearer with another's routing data.
*/
export async function failoverAccountSnapshot(
providerName: string,
accountId: string,
): Promise<OAuthAccessSnapshot> {
return getValidAccessSnapshotForAccount(providerName, accountId);
}
/**
* Which account should serve the FIRST attempt of a request.
*
* Rotation only ever ran after a 429, so a turn still opened on whichever account happened
* to be active — including one a previous probe already measured as spent. That costs a
* full upstream round trip and one of three rotations to rediscover what the cache knew.
*
* Returns null whenever the ordinary active-account path should be used unchanged: no
* quorum, rotation disabled, a single account, or no quota evidence to act on. This is a
* preference, never a gate — a cooled or unmeasured account is still perfectly usable, so
* an empty answer means "carry on", not "refuse".
*/
export function preferredInitialAccount(
config: OcxConfig,
providerName: string,
now = Date.now(),
requestedModelId?: string | null,
activeAccountOverride?: string,
): string | null {
// The PROACTIVE predicate, not the reactive one: this steers a request upstream has not
// refused, so `oauthAccountFailover.enabled: false` must still be able to refuse it.
if (!isProactivePreferenceEnabled(config, providerName, now)) return null;
// Read the same authoritative selection the management writer commits. Caching the
// active id separately would delay manual selection and account removal.
const selected = getAccountSet(providerName);
if (!selected) return null;
const active = activeAccountOverride ?? selected.activeAccountId;
const accountRows = new Map(selected.accounts.map(account => [account.id, account]));
const order = selected.accounts
.filter(account => account.paused !== true && account.needsReauth !== true)
.map(account => account.id);
if (order.length < 2) return null;
const modelEligible = preferKiroModelSupport(providerName,
eligibleFailoverAccounts(providerName, now, classifyModelFamilyForQuota(providerName, requestedModelId)),
requestedModelId, config);
const activeLacks = (providerName === "kiro" || providerName === "google-antigravity")
&& !!active && !!requestedModelId
&& accountListsModel(providerName, active, requestedModelId) === false;
if (providerName === "google-antigravity" && activeLacks) {
const route = routeAntigravityModel(config, providerName, active, requestedModelId, now);
return route.kind === "reroute" ? route.accountId : null;
}
const listingSibling = activeLacks
? modelEligible.find(id => id !== active && (providerName === "kiro"
? kiroModelListingHasRoom(config, id, requestedModelId!)
: accountListsModel(providerName, id, requestedModelId!) === true))
: undefined;
if (listingSibling) return listingSibling;
// A configured strategy answers this question itself. Both guards below exist to protect the
// QUOTA answer, and both are fatal to the other two: hasHeadroomEvidence refuses every
// provider with no quota data, which is exactly where round-robin is the point, and the
// healthy-active return fires before autoSwitchThreshold can ever be read, so fill-first
// would never reach its own test. Cooldowns and reauth are still honoured inside each pick.
const strategy = activeGenericStrategy(config, providerName);
if (strategy === "least-loaded" && providerName === "kiro") {
const family = classifyModelFamilyForQuota(providerName, requestedModelId);
let candidates = eligibleFailoverAccounts(providerName, now, family);
const cap = config.providers?.kiro?.oauthAccountFailover?.maxConcurrentPerAccount;
if (typeof cap === "number" && Number.isInteger(cap) && cap >= 1 && cap <= 100) {
candidates = candidates.filter(id => accountInFlight("kiro", id) < cap);
}
candidates = preferKiroModelSupport(providerName, candidates, requestedModelId, config);
const picked = candidates.sort((a, b) => accountInFlight("kiro", a) - accountInFlight("kiro", b))[0];
return picked && picked !== active ? picked : null;
}
if (strategy === "round-robin") {
const family = classifyModelFamilyForQuota(providerName, requestedModelId);
const eligibleNow = preferKiroModelSupport(providerName,
eligibleFailoverAccounts(providerName, now, family), requestedModelId, config);
if (eligibleNow.length === 0) return null;
// PEEK, not pick: this proposal is discardable, and advancing the ring for an account the
// resolver then rejects would skip a turn for nothing. noteGenericPoolSelection commits.
const picked = peekRoundRobinAccount(
genericPoolKey(providerName, family),
eligibleNow,
genericStickyLimit(config, providerName),
);
return picked && picked !== active ? picked : null;
}
if (strategy === "fill-first") {
const picked = pickFillFirstGenericAccount(config, providerName, active, now, requestedModelId, accountRows, activeAccountOverride !== undefined);
return picked && picked !== active ? picked : null;
}
if (strategy === "reset-first") {
// Same candidate set as round-robin: paused, reauth-needed and cooled accounts are out,
// and an account whose model listing omits the requested model loses to one that has it.
// Reset ordering needs quota evidence; without it the ordinary active-account path runs.
const family = classifyModelFamilyForQuota(providerName, requestedModelId);
const eligibleNow = preferKiroModelSupport(providerName,
eligibleFailoverAccounts(providerName, now, family), requestedModelId, config);
if (eligibleNow.length === 0 || !hasHeadroomEvidence(providerName, eligibleNow, requestedModelId, accountRows)) return null;
const best = rankAccountsByResetFirst(providerName, eligibleNow, requestedModelId, now, accountRows)[0] ?? null;
return best && best !== active ? best : null;
}
const activeRow = selected.accounts.find(account => account.id === active);
if (activeRow && activeRow.paused !== true && activeRow.needsReauth !== true
&& !isCooled(providerName, activeRow.id, now, classifyModelFamilyForQuota(providerName, requestedModelId))
&& !isAccountQuotaExhausted(providerName, activeRow.id, requestedModelId, activeRow)) return null;
// Evidence is required BEFORE eligibility narrows the field. Without this, a provider
// with no quota data at all could still be redirected: cool the active account with a
// 429 and the eligible list collapses to one candidate, which any ranking returns
// unchanged — an answer that looks ranked but was never measured. The no-op guarantee
// for quota-less providers has to be checked on the full roster.
if (!hasHeadroomEvidence(providerName, order, requestedModelId, accountRows)) return null;
// Cooldowns are respected here, unlike in the presence count: this picks the account to
// send to right now, and one inside its 429 window is the single candidate we hold
// positive evidence against.
const eligible = preferKiroModelSupport(providerName,
order.filter(id => !isCooled(providerName, id, now, classifyModelFamilyForQuota(providerName, requestedModelId))),
requestedModelId, config);
if (eligible.length === 0) return null;
// Start the ring at the active account so an unranked outcome reproduces today's choice.
const start = active ? order.indexOf(active) : -1;
const ring = start >= 0 ? [...order.slice(start), ...order.slice(0, start)] : order;
const candidates = ring.filter(id => eligible.includes(id));
if (candidates.length === 0) return null;
const best = rankAccountsByHeadroom(providerName, candidates, requestedModelId, accountRows)[0] ?? null;
// Nothing to do when the ranking agrees with the account we would have used anyway.
//
// A proposal still needs guarded selection commit after credential resolution: a
// removal, reauth verdict, or manual choice can arrive during that await.
return best && best !== active ? best : null;
}
/** Earliest remaining cooldown, for a client-facing Retry-After when every account is cooled. */
export function genericFailoverRetryAfterSeconds(providerName: string, now = Date.now()): number | null {
const prefix = `${oauthCredentialSlot(providerName)}\u0000`;
let earliest: number | null = null;
for (const [key, entry] of health) {
if (!key.startsWith(prefix) || entry.cooldownUntil <= now) continue;
if (earliest === null || entry.cooldownUntil < earliest) earliest = entry.cooldownUntil;
}
return earliest === null ? null : Math.max(1, Math.ceil((earliest - now) / 1000));
}
/** A manual selection must win in every ring a request can read, including the per-family ones. */
export function seedGenericPoolSelection(providerName: string, accountId: string): void {
// Dispatch keys the cursor by row name, so seed every row that shares this login (kimi-responses).
const slot = oauthCredentialSlot(providerName);
for (const row of new Set([providerName, slot, ...oauthCredentialSiblingsOf(slot)])) seedPoolRotationAccount(genericPoolKey(row), accountId);
if (providerName !== "google-antigravity") return;
for (const family of ["gem", "cla"] satisfies QuotaModelFamily[]) seedPoolRotationAccount(genericPoolKey(providerName, family), accountId);
}
/** Test seam and manual-recovery hook. */
export function forgetGenericFailoverRoster(providerName: string): void {
presence.delete(presenceKey(providerName));
}
/** Test seam and manual-recovery hook. */
export function clearGenericFailoverHealth(providerName?: string): void {
if (!providerName) {
health.clear();
presence.clear();
return;
}
presence.delete(presenceKey(providerName));
const slot = oauthCredentialSlot(providerName);
for (const key of [...health.keys()]) {
if (key.startsWith(`${slot}\u0000`)) health.delete(key);
}
}
/**
* Retire one account's superseded auth evidence after an explicit (re-)login. An
* auth cooldown recorded against the old grant must not hold out the fresh
* credential, so it leaves with the login rather than lingering until its
* deadline. Rate, quota, and suspension evidence is preserved: a re-login says
* nothing about those verdicts, and dropping them would make a throttled account
* eligible early. Entries bound to the still-current generation stay as well.
*/
export function clearGenericFailoverHealthForAccount(providerName: string, accountId: string): void {
const live = getAccountSet(providerName)?.accounts.find(row => row.id === accountId);
const current = live ? credentialGeneration(live.credential) : undefined;
const bare = healthKey(providerName, accountId);
for (const key of [...health.keys()]) {
if (key !== bare && !key.startsWith(`${bare}\u0000`)) continue;
const entry = health.get(key);
if (!entry || entry.cooldownSource !== "auth") continue;
if (current !== undefined && entry.identity === current) continue;
health.delete(key);
}
}