UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

737 lines (684 loc) • 36.8 kB
/** * Generic OAuth multi-account 429 failover (#2568). * * The API-key twin (`providers/key-failover.ts`) rotates by default for any key provider with a * 2+ pool, but it returns false for `authMode === "oauth"`, and the only OAuth rotator that * exists is Anthropic's — behind its own opt-in. So xAI, Cursor, Kimi, GitHub Copilot, * Antigravity and Nous have no recovery path on a 429 even with several accounts logged in. * * Deliberately narrower than the Anthropic pool: no session affinity, no quota-ranked selection, * no probe leases. Those carry provider-specific meaning; this module only answers "the account * that just 429'd is cooled, is there another one we may use". * * NOT a home for Codex (`codex/routing.ts` owns quota scopes and probe leases) or Anthropic * (`oauth/anthropic-routing.ts` owns affinity and a fail-closed local-cli credential rule). * Both are excluded by `isGenericFailoverProvider`. */ import { credentialGeneration, getAccountSet } from "./store"; import { accountInFlight } from "./kiro-account-load"; import type { ProviderAccount } from "./types"; import { getValidAccessSnapshotForAccount, type OAuthAccessSnapshot } from "./index"; import { accountHeadroomPercent, exhaustedCooldownMs, hasHeadroomEvidence, isAccountQuotaExhausted, rankAccountsByHeadroom, classifyModelFamilyForQuota, type QuotaModelFamily, } from "./account-quota-rank"; import { genericPoolKey, normalizeAccountPoolStickyLimit, notePoolRotationSuccess, peekRoundRobinAccount, pickRoundRobinAccount, seedPoolRotationAccount, } from "./pool-kernel"; import { parseRetryAfterMs } from "../combos/failover"; import type { KiroRefusalKind } from "../adapters/kiro-refusal"; import { kiroAccountEvidence } from "../providers/kiro-usage"; import { kiroEvidenceIdentity } from "../providers/kiro-account-state-disk"; import { kiroAccountSupportsModel } from "../providers/kiro-model-catalog"; import { ACCOUNT_QUOTA_TTL_MS } from "../providers/quota-wire"; import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; import { subscribeOAuthAccountPauseChanges } from "../lib/account-selection-events"; import type { OcxConfig, OcxProviderConfig } from "../types"; /** Cap same-request rotations so a short Retry-After cannot spin. Mirrors the Anthropic bound. */ export const GENERIC_OAUTH_MAX_FAILOVERS_PER_REQUEST = 3; /** * Most accounts one request may fund and rotate through, whatever the roster size. Each funded * account gets the normal transient ladder, so this fixes the default ingress ceiling at * 6 × TRANSIENT_RETRY_MAX_ATTEMPTS physical sends: enrolling more accounts cannot turn one request * into a pool-wide 429 storm. */ export const GENERIC_OAUTH_MAX_ACCOUNTS_PER_REQUEST = 6; const DEFAULT_COOLDOWN_MS = 60_000; /** * How long a presence answer may be reused before the store is consulted again. * * `loadAuthStore` has no cache: every call chmods the config dir and the secret, reads the whole * file, parses it and normalizes the store (store.ts:136-151). Since presence now decides * activation, this predicate runs on paths that have not seen a 429 at all — the streaming and * non-streaming runTurn entry points evaluate it once per request — so an uncached check would put * a synchronous file read in front of every request for every OAuth provider. * * Two seconds is short enough that a login in another window is picked up before the operator can * switch back and send a prompt, and long enough that a burst of requests shares one read. The * cache holds a COUNT, never a credential. */ const PRESENCE_CACHE_TTL_MS = 2_000; /** * Providers whose rotation is owned elsewhere and must not be handled here. * * `openai` is the Codex pool: quota scopes, probe leases and affinity semantics that this * module deliberately does not reimplement. `anthropic` has its own pool with a fail-closed * rule about background local-cli credential slots. */ const EXCLUDED_PROVIDERS = new Set(["openai", "anthropic"]); interface AccountHealth { cooldownUntil: number; cooldownSource: "retry-after" | "default" | "kiro-suspension" | "auth"; identity?: string; } interface PresenceEntry { eligible: number; readAt: number; } /** Process-local, like the Anthropic pool's: a restart is allowed to forget a cooldown. */ const health = new Map<string, AccountHealth>(); /** Provider -> recent eligible-account count. TTL-bounded; never holds credential material. */ const presence = new Map<string, PresenceEntry>(); subscribeOAuthAccountPauseChanges(provider => presence.delete(provider)); const healthKey = (provider: string, accountId: string, family?: QuotaModelFamily) => family ? `${provider}\u0000${accountId}\u0000${family}` : `${provider}\u0000${accountId}`; export function quarantineKiroSuspendedAccount(accountId: string, generation?: string, now = Date.now()): void { const live = getAccountSet("kiro")?.accounts.find(row => row.id === accountId); if (!live || (generation && credentialGeneration(live.credential) !== generation)) return; health.set(healthKey("kiro", accountId), { cooldownUntil: now + 24 * 60 * 60_000, cooldownSource: "kiro-suspension", identity: kiroEvidenceIdentity(live), }); sweepExpiredOnWrite(now); } function isCooled(provider: string, accountId: string, now: number, family?: QuotaModelFamily): boolean { const entry = health.get(healthKey(provider, accountId, family)); if (!entry) return false; if (provider === "google-antigravity" && entry.cooldownSource === "auth") { const live = getAccountSet(provider)?.accounts.find(row => row.id === accountId); if (!live || entry.identity !== credentialGeneration(live.credential)) { health.delete(healthKey(provider, accountId, family)); return false; } } if (provider === "kiro" && entry.cooldownSource === "kiro-suspension") { const live = getAccountSet("kiro")?.accounts.find(row => row.id === accountId); if (!live || entry.identity !== kiroEvidenceIdentity(live)) { health.delete(healthKey(provider, accountId, family)); return false; } } if (entry.cooldownUntil <= now) { health.delete(healthKey(provider, accountId, family)); return false; } return true; } export type KiroSkipReason = "paused" | "needs_reauth" | "suspended" | "cooldown" | "quota_exhausted"; /** Eligibility for automatic alternatives; an active singleton can still send. */ export function kiroAutoSelection( account: ProviderAccount, now = Date.now(), ): { autoSelectable: boolean; skipReason?: KiroSkipReason } { if (account.paused === true) return { autoSelectable: false, skipReason: "paused" }; if (account.needsReauth === true) return { autoSelectable: false, skipReason: "needs_reauth" }; const cooled = isCooled("kiro", account.id, now); if (cooled && health.get(healthKey("kiro", account.id))?.cooldownSource === "kiro-suspension") return { autoSelectable: false, skipReason: "suspended" }; if (kiroAccountEvidence(account, now).exhausted === true) return { autoSelectable: false, skipReason: "quota_exhausted" }; if (cooled) return { autoSelectable: false, skipReason: "cooldown" }; return { autoSelectable: true }; } /** True when this provider participates in generic rotation at all. */ export function isGenericFailoverProvider(providerName: string, provider: OcxProviderConfig): boolean { return provider.authMode === "oauth" && !EXCLUDED_PROVIDERS.has(providerName); } /** * Stored accounts that could serve traffic if asked, ignoring cooldowns. * * Cooldowns are excluded on purpose: they are transient and per-request, while this answers the * durable question "did the operator log in more than one account". Treating a cooled account as * absent would switch the feature off for the rest of the cooldown, which is exactly when it is * needed. */ function eligibleAccountCount(providerName: string, now: number): number { const cached = presence.get(providerName); if (cached && now >= cached.readAt && now - cached.readAt < PRESENCE_CACHE_TTL_MS) return cached.eligible; const set = getAccountSet(providerName); // Kiro terminal refresh marks the just-refused account needsReauth before the alternate // selector runs. Both stored logins still express consent to recover through the survivor. const eligible = set ? set.accounts.filter(account => account.paused !== true && (providerName === "kiro" || account.needsReauth !== true)).length : 0; presence.set(providerName, { eligible, readAt: now }); return eligible; } /** * Presence IS consent (#2568d). * * `hasKeyPoolFailover` already reads a 2+ key pool as the operator asking for rotation, and a * second OAuth login is the same statement. One account stays a strict no-op either way, so this * only changes behaviour for someone who deliberately logged in twice. */ export function hasFailoverAccountQuorum(providerName: string, now = Date.now()): boolean { return eligibleAccountCount(providerName, now) >= 2; } /** * Whether REACTIVE 429 rotation is active for this provider. * * Presence is the only rule: two or more eligible stored accounts. The * `oauthAccountFailover.enabled` booleans no longer suppress it. * * That is a deliberate narrowing of #2568d. Rotation here runs only after upstream has already * refused the request, so the choice the old knob offered was between "retry on the second * account you deliberately logged in" and "return a 429 while that account sits idle". The * second is a defect, not a preference — and an operator who does not want rotation expresses * that by not storing a second account, exactly as they do for `apiKeyPool`. * * The knob is not gone. It still governs {@link isProactivePreferenceEnabled}, which decides * whether a HEALTHY request may be steered to a different account before dispatch — a real * behavioural choice that remains refusable — and it still carries `strategy` and * `autoSwitchThreshold`. */ export function isGenericOAuthFailoverEnabled( config: OcxConfig, providerName: string, now = Date.now(), ): boolean { const provider = config.providers?.[providerName]; if (!provider || !isGenericFailoverProvider(providerName, provider)) return false; return hasFailoverAccountQuorum(providerName, now); } /** * Whether the pre-dispatch account PREFERENCE may run for this provider. * * Unlike reactive rotation, this moves a request that upstream has not refused, so it stays * refusable: an explicit provider value wins over the global default, and a global `false` * turns it off only when the provider has no override. A malformed value falls through rather * than taking a provider out of service. */ export function isProactivePreferenceEnabled(config: OcxConfig, providerName: string, now: number): boolean { const provider = config.providers?.[providerName]; if (!provider || !isGenericFailoverProvider(providerName, provider)) return false; const perProvider = provider.oauthAccountFailover?.enabled; // Preserve the published narrow-over-broad precedence. A provider-specific true may // opt this provider into proactive preference even when the global default is false; // a provider-specific false refuses it even when the global setting is true. if (typeof perProvider === "boolean") { return perProvider && hasFailoverAccountQuorum(providerName, now); } return config.oauthAccountFailover?.enabled === true && hasFailoverAccountQuorum(providerName, now); } /** Accounts that may serve traffic right now: not cooled, not flagged for reauth. */ export function eligibleFailoverAccounts(providerName: string, now = Date.now(), family?: QuotaModelFamily): string[] { return eligibleIdsIn(getAccountSet(providerName), providerName, now, family); } /** Model evidence only ranks accounts already eligible and below any configured cap. */ export function preferKiroModelSupport( providerName: string, ids: string[], model?: string | null, config?: OcxConfig, ): string[] { if (providerName !== "kiro" || !model) return ids; const cap = config?.providers?.kiro?.oauthAccountFailover?.maxConcurrentPerAccount; const withRoom = typeof cap === "number" && Number.isInteger(cap) && cap >= 1 && cap <= 100 ? ids.filter(id => accountInFlight("kiro", id) < cap) : ids; const positive = withRoom.filter(id => kiroAccountSupportsModel(id, model) === true); return positive.length ? positive : ids; } function kiroModelListingHasRoom(config: OcxConfig, accountId: string, model: string): boolean { if (kiroAccountSupportsModel(accountId, model) !== true) return false; const cap = config.providers?.kiro?.oauthAccountFailover?.maxConcurrentPerAccount; return !(typeof cap === "number" && Number.isInteger(cap) && cap >= 1 && cap <= 100) || accountInFlight("kiro", accountId) < cap; } function eligibleIdsIn( set: ReturnType<typeof getAccountSet>, providerName: string, now: number, family?: QuotaModelFamily, ): string[] { if (!set) return []; return set.accounts .filter(account => account.paused !== true && (providerName === "kiro" ? kiroAutoSelection(account, now).autoSelectable : account.needsReauth !== true && !isCooled(providerName, account.id, now, family))) .map(account => account.id); } /** * Whether reactive rotation has an alternate account it could select right now. * * Answers from the same live roster read and the same guards `rotateGenericOAuthAccountOn429` * applies: a roster of fewer than two accounts has nowhere to go, even when a cached quorum * count or a stale failed id would suggest otherwise. It applies no cooldown and advances no * rotation state; like every eligibility read, it may prune an already-expired cooldown entry. */ export function hasEligibleGenericOAuthFailoverTarget( providerName: string, failedAccountId: string, now = Date.now(), requestedModelId?: string | null, ): boolean { const set = getAccountSet(providerName); if (!set || set.accounts.length < 2) return false; const family = classifyModelFamilyForQuota(providerName, requestedModelId); return eligibleIdsIn(set, providerName, now, family).some(id => id !== failedAccountId); } /** Select a live Antigravity sibling using activation captured before the refused send. */ export function rotateAntigravityAccountOnAuthRefusal( poolActivated: boolean, failedAccountId: string, sentGeneration: string, requestedModelId: string | null | undefined, now = Date.now(), ): string | null { if (!poolActivated) return null; const set = getAccountSet("google-antigravity"); if (!set || set.accounts.length < 2) return null; const family = classifyModelFamilyForQuota("google-antigravity", requestedModelId); const failed = set.accounts.find(row => row.id === failedAccountId); if (failed && credentialGeneration(failed.credential) === sentGeneration) { health.set(healthKey("google-antigravity", failedAccountId, family), { cooldownUntil: now + DEFAULT_COOLDOWN_MS, cooldownSource: "auth", identity: sentGeneration, }); sweepExpiredOnWrite(now); } const eligible = new Set(eligibleIdsIn(set, "google-antigravity", now, family)); const order = set.accounts.map(row => row.id); const start = order.indexOf(failedAccountId); const ring = start >= 0 ? [...order.slice(start + 1), ...order.slice(0, start)] : order; return ring.find(id => id !== failedAccountId && eligible.has(id)) ?? null; } /** Generic pool strategies the kernel can actually run. `quota` IS the pre-kernel path. */ type ActiveGenericStrategy = "round-robin" | "fill-first" | "least-loaded"; /** Matches the Codex and Anthropic pools; the DTO still reports `null` for "not stored". */ const DEFAULT_GENERIC_AUTO_SWITCH_THRESHOLD = 80; /** * The strategy this provider's pool actually runs, or null for today's behaviour. * * Three different inputs answer null and they all mean the same thing to a caller: the flag is * off, no strategy is stored, or the stored strategy is `quota` — which is precisely what the * unflagged code already does. Collapsing them here is what keeps every call site a two-way * branch instead of a four-way one. */ function activeGenericStrategy(config: OcxConfig, providerName: string): ActiveGenericStrategy | null { if (config.pool?.kernel !== true) return null; const raw = config.providers?.[providerName]?.oauthAccountFailover?.strategy; if (providerName === "kiro" && raw === "least-loaded") return raw; return raw === "round-robin" || raw === "fill-first" ? raw : null; } function genericStickyLimit(config: OcxConfig, providerName: string): number { return normalizeAccountPoolStickyLimit(config.providers?.[providerName]?.oauthAccountFailover?.stickyLimit); } /** * The FULL roster in a stable order, not the eligible subset. * * Two load-bearing reasons. The store holds accounts in LOGIN order, so two operators who added * the same accounts in a different sequence would otherwise rotate differently; sorting makes * the ring a property of the accounts rather than of the history. And walking the eligible * subset instead of the full roster changes the wrap order whenever an ineligible id sits * between two eligible ones — the bug the Codex and Anthropic copies carry a `stableAll` * argument to avoid. */ function stableGenericRoster(providerName: string): string[] { const set = getAccountSet(providerName); if (!set) return []; return set.accounts.map(account => account.id).sort((left, right) => left.localeCompare(right)); } /** * Has this account spent enough of its allowance for fill-first to move on? * * An unmeasured account reads as UNDER the threshold, matching the Codex pool: a threshold is a * statement about observed usage, and treating "no observation" as "spent" would evacuate every * quota-less provider off its active account on the very first request. */ function isOverAutoSwitchThreshold(providerName: string, accountId: string, threshold: number, requestedModelId?: string | null, account?: ProviderAccount): boolean { if (threshold <= 0) return false; const headroom = accountHeadroomPercent(providerName, accountId, requestedModelId, account); if (headroom === null) return false; return 100 - headroom >= threshold; } /** * Fill-first: stay on the active account until it crosses its threshold, then take the next * eligible account in the stable ring. Null means "keep the active account". */ function pickFillFirstGenericAccount( config: OcxConfig, providerName: string, activeId: string | undefined, now: number, requestedModelId?: string | null, accountRows?: ReadonlyMap<string, ProviderAccount>, ): string | null { const stableAll = stableGenericRoster(providerName); if (stableAll.length < 2) return null; const family = classifyModelFamilyForQuota(providerName, requestedModelId); const eligible = new Set(preferKiroModelSupport(providerName, eligibleFailoverAccounts(providerName, now, family), requestedModelId, config)); const stored = config.providers?.[providerName]?.oauthAccountFailover?.autoSwitchThreshold; const threshold = typeof stored === "number" && Number.isInteger(stored) && stored >= 0 && stored <= 100 ? stored : DEFAULT_GENERIC_AUTO_SWITCH_THRESHOLD; if (activeId && eligible.has(activeId) && !isOverAutoSwitchThreshold(providerName, activeId, threshold, requestedModelId, accountRows?.get(activeId))) { return null; } const start = activeId ? stableAll.indexOf(activeId) : -1; const ring = start >= 0 ? [...stableAll.slice(start + 1), ...stableAll.slice(0, start)] : stableAll; for (const id of ring) { if (id !== activeId && eligible.has(id)) return id; } return null; } /** * Advance the round-robin cursor once a dispatch has actually been admitted on this account. * * The early return is the whole safety story for the core path: this is reached on EVERY * generic first dispatch, including quota pools and the fallback after a preferred account was * dropped, so anything but round-robin must leave the cursor untouched. * * The live pick belongs here rather than in the proposal, and that is not stylistic. * `peekRoundRobinAccount` never creates the pool state and `notePoolRotationSuccess` returns * immediately when there is none, so a peek-only path would leave the ring with nothing to * advance and round-robin would propose the same account forever. This is the same shape * `commitAnthropicSelectionRouting` already commits with. */ export function noteGenericPoolSelection( config: OcxConfig, providerName: string, accountId: string, requestedModelId?: string | null, ): void { if (activeGenericStrategy(config, providerName) !== "round-robin") return; const poolKey = genericPoolKey(providerName); const limit = genericStickyLimit(config, providerName); const family = classifyModelFamilyForQuota(providerName, requestedModelId); const picked = pickRoundRobinAccount(poolKey, eligibleFailoverAccounts(providerName, Date.now(), family), limit); // The resolver may have admitted a different account than the ring proposed: a removal, a // reauth verdict or a manual selection can land during credential resolution. Realign the // cursor onto what actually served rather than leaving it on a road not taken. if (picked !== accountId) seedPoolRotationAccount(poolKey, accountId); notePoolRotationSuccess(poolKey, accountId, limit); } /** * Cool the account that actually 429'd and name the next eligible one, or null. * * Returns the id only; the caller mints the credential so a failed refresh does not leave the * cooldown applied to an account we then could not use. */ export function rotateGenericOAuthAccountOn429( config: OcxConfig, providerName: string, failedAccountId: string, retryAfterHeader: string | null | undefined, now = Date.now(), requestedModelId?: string | null, ): string | null { return rotateGenericOAuthAccountOnRefusal(config, providerName, failedAccountId, "rate", retryAfterHeader, now, requestedModelId); } /** Kiro adds account-scoped refusal classes; all other providers retain their 429 policy. */ export function rotateGenericOAuthAccountOnRefusal( config: OcxConfig, providerName: string, failedAccountId: string, kind: KiroRefusalKind, retryAfterHeader: string | null | undefined, now = Date.now(), requestedModelId?: string | null, monthlyCooldownMs?: number, ): string | null { if (kind === "other" || (providerName !== "kiro" && kind !== "rate")) return null; if (!isGenericOAuthFailoverEnabled(config, providerName)) return null; const set = getAccountSet(providerName); // A single stored account has nowhere to go; rotating to itself would just replay the 429. if (!set || set.accounts.length < 2) return null; // `preserveServerDelay` keeps the delay the server actually stated, bounded by the parser's // one-day ceiling, exactly as the combo path does. Truncating it locally only guarantees a // second 429 on an account we were told to leave alone. const parsed = kind === "rate" ? parseRetryAfterMs(retryAfterHeader, now, { preserveImmediate: true, preserveServerDelay: true }) : undefined; // An account whose allowance is provably spent gets a reset-aligned cooldown instead of // the default minute: retrying it every 60s until the window rolls over is pure waste. // A Retry-After from upstream still wins — it is the server's own instruction. const exhausted = parsed === undefined && providerName !== "kiro" ? exhaustedCooldownMs(providerName, failedAccountId, now, set.accounts.find(account => account.id === failedAccountId)) : null; const cooldownMs = kind === "suspended" ? 24 * 60 * 60_000 : kind === "monthly_quota" ? (monthlyCooldownMs ?? ACCOUNT_QUOTA_TTL_MS) : providerName === "kiro" ? (parsed ?? 10_000) : (exhausted ?? parsed ?? DEFAULT_COOLDOWN_MS); const family = classifyModelFamilyForQuota(providerName, requestedModelId); // Suspension is recorded by the caller with the sent generation before the quorum check. // Rewriting it here would lose its login-identity fence. if (kind !== "suspended") health.set(healthKey(providerName, failedAccountId, providerName === "kiro" ? undefined : family), { cooldownUntil: now + cooldownMs, cooldownSource: parsed ? "retry-after" : "default", }); sweepExpiredOnWrite(now); const eligible = eligibleFailoverAccounts(providerName, now, family).filter(id => id !== failedAccountId); if (eligible.length === 0) return null; // A rotation means the roster in use just changed; do not answer the next activation question // from a count read before the failure. presence.delete(providerName); // Deterministic: start after the failed account so repeated 429s walk the roster instead of // hammering whichever id happens to sort first. The ring is built BEFORE ranking — ranking // the store's own order would change which account a quota-less provider rotates to. const order = set.accounts.map(account => account.id); const start = order.indexOf(failedAccountId); const ring = start >= 0 ? [...order.slice(start + 1), ...order.slice(0, start)] : order; let candidates = ring.filter(id => id !== failedAccountId && eligible.includes(id)); if (candidates.length === 0) return null; const cap = providerName === "kiro" ? config.providers?.kiro?.oauthAccountFailover?.maxConcurrentPerAccount : undefined; if (typeof cap === "number" && Number.isInteger(cap) && cap >= 1 && cap <= 100) { const withRoom = candidates.filter(id => accountInFlight("kiro", id) < cap); if (withRoom.length > 0) candidates = withRoom; } candidates = preferKiroModelSupport(providerName, candidates, requestedModelId, config); // The 429 path branches too. Leaving it on the quota ranking would make a configured // strategy inert in practice the moment anything actually failed, which is the case the // operator chose the strategy for. const strategy = activeGenericStrategy(config, providerName); if (strategy === "least-loaded" && isProactivePreferenceEnabled(config, "kiro", now)) { return candidates.sort((a, b) => accountInFlight("kiro", a) - accountInFlight("kiro", b))[0] ?? null; } if (strategy === "round-robin") { // PICK here, not peek: the failure already happened and this answer is the one being used, // so the ring genuinely advances. return pickRoundRobinAccount( genericPoolKey(providerName), candidates, genericStickyLimit(config, providerName), ); } if (strategy === "fill-first") { // Not "keep the active account": the one that just 429'd is cooled, so fill-first takes // the next eligible account in the stable ring rather than its usual hold. const stableAll = stableGenericRoster(providerName); const from = stableAll.indexOf(failedAccountId); const walk = from >= 0 ? [...stableAll.slice(from + 1), ...stableAll.slice(0, from)] : stableAll; for (const id of walk) { if (id !== failedAccountId && candidates.includes(id)) return id; } return null; } // With no quota evidence this returns the ring untouched, so providers without // per-account quota keep exactly the traversal they have today. return rankAccountsByHeadroom(providerName, candidates, requestedModelId, new Map(set.accounts.map(account => [account.id, account])))[0] ?? null; } /** Known dead-account exclusion is a proactive choice with narrow-over-broad precedence. */ export function refusalAwareInitialKiroAccount( config: OcxConfig, activeId: string, now = Date.now(), requestedModelId?: string | null, ): string | null { if (!isProactivePreferenceEnabled(config, "kiro", now)) return null; const set = getAccountSet("kiro"); if (!set || set.accounts.length < 2) return null; const active = set.accounts.find(account => account.id === activeId); if (!active) return null; const eligibleIds = eligibleIdsIn(set, "kiro", now, classifyModelFamilyForQuota("kiro", requestedModelId)); const preferred = preferKiroModelSupport("kiro", eligibleIds, requestedModelId, config); const activeLacks = requestedModelId && kiroAccountSupportsModel(activeId, requestedModelId) === false && preferred.some(id => id !== activeId && kiroModelListingHasRoom(config, id, requestedModelId)); if (!activeLacks && !isCooled("kiro", activeId, now) && kiroAccountEvidence(active, now).exhausted !== true) return null; const eligible = new Set(preferred); const start = set.accounts.findIndex(account => account.id === activeId); for (const account of [...set.accounts.slice(start + 1), ...set.accounts.slice(0, start)]) { if (eligible.has(account.id)) return account.id; } return null; } /** * Full credential snapshot for a rotated account. * * Returns the snapshot rather than a bare bearer: Antigravity pairs an account-matched * `projectId` with its token and Kiro carries routing metadata, so a token-only swap would mix * one account's bearer with another's routing data. */ export async function failoverAccountSnapshot( providerName: string, accountId: string, ): Promise<OAuthAccessSnapshot> { return getValidAccessSnapshotForAccount(providerName, accountId); } /** * Which account should serve the FIRST attempt of a request. * * Rotation only ever ran after a 429, so a turn still opened on whichever account happened * to be active — including one a previous probe already measured as spent. That costs a * full upstream round trip and one of three rotations to rediscover what the cache knew. * * Returns null whenever the ordinary active-account path should be used unchanged: no * quorum, rotation disabled, a single account, or no quota evidence to act on. This is a * preference, never a gate — a cooled or unmeasured account is still perfectly usable, so * an empty answer means "carry on", not "refuse". */ export function preferredInitialAccount( config: OcxConfig, providerName: string, now = Date.now(), requestedModelId?: string | null, ): string | null { // The PROACTIVE predicate, not the reactive one: this steers a request upstream has not // refused, so `oauthAccountFailover.enabled: false` must still be able to refuse it. if (!isProactivePreferenceEnabled(config, providerName, now)) return null; // Read the same authoritative selection the management writer commits. Caching the // active id separately would delay manual selection and account removal. const selected = getAccountSet(providerName); if (!selected) return null; const active = selected.activeAccountId; const accountRows = new Map(selected.accounts.map(account => [account.id, account])); const order = selected.accounts .filter(account => account.paused !== true && account.needsReauth !== true) .map(account => account.id); if (order.length < 2) return null; const modelEligible = preferKiroModelSupport(providerName, eligibleFailoverAccounts(providerName, now, classifyModelFamilyForQuota(providerName, requestedModelId)), requestedModelId, config); const activeLacks = providerName === "kiro" && !!active && !!requestedModelId && kiroAccountSupportsModel(active, requestedModelId) === false; const listingSibling = activeLacks ? modelEligible.find(id => id !== active && kiroModelListingHasRoom(config, id, requestedModelId!)) : undefined; if (listingSibling) return listingSibling; // A configured strategy answers this question itself. Both guards below exist to protect the // QUOTA answer, and both are fatal to the other two: hasHeadroomEvidence refuses every // provider with no quota data, which is exactly where round-robin is the point, and the // healthy-active return fires before autoSwitchThreshold can ever be read, so fill-first // would never reach its own test. Cooldowns and reauth are still honoured inside each pick. const strategy = activeGenericStrategy(config, providerName); if (strategy === "least-loaded" && providerName === "kiro") { const family = classifyModelFamilyForQuota(providerName, requestedModelId); let candidates = eligibleFailoverAccounts(providerName, now, family); const cap = config.providers?.kiro?.oauthAccountFailover?.maxConcurrentPerAccount; if (typeof cap === "number" && Number.isInteger(cap) && cap >= 1 && cap <= 100) { candidates = candidates.filter(id => accountInFlight("kiro", id) < cap); } candidates = preferKiroModelSupport(providerName, candidates, requestedModelId, config); const picked = candidates.sort((a, b) => accountInFlight("kiro", a) - accountInFlight("kiro", b))[0]; return picked && picked !== active ? picked : null; } if (strategy === "round-robin") { const family = classifyModelFamilyForQuota(providerName, requestedModelId); const eligibleNow = preferKiroModelSupport(providerName, eligibleFailoverAccounts(providerName, now, family), requestedModelId, config); if (eligibleNow.length === 0) return null; // PEEK, not pick: this proposal is discardable, and advancing the ring for an account the // resolver then rejects would skip a turn for nothing. noteGenericPoolSelection commits. const picked = peekRoundRobinAccount( genericPoolKey(providerName), eligibleNow, genericStickyLimit(config, providerName), ); return picked && picked !== active ? picked : null; } if (strategy === "fill-first") { const picked = pickFillFirstGenericAccount(config, providerName, active, now, requestedModelId, accountRows); return picked && picked !== active ? picked : null; } const activeRow = selected.accounts.find(account => account.id === active); if (activeRow && activeRow.paused !== true && activeRow.needsReauth !== true && !isCooled(providerName, activeRow.id, now, classifyModelFamilyForQuota(providerName, requestedModelId)) && !isAccountQuotaExhausted(providerName, activeRow.id, requestedModelId, activeRow)) return null; // Evidence is required BEFORE eligibility narrows the field. Without this, a provider // with no quota data at all could still be redirected: cool the active account with a // 429 and the eligible list collapses to one candidate, which any ranking returns // unchanged — an answer that looks ranked but was never measured. The no-op guarantee // for quota-less providers has to be checked on the full roster. if (!hasHeadroomEvidence(providerName, order, requestedModelId, accountRows)) return null; // Cooldowns are respected here, unlike in the presence count: this picks the account to // send to right now, and one inside its 429 window is the single candidate we hold // positive evidence against. const eligible = preferKiroModelSupport(providerName, order.filter(id => !isCooled(providerName, id, now, classifyModelFamilyForQuota(providerName, requestedModelId))), requestedModelId, config); if (eligible.length === 0) return null; // Start the ring at the active account so an unranked outcome reproduces today's choice. const start = active ? order.indexOf(active) : -1; const ring = start >= 0 ? [...order.slice(start), ...order.slice(0, start)] : order; const candidates = ring.filter(id => eligible.includes(id)); if (candidates.length === 0) return null; const best = rankAccountsByHeadroom(providerName, candidates, requestedModelId, accountRows)[0] ?? null; // Nothing to do when the ranking agrees with the account we would have used anyway. // // A proposal still needs guarded selection commit after credential resolution: a // removal, reauth verdict, or manual choice can arrive during that await. return best && best !== active ? best : null; } /** Earliest remaining cooldown, for a client-facing Retry-After when every account is cooled. */ export function genericFailoverRetryAfterSeconds(providerName: string, now = Date.now()): number | null { const prefix = `${providerName}\u0000`; let earliest: number | null = null; for (const [key, entry] of health) { if (!key.startsWith(prefix) || entry.cooldownUntil <= now) continue; if (earliest === null || entry.cooldownUntil < earliest) earliest = entry.cooldownUntil; } return earliest === null ? null : Math.max(1, Math.ceil((earliest - now) / 1000)); } /** Test seam and manual-recovery hook. */ export function forgetGenericFailoverRoster(providerName: string): void { presence.delete(providerName); } /** Test seam and manual-recovery hook. */ export function clearGenericFailoverHealth(providerName?: string): void { if (!providerName) { health.clear(); presence.clear(); return; } presence.delete(providerName); for (const key of [...health.keys()]) { if (key.startsWith(`${providerName}\u0000`)) health.delete(key); } }