UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

909 lines (854 loc) • 40.7 kB
/** * Multi-key 429 failover for non-OpenAI providers. * * When a provider's upstream returns 429, this module picks the next available key * from `apiKeyPool`, puts the exhausted key into cooldown (respecting Retry-After), * and returns a fresh provider config with the swapped key. If all keys are in * cooldown, returns null so the caller surfaces the 429 to the client. * * Modelled after src/codex/routing.ts cooldown logic but scoped to plain API-key pools. */ import { commitProviderApiKeySelection } from "./api-key-selection"; import type { ProviderApiKeySelection } from "../types/provider"; import { routedProviderConfig } from "../router"; import { getProviderRegistryEntry } from "./registry"; import { isCanonicalCommandCodeBaseUrl, normalizedBaseUrl } from "./quota/vendor-probes-key"; import type { OcxConfig, OcxProviderConfig, RateLimitRetryPolicy, ResetReplayPolicy, TransientRetryPolicy } from "../types"; import { OPENCODE_GO_SESSION_HEADER } from "./opencode-go-transport"; import { resolveProviderTransport, type OcxProviderTransport } from "./xai-transport"; import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; // quota-key-accounts imports only node:crypto, the key store and the quota types -- NOT // providers/quota.ts -- so the cached reader reaches the dispatch path without dragging the // probe machinery onto it. import { cachedApiKeyQuota } from "./quota-key-accounts"; // ---- cooldown state (in-memory, same as codex/routing.ts) ---- interface KeyCooldown { cooldownUntil: number; } const DEFAULT_COOLDOWN_MS = 60_000; const MAX_COOLDOWN_MS = 10 * 60_000; // cap at 10 min for api-key rotation /** * Cap for a cooldown the upstream itself dated, as opposed to one we inferred. * * `MAX_COOLDOWN_MS` is deliberately short because an undated 429 is a guess: ten * minutes bounds how long a transient limit can park a working key. A free-tier * quota is not a guess — OpenRouter replies `Weekly/Monthly Limit Exhausted ... * will reset at <date>`, and until that date the key cannot serve anything. Held * for ten minutes instead, it comes back, takes a 429, and rotates again, every * ten minutes for the rest of the week (#4024). * * 32 days rather than unbounded. The wording this parses is * `Weekly/Monthly Limit Exhausted`, so the cap has to clear a monthly window — * 31 days plus a day of slack for timezone and month length. An earlier 8-day * cap looked generous against the weekly case in the issue and silently clamped * every monthly reset to ~23 days early, which puts the key back into exactly * the 429 loop this exists to stop. Caught by the cap's own test. * * Bounded at all because the date is upstream-controlled input: a malformed or * hostile `reset at 2999-01-01` must not park a working key past any horizon an * operator would think to look at. */ const MAX_QUOTA_COOLDOWN_MS = 32 * 24 * 60 * 60_000; const QUOTA_RESET_PEEK_TIMEOUT_MS = 250; interface QuotaResetReadOptions { now?: number; signal?: AbortSignal; timeoutMs?: number; } function rebuiltResponse(response: Response, body: ReadableStream<Uint8Array>): Response { return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers, }); } function replayOnlyResponse(response: Response, chunks: readonly Uint8Array[]): Response { return rebuiltResponse(response, new ReadableStream<Uint8Array>({ start(controller) { for (const chunk of chunks) controller.enqueue(chunk); controller.close(); }, })); } /** * Read a bounded prefix of a 429 body and pull the upstream's declared reset instant. * * The returned response replays the bounded prefix and any boundary-chunk overflow * before streaming the unread remainder. A rotation storm must not be gated on * reading N full error payloads. Any failure — no body, already consumed, slow, * malformed — returns undefined and leaves the `Retry-After` path in charge. */ export async function readQuotaResetAt( response: Response, nowOrOptions: number | QuotaResetReadOptions = {}, ): Promise<{ at: number | undefined; response: Response }> { if (!response.body) return { at: undefined, response }; const options = typeof nowOrOptions === "number" ? { now: nowOrOptions } : nowOrOptions; const now = options.now ?? Date.now(); let reader: ReadableStreamDefaultReader<Uint8Array>; try { reader = response.body.getReader(); } catch { return { at: undefined, response }; } const chunks: Uint8Array[] = []; let transferred = false; let timer: ReturnType<typeof setTimeout> | undefined; const deadline = new AbortController(); const timeoutReason = new DOMException("Quota reset body peek timed out", "TimeoutError"); try { const decoder = new TextDecoder(); let seen = 0; let text = ""; timer = setTimeout(() => deadline.abort(timeoutReason), options.timeoutMs ?? QUOTA_RESET_PEEK_TIMEOUT_MS); const signal = options.signal ? AbortSignal.any([options.signal, deadline.signal]) : deadline.signal; while (seen < QUOTA_RESET_SCAN_BYTES) { const read = reader.read(); let rejectAbort: ((reason: unknown) => void) | undefined; const onAbort = () => rejectAbort?.(signal.reason); const aborted = new Promise<never>((_resolve, reject) => { rejectAbort = reject; if (signal.aborted) reject(signal.reason); else signal.addEventListener("abort", onAbort, { once: true }); }); // Derived from the reader rather than named directly: Bun's lib types // `ReadableStreamDefaultReader.read()` as returning // `ReadableStreamDefaultReadResult`, which is not assignable to the // `ReadableStreamReadResult` alias. let result: Awaited<ReturnType<typeof reader.read>>; try { result = await Promise.race([read, aborted]); } finally { signal.removeEventListener("abort", onAbort); } const { done, value } = result; if (done) break; const remaining = QUOTA_RESET_SCAN_BYTES - seen; const prefix = value.byteLength > remaining ? value.subarray(0, remaining) : value; const overflow = value.byteLength > remaining ? value.subarray(remaining) : undefined; chunks.push(prefix); if (overflow?.byteLength) chunks.push(overflow); seen += prefix.byteLength; text += decoder.decode(prefix, { stream: true }); } // Hand back a Response carrying the bytes already pulled followed by whatever // is left, so the caller can still read or cancel it. `response.clone()` is // NOT usable here: it tees, and with the original branch undrained the tee // stalls once its buffer fills — a 5MB error body hangs the rotation path, // which is worse than the unbounded read this replaced. const rest = new ReadableStream<Uint8Array>({ start(controller) { for (const c of chunks) controller.enqueue(c); }, async pull(controller) { try { const { done, value } = await reader.read(); if (done) { controller.close(); reader.releaseLock(); return; } controller.enqueue(value); } catch (error) { controller.error(error); reader.releaseLock(); } }, async cancel(reason) { try { await reader.cancel(reason); } finally { reader.releaseLock(); } }, }); transferred = true; return { at: parseQuotaResetAt(text, now), response: rebuiltResponse(response, rest) }; } catch (error) { const clientAborted = options.signal?.aborted === true; void reader.cancel(error).catch(() => {}).finally(() => { try { reader.releaseLock(); } catch { /* already released */ } }); if (clientAborted) throw options.signal!.reason ?? error; return { at: undefined, response: replayOnlyResponse(response, chunks) }; } finally { if (timer !== undefined) clearTimeout(timer); if (!transferred) { try { reader.releaseLock(); } catch { /* already released */ } } } } /** * How much of a 429 body is read and scanned for the reset instant. * * Bounds the READ, not just the parse: this runs on the rotation path, once per * rotated key under a rate-limit storm, and the body is upstream-controlled. * OpenRouter's rate_limit_error JSON is a few hundred bytes. */ const QUOTA_RESET_SCAN_BYTES = 4_096; /** * Reset instant an upstream declared in a 429 *body*, in epoch ms. * * Only the body carries this: OpenRouter sends no `Retry-After` for a quota * exhaustion, so the header path (`parseRetryAfterMs`) sees nothing and falls * back to `DEFAULT_COOLDOWN_MS`. Returns undefined for anything it cannot read * as a date, so an unparsable body keeps today's behaviour exactly. */ /** * Whether `YYYY-MM-DD…` names a day that exists. * * `Date.parse` does NOT reject an out-of-range day: measured on Bun, * `2026-02-30T00:00:00Z` yields March 2 and `2026-04-31T00:00:00Z` yields * May 1, so a malformed upstream body would park a key past the instant it * actually named. Only the month is rejected outright (`2026-13-01` is NaN). * * Checked on the date text alone rather than by round-tripping the parsed * instant, because a value carrying an explicit offset (`…T23:00+05:30`) * legitimately lands on a different UTC day than the one written. */ function isRealCalendarDate(value: string): boolean { const [year, month, day] = value.slice(0, 10).split("-").map(Number); if (month < 1 || month > 12 || day < 1) return false; const leap = (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0; const lengths = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; return day <= lengths[month - 1]!; } export function parseQuotaResetAt(body: string | null | undefined, now = Date.now()): number | undefined { const text = body?.slice(0, QUOTA_RESET_SCAN_BYTES); if (!text) return undefined; let parsed: unknown; try { parsed = JSON.parse(text); } catch { return undefined; } if (!parsed || typeof parsed !== "object") return undefined; const error = (parsed as { error?: unknown }).error; if (!error || typeof error !== "object") return undefined; const code = (error as { code?: unknown }).code; const message = (error as { message?: unknown }).message; if ((code !== "rate_limit_error" && code !== 429) || typeof message !== "string") return undefined; if (!/^(?:Weekly|Monthly) Limit Exhausted\b/i.test(message.trim())) return undefined; // `will reset at 2026-09-09 03:30:06` / `... at 2026-09-09T03:30:06Z` / `resets at <date>` const match = /reset[s]?\s+at\s+([0-9]{4}-[0-9]{2}-[0-9]{2}(?:[T ][0-9]{2}:[0-9]{2}(?::[0-9]{2})?(?:\.[0-9]+)?(?:Z|[+-][0-9]{2}:?[0-9]{2})?)?)/i.exec(message); if (!match) return undefined; // Pin a bare `YYYY-MM-DD hh:mm:ss` to UTC explicitly. // // ECMA-262 says a date-TIME form with no offset is LOCAL time, and Node follows // that: `Date.parse("2026-09-09 03:30:06")` differs from the UTC reading by the // host offset (7h on a PDT box, measured). Bun currently returns the UTC value // for the same string, so on this runtime the normalisation is a no-op today — // which is exactly why it is written out rather than relied upon. If Bun ever // conforms, an un-normalised parse would silently shift every park-until by the // operator's offset, and the early direction resumes the 429 loop. // // A consequence worth knowing: no Bun test can observe this branch being // removed. The explicit-zone case below is the part the suite can pin. const raw = match[1].includes("T") || /(?:Z|[+-][0-9]{2}:?[0-9]{2})$/.test(match[1]) ? match[1] : `${match[1].replace(" ", "T")}Z`; if (!isRealCalendarDate(match[1])) return undefined; const at = Date.parse(raw); if (!Number.isFinite(at)) return undefined; // Already past, or beyond the cap: not usable as a park-until instant. if (at <= now) return undefined; return Math.min(at, now + MAX_QUOTA_COOLDOWN_MS); } /** * Default same-target 429 retry policy used when a provider opts in via a bare * `retryOn429: {}` (presence = opt-in with these defaults). */ const DEFAULT_RATE_LIMIT_RETRY = { enabled: true, attempts: 3, intervalMs: 5_000, maxIntervalMs: 60_000, respectRetryAfter: true, } as const satisfies Required<RateLimitRetryPolicy>; /** * Patient same-target 429 fallback for the OpenCode Go destination * (https://opencode.ai/zen/go/v1), which serves subscription traffic such as Muse Spark. * Single-key pools cannot fail over, so without this a burst 429 surfaces immediately and * the client retry budget aborts the goal. Six 10s-paced attempts absorb a short burst * window (effective replays are additionally bounded by the shared per-request send * budget); Retry-After is still honored and capped. An explicit `retryOn429` (including * `enabled: false`) always overrides this fallback. */ const OPENCODE_GO_RATE_LIMIT_RETRY = { enabled: true, attempts: 6, intervalMs: 10_000, maxIntervalMs: 60_000, respectRetryAfter: true, } as const satisfies Required<RateLimitRetryPolicy>; /** True when the provider row points at the OpenCode Go destination. */ export function isOpenCodeGoDestination( provider: Partial<Pick<OcxProviderConfig, "baseUrl" | "authMode">>, ): boolean { const raw = typeof provider.baseUrl === "string" ? provider.baseUrl : ""; if (!raw.trim()) return false; // Endpoint identity, not adapter identity: the runtime adapter is already overridden // per model by the time the recovery loop runs (muse-spark rides `openai-responses` // while the preset declares `openai-chat`), so an adapter-strict lookup misses it. // Canonicalize with the shared quota-probe normalizer so host case and explicit // default ports compare equal; userinfo, query, and fragment never match // (follow-up to the review on #5067). const endpoint = normalizedBaseUrl(raw.trim()); if (!endpoint) return false; const entry = getProviderRegistryEntry("opencode-go"); if (!entry) return false; const candidates = [entry.baseUrl, ...(entry.destinationAliases ?? []).map(alias => alias.baseUrl)]; return candidates.some(url => normalizedBaseUrl(url.trim()) === endpoint); } /** * Default transient-5xx retry used when a provider opts in with a bare * `transientRetryOn5xx: {}`. `attempts` is a TOTAL send budget, not extra retries. */ const DEFAULT_TRANSIENT_RETRY = { enabled: true, attempts: 3, } as const satisfies Required<TransientRetryPolicy>; /** * Default used when a provider opts in with a bare `retryOnReset: {}`: one replacement for * the whole logical request. `replacements` counts duplicate inferences the operator accepts, * not retries and not sends. */ const DEFAULT_RESET_REPLAY = { enabled: true, replacements: 1, } as const satisfies Required<ResetReplayPolicy>; /** Map<`${providerName}\0${keyId}`, KeyCooldown> */ const keyCooldowns = new Map<string, KeyCooldown>(); function cooldownKey(providerName: string, keyId: string): string { return `${providerName}\0${keyId}`; } /** * Parse an upstream `Retry-After` header: numeric seconds (including `0`) or an HTTP-date. * Returns a bounded delay in ms (1..MAX_COOLDOWN_MS), or undefined when the value is * malformed. An HTTP-date already in the past yields an immediate (1 ms) retry. */ function parseRetryAfterMs(value: string | null | undefined, now = Date.now()): number | undefined { const text = value?.trim(); if (!text) return undefined; if (/^\d+(?:\.\d+)?$/.test(text)) { const seconds = Number(text); if (Number.isFinite(seconds) && seconds >= 0) { return Math.min(Math.max(Math.ceil(seconds * 1000), 1), MAX_COOLDOWN_MS); } } const timestamp = Date.parse(text); if (!Number.isFinite(timestamp)) return undefined; const delay = timestamp - now; // A valid HTTP-date whose retry time has already passed is an immediate retry, exactly like // numeric `Retry-After: 0` — never a malformed-header fallback to the fixed interval. return Math.min(Math.max(delay, 1), MAX_COOLDOWN_MS); } /** * True while the given key is inside its 429 cooldown window (lazily evicting the entry once the * window expires). Used to skip keys that the upstream just rate-limited during failover. */ function isKeyInCooldown(providerName: string, keyId: string, now = Date.now()): boolean { const entry = keyCooldowns.get(cooldownKey(providerName, keyId)); if (!entry) return false; if (entry.cooldownUntil <= now) { keyCooldowns.delete(cooldownKey(providerName, keyId)); return false; } return true; } // ---- public API ---- /** * Check whether a provider has multiple keys available for failover. * Returns true only for key-auth providers with 2+ pool entries. */ export function hasKeyPoolFailover(provider: OcxProviderConfig): boolean { if (provider.authMode === "oauth" || provider.authMode === "forward") return false; return (provider.apiKeyPool?.length ?? 0) >= 2; } /** * Process-local round-robin cursor per provider, deliberately parallel to `keyCooldowns` * rather than borrowing the Codex pool-rotation state: an API key is not an OAuth account * and must not share a quota scope key. Multi-process desync is the same accepted limit * the cooldown map already carries. */ const keyRotationCursor = new Map<string, string>(); /** * Forget a provider's cursor so an operator's manual key selection is not second-guessed. * * Optional name, mirroring `clearKeyCooldowns`, because the batch provider PUT rewrites the * entire roster: a cursor that survives a reorder still names a real id, so round-robin * resumes after the pre-edit position and can skip the first eligible key in the new pool. */ export function forgetApiKeyRotationCursor(providerName?: string): void { if (!providerName) { keyRotationCursor.clear(); return; } keyRotationCursor.delete(providerName); } /** The pool entry shape is inline on OcxProviderConfig; name it once rather than re-spelling it. */ type ApiKeyPoolEntry = NonNullable<OcxProviderConfig["apiKeyPool"]>[number]; /** * Remaining headroom for one key, or null when nothing current measures it. * * Same definition as `headroomOf` on the OAuth side, so the two pools cannot disagree about * what "more room" means. `creditsUsd` is deliberately excluded: it is a currency amount, not * a percentage, and ranking one against the other produces an order that means nothing. */ function keyHeadroom(providerName: string, provider: OcxProviderConfig, entry: ApiKeyPoolEntry): number | null { const quota = cachedApiKeyQuota(providerName, provider, entry.id, entry.key); if (!quota) return null; const percents = [ quota.fiveHourPercent, quota.weeklyPercent, quota.monthlyPercent, ...(quota.customWindows ?? []).map((window: { percent?: number }) => window.percent), ].filter((value): value is number => typeof value === "number"); if (percents.length === 0) return null; return 100 - Math.max(...percents); } /** * Order eligible keys best-first, in the same three buckets `rankAccountsByHeadroom` uses: * measured-with-headroom, then unmeasured, then measured-and-spent. Ties keep the roster order. * * An unmeasured key is NOT assumed spent, and not assumed fresh either -- it sits between the * two, which is the only honest position for a key nothing has looked at. A provider that * publishes no per-key differentiation (DeepSeek reports every key at the same percent) ties * across the board and falls through to the roster order, which is exactly today's behaviour. */ function rankKeysByHeadroom( providerName: string, provider: OcxProviderConfig, eligible: readonly ApiKeyPoolEntry[], ): ApiKeyPoolEntry[] { return eligible .map((entry, index) => { const headroom = keyHeadroom(providerName, provider, entry); const bucket = headroom === null ? 1 : headroom <= 0 ? 2 : 0; return { entry, bucket, headroom: headroom ?? 0, index }; }) .sort((left, right) => (left.bucket - right.bucket) || (right.headroom - left.headroom) || (left.index - right.index)) .map(row => row.entry); } /** * Pick a better key BEFORE the first attempt when the committed one is already cooling. * * This is intentionally narrow. It never overrides a healthy key: if the committed * `apiKey` is not in cooldown it returns null, so an operator's manual selection stands * and no config write happens. It only acts when the committed key is known-cooled (or * missing from the pool), which is exactly the case where the first request would * otherwise be spent earning a 429 the runtime could already predict. * * Returning null is the common path, so the persisted-selection transaction is not on * the per-request hot path. * * Like `rotateKeyAfterFailure`, the returned object is a snapshot of the PERSISTED config * and carries none of the registry backfills `routedProviderConfig` merges in at request * time. A request path must not assign it to an active route wholesale -- for a built-in * provider stored in its valid minimal form that would drop the adapter id, the base URL and * the static headers, so `resolveAdapter()` throws `Unknown adapter: undefined` and a * hand-built URL dereferences a missing `baseUrl`. Use * `selectProactiveApiKeyTransport`, the pre-dispatch twin of `rotateProviderTransportOn429`. */ export function selectProactiveApiKey( config: OcxConfig, providerName: string, now = Date.now(), ): OcxProviderConfig | null { const provider = config.providers?.[providerName]; if (!provider) return null; const strategy = provider.apiKeyPoolStrategy; if (!strategy) return null; if (!hasKeyPoolFailover(provider)) return null; const pool = provider.apiKeyPool ?? []; const activeEntry = pool.find(entry => entry.key === provider.apiKey); // A healthy committed key wins, whether the operator chose it or a previous rotation did. if (activeEntry && !isKeyInCooldown(providerName, activeEntry.id, now)) return null; const eligible = pool.filter(entry => !isKeyInCooldown(providerName, entry.id, now)); if (eligible.length === 0) return null; let chosen = eligible[0]!; if (strategy === "round-robin") { const lastId = keyRotationCursor.get(providerName); const lastIndex = lastId ? pool.findIndex(entry => entry.id === lastId) : -1; for (let offset = 1; offset <= pool.length; offset += 1) { const candidate = pool[(lastIndex + offset) % pool.length]!; if (isKeyInCooldown(providerName, candidate.id, now)) continue; chosen = candidate; break; } } else if (strategy === "quota") { // else-if, deliberately. `fill-first` is not a named branch here -- it is the eligible[0] // default above, so replacing that default would silently retarget it. chosen = rankKeysByHeadroom(providerName, provider, eligible)[0] ?? chosen; } if (chosen.key === provider.apiKey) return null; const outcome = commitProviderApiKeySelection<string | null>(config, providerName, freshProvider => { const freshPool = freshProvider.apiKeyPool ?? []; const target = freshPool.find(entry => entry.id === chosen.id); if (!target) return { changed: false, value: null }; if (freshProvider.apiKey === target.key) return { changed: false, value: null }; const freshActive = freshPool.find(entry => entry.key === freshProvider.apiKey); // Re-check under the lock: a concurrent manual selection may have landed a healthy key. if (freshActive && !isKeyInCooldown(providerName, freshActive.id, now)) { return { changed: false, value: null }; } freshProvider.apiKey = target.key; return { changed: true, value: target.id }; }); if (outcome.status !== "committed" || outcome.value === null) return null; keyRotationCursor.set(providerName, outcome.value); const committed = structuredClone(outcome.provider); config.providers[providerName] = committed; return structuredClone(committed); } /** * Pre-dispatch twin of `rotateProviderTransportOn429`: pick a warm key, then rebuild the * active route from the committed row through the same seam the 429 path uses, so the * registry backfills survive and only explicit runtime transport state (`fetch` and a * generated OpenCode session header) is carried over from the route being replaced. * * Every request path that assigns the result to a live route must call THIS, not * `selectProactiveApiKey`, which answers with a persisted snapshot. */ export function selectProactiveApiKeyTransport( config: OcxConfig, providerName: string, routedProvider: OcxProviderTransport, promptCacheKey?: string, now = Date.now(), ): OcxProviderTransport | null { const committed = selectProactiveApiKey(config, providerName, now); if (!committed) return null; return applyRotatedTransport(providerName, routedProvider, committed, promptCacheKey); } /** * Normalize a provider's `retryOn429` policy. An explicit object always wins (including * `enabled: false` to opt out). When the knob is absent, the OpenCode Go destination * (subscription traffic such as Muse Spark) falls back to a patient same-key policy so a * burst 429 waits and replays instead of surfacing to the client and aborting a long * session; key-auth Command Code at its canonical endpoints gets the same fallback (#5180), and * every other provider without the knob keeps today's fail-fast behavior. * OAuth/forward/local credentials are never replayed on the same token. The returned * policy is fully defaulted so callers never re-check fields. */ export function rateLimitRetryPolicyFor( provider: Pick<OcxProviderConfig, "retryOn429" | "authMode"> & Partial<Pick<OcxProviderConfig, "baseUrl" | "adapter">>, ): Required<RateLimitRetryPolicy> | null { const policy = provider.retryOn429; if (policy) { if (policy.enabled === false) return null; // Fail closed: only explicit key auth or the documented omitted-default (undefined == key for // custom API-key providers) may use same-key replays. OAuth/forward are never replayed on the // same token, local runtimes have no remote key to preserve, and unknown/custom values are // rejected rather than guessed at. if (provider.authMode !== undefined && provider.authMode !== "key") return null; return { enabled: policy.enabled ?? DEFAULT_RATE_LIMIT_RETRY.enabled, attempts: policy.attempts ?? DEFAULT_RATE_LIMIT_RETRY.attempts, intervalMs: policy.intervalMs ?? DEFAULT_RATE_LIMIT_RETRY.intervalMs, maxIntervalMs: policy.maxIntervalMs ?? DEFAULT_RATE_LIMIT_RETRY.maxIntervalMs, respectRetryAfter: policy.respectRetryAfter ?? DEFAULT_RATE_LIMIT_RETRY.respectRetryAfter, }; } // No explicit knob: patient fallback for OpenCode Go and canonical Command Code only. if (provider.authMode !== undefined && provider.authMode !== "key") return null; if (isOpenCodeGoDestination(provider)) return { ...OPENCODE_GO_RATE_LIMIT_RETRY }; // Command Code's Provider API rate-limits long muse-spark turns the same way (#5180): a single // key cannot fail over, and the Codex client does not retry a 429, so the turn aborts. The key // endpoint gets the same patient same-key policy. A row repointed at a custom relay is not the // canonical endpoint and keeps fail-fast unless it sets `retryOn429`. if (typeof provider.baseUrl === "string" && isCanonicalCommandCodeBaseUrl(provider.baseUrl.trim())) { return { ...OPENCODE_GO_RATE_LIMIT_RETRY }; } return null; } /** * Normalize a provider's `transientRetryOn5xx` policy, or return null when it is absent, * explicitly disabled, not key-auth, or not an adapter this policy governs. * * The adapter gate is part of the accepted scope, not incidental: it names the adapters whose * lanes actually read this policy, so no generic key-auth adapter can opt in by accident. * Auth mode follows the same fail-closed rule as `rateLimitRetryPolicyFor` — * explicit `key` or the documented omitted default, never OAuth, forward, local, or an * unknown value. */ export function transientRetryPolicyFor( provider: Pick<OcxProviderConfig, "transientRetryOn5xx" | "authMode" | "adapter">, ): Required<TransientRetryPolicy> | null { const policy = provider.transientRetryOn5xx; if (!policy || policy.enabled === false) return null; // Both adapters this policy governs. The first version covered chat only, which left a // key-auth Responses provider unable to tune its ladder in either direction, because the // Responses passthrough lane hard-coded TRANSIENT_RETRY_MAX_ATTEMPTS (#4893). Widening this // gate is necessary and not sufficient: the lane also has to call this function, which it // now does. Still an explicit list, so no generic key-auth adapter opts in by accident, and // the auth check below keeps the ChatGPT forward pool out -- those providers are // `authMode: "forward"` and keep the default ladder they have always had. if (provider.adapter !== "openai-chat" && provider.adapter !== "openai-responses") return null; if (provider.authMode !== undefined && provider.authMode !== "key") return null; return { enabled: policy.enabled ?? DEFAULT_TRANSIENT_RETRY.enabled, attempts: policy.attempts ?? DEFAULT_TRANSIENT_RETRY.attempts, }; } /** * Normalize a provider's `retryOnReset` policy, or return null when it is absent or explicitly * disabled. * * No auth-mode gate: the canonical ChatGPT backend is `forward` auth and is the send this * policy exists for. Whether a given REQUEST may be replaced is a per-body decision made in * src/server/responses/reset-replay.ts, and how many replacements the request gets is the * shared allowance on its execution budget. This function only reads the operator's intent. */ export function resetReplayPolicyFor( provider: Pick<OcxProviderConfig, "retryOnReset">, ): Required<ResetReplayPolicy> | null { const policy = provider.retryOnReset; if (!policy || policy.enabled === false) return null; return { enabled: policy.enabled ?? DEFAULT_RESET_REPLAY.enabled, replacements: policy.replacements ?? DEFAULT_RESET_REPLAY.replacements, }; } /** * Wait before the next same-target replay: upstream Retry-After (seconds or HTTP-date) when * `respectRetryAfter` is on and the header parses, capped at `maxIntervalMs`; otherwise the * fixed `intervalMs`, also capped at `maxIntervalMs` (a single wait never exceeds the cap). * Malformed headers fall back to the fixed interval. */ export function rateLimitRetryDelayMs( policy: Required<RateLimitRetryPolicy>, retryAfterHeader: string | null | undefined, now = Date.now(), ): number { const raw = retryAfterHeader?.trim(); if (policy.respectRetryAfter && raw) { const parsed = parseRetryAfterMs(raw, now); if (parsed !== undefined) return Math.min(parsed, policy.maxIntervalMs); } return Math.min(policy.intervalMs, policy.maxIntervalMs); } /** * Record a 429 for the current key and attempt to switch to the next available one. * * @returns A new OcxProviderConfig with the swapped key (and mutated config on disk), * or `null` when no alternative key is available (all in cooldown or pool < 2). * * The returned object is a snapshot of the PERSISTED config — it carries none of the * registry backfills `routedProviderConfig` merges in at request time. Request paths must * not assign it to an active route wholesale; use `rotateProviderTransportOn429`, which * rebuilds from this committed row, reapplies registry metadata, and retains only explicit * runtime transport state (`fetch` and generated OpenCode session affinity). */ function rotateKeyAfterFailure( config: OcxConfig, providerName: string, failureStatus: 401 | 429, retryAfterHeader: string | null | undefined, now = Date.now(), attemptedKey?: string, attemptedSelection?: ProviderApiKeySelection, quotaResetAt?: number, ): OcxProviderConfig | null { const provider = config.providers[providerName]; if (!provider) return null; if (provider.authMode === "oauth" || provider.authMode === "forward") return null; const failedKey = attemptedSelection?.reference ?? attemptedKey ?? provider.apiKey; type Rotation = | { failedId?: string; candidateId?: string } | { exhaustedCount: number; failedId?: string }; const outcome = commitProviderApiKeySelection<Rotation | null>(config, providerName, freshProvider => { const pool = freshProvider.apiKeyPool; if (!pool || pool.length < 2) return { changed: false, value: null }; // The callback can be rerun after rebasing, so identify the failed key here but // defer the in-memory cooldown side effect until persistence has succeeded. const failedEntry = attemptedSelection?.entryId ? pool.find(entry => entry.id === attemptedSelection.entryId && entry.key === failedKey) : pool.find(entry => entry.key === failedKey); if (freshProvider.apiKey !== failedKey) { const activeEntry = pool.find(entry => entry.key === freshProvider.apiKey); if (activeEntry && !isKeyInCooldown(providerName, activeEntry.id, now)) { return { changed: false, value: { failedId: failedEntry?.id }, }; } } const currentIndex = failedEntry ? pool.indexOf(failedEntry) : -1; const candidateCount = failedEntry ? pool.length - 1 : pool.length; for (let offset = 1; offset <= candidateCount; offset += 1) { const candidate = pool[(currentIndex + offset) % pool.length]!; if (isKeyInCooldown(providerName, candidate.id, now)) continue; freshProvider.apiKey = candidate.key; return { changed: true, value: { failedId: failedEntry?.id, candidateId: candidate.id, }, }; } return { changed: false, value: { exhaustedCount: pool.length, failedId: failedEntry?.id } }; }, attemptedSelection); if (outcome.status === "unavailable") return null; if (outcome.status === "superseded") { // A newer manual selection (including A→B→A) owns subsequent dispatch. Reusing the // same failed key here would loop forever; preserve its original failure instead. return outcome.provider.apiKey !== failedKey ? structuredClone(outcome.provider) : null; } if (outcome.value === null) return null; if (outcome.value.failedId) { // A 401 is a verdict about the credential itself, not a timing signal: the key is rejected // until an operator replaces it, and upstreams send no Retry-After for it. Hold it for the // full cap instead of the 429 default so a dead key is not re-tried once a minute. const cooldownMs = failureStatus === 401 ? MAX_COOLDOWN_MS // A reset instant the upstream dated outranks both the header and the // default: it is the only one of the three that knows when the quota // actually returns (#4024). : quotaResetAt !== undefined ? Math.max(quotaResetAt - now, 1) : parseRetryAfterMs(retryAfterHeader, now) ?? DEFAULT_COOLDOWN_MS; keyCooldowns.set(cooldownKey(providerName, outcome.value.failedId), { cooldownUntil: now + cooldownMs }); sweepExpiredOnWrite(now); } if ("exhaustedCount" in outcome.value) { console.warn(`[key-failover] ${providerName}: all ${outcome.value.exhaustedCount} keys in cooldown after ${failureStatus}; returning the upstream status to the client`); return null; } const committed = structuredClone(outcome.provider); config.providers[providerName] = committed; if (outcome.value.candidateId) { console.warn( // Log ids only — labels are user-supplied free text and could carry secret material. `[key-failover] ${providerName}: ${failureStatus} on key ${outcome.value.failedId ?? "?"}; rotating to key ${outcome.value.candidateId}`, ); } return structuredClone(committed); } export function rotateKeyOn429( config: OcxConfig, providerName: string, retryAfterHeader: string | null | undefined, now = Date.now(), attemptedKey?: string, attemptedSelection?: ProviderApiKeySelection, quotaResetAt?: number, ): OcxProviderConfig | null { return rotateKeyAfterFailure(config, providerName, 429, retryAfterHeader, now, attemptedKey, attemptedSelection, quotaResetAt); } /** * Record a 401 for the current key and attempt to switch to the next available one. * * A static key pool can recover a credential-scoped 401 without abandoning the provider: one * revoked or mistyped key in a pool of several says nothing about its siblings. OAuth and * forward providers never reach here — they refresh or re-authenticate instead, and * `rotateKeyAfterFailure` rejects both auth modes outright. */ export function rotateKeyOn401( config: OcxConfig, providerName: string, now = Date.now(), attemptedKey?: string, attemptedSelection?: ProviderApiKeySelection, ): OcxProviderConfig | null { return rotateKeyAfterFailure(config, providerName, 401, null, now, attemptedKey, attemptedSelection); } export function sweepExpiredApiKeyCooldowns(now = Date.now()): number { let removed = 0; for (const [key, cooldown] of keyCooldowns) { if (cooldown.cooldownUntil > now) continue; keyCooldowns.delete(key); removed += 1; } return removed; } interface RotateProviderTransportOptions { retryAfter?: string | null; /** Epoch ms from `parseQuotaResetAt`, when the upstream dated the reset in its body. */ quotaResetAt?: number; now?: number; attemptedKey?: string; attemptedSelection?: ProviderApiKeySelection; promptCacheKey?: string; } /** * Rotate a failed key and re-apply provider-specific transport metadata to the replacement. * * Route the authoritative committed row again so concurrent provider edits take effect, then * restore only transport-only state that can never come from persisted configuration. */ export function rotateProviderTransportOn429( config: OcxConfig, providerName: string, routedProvider: OcxProviderTransport, options: RotateProviderTransportOptions = {}, ): OcxProviderTransport | null { const rotated = rotateKeyOn429( config, providerName, options.retryAfter, options.now, options.attemptedKey, options.attemptedSelection ?? routedProvider._apiKeyAttempt, options.quotaResetAt, ); if (!rotated) return null; return applyRotatedTransport(providerName, routedProvider, rotated, options.promptCacheKey); } /** 401 counterpart of `rotateProviderTransportOn429`; shares its transport-rebuild rules. */ export function rotateProviderTransportOn401( config: OcxConfig, providerName: string, routedProvider: OcxProviderTransport, options: Omit<RotateProviderTransportOptions, "retryAfter"> = {}, ): OcxProviderTransport | null { const rotated = rotateKeyOn401(config, providerName, options.now, options.attemptedKey, options.attemptedSelection ?? routedProvider._apiKeyAttempt); if (!rotated) return null; return applyRotatedTransport(providerName, routedProvider, rotated, options.promptCacheKey); } function applyRotatedTransport( providerName: string, routedProvider: OcxProviderTransport, rotated: OcxProviderConfig, promptCacheKey?: string, ): OcxProviderTransport { const committedRoute = routedProviderConfig(providerName, rotated); const routedSession = routedProvider.headers?.[OPENCODE_GO_SESSION_HEADER]; const retryProvider: OcxProviderTransport = { ...committedRoute, ...(routedProvider.fetch !== undefined ? { fetch: routedProvider.fetch } : {}), ...(routedSession !== undefined ? { headers: { ...(committedRoute.headers ?? {}), [OPENCODE_GO_SESSION_HEADER]: routedSession, }, } : {}), }; return resolveProviderTransport(providerName, retryProvider, promptCacheKey); } /** Clear cooldown state for a provider (e.g. after manual key management). */ export function clearKeyCooldowns(providerName?: string): void { if (!providerName) { keyCooldowns.clear(); return; } const prefix = `${providerName}\0`; for (const key of keyCooldowns.keys()) { if (key.startsWith(prefix)) keyCooldowns.delete(key); } } /** Visible-for-testing: get the cooldown-until timestamp for a key. */ export function getKeyCooldownUntil(providerName: string, keyId: string, now = Date.now()): number | null { const entry = keyCooldowns.get(cooldownKey(providerName, keyId)); if (!entry) return null; return entry.cooldownUntil > now ? entry.cooldownUntil : null; }