@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
872 lines (858 loc) • 47 kB
text/typescript
import { credentialRecoveryStatus } from "../request-log-attempt-recovery";
import { sealCredentialRefusal } from "../credential-recovery-attempt";
import { carryProxyError, proxyErrorPresentation, isProxyError, rememberProxyErrorMessage } from "../../lib/client-error-body";
import { providerAffinityGenerationIsCurrent } from "../../providers/api-key-selection";
import { bindAnthropicRejectionResponse } from "../../adapters/anthropic-rejection-memo";
import { hostedAdvisorCredentials } from "../../adapters/anthropic/advisor-tool";
import { deferPhysicalSendReporter } from "./deferred-send-reporter";
import { responsesSpendTarget } from "./request-spend-target";
import { adapterResponseSpendObserver, bindAdapterResponseSpend } from "./request-spend";
import { anthropicRefusalCredentialChanged, rotateAnthropicAccountOnResponseForInstance } from "../../oauth/anthropic-account-refusal";
import { responsesTimeoutProvider, type ResponsesRequestContext } from "./core-options";
import type { PreparedResponsesRequest } from "./request-prepare";
import type { ResponsesTransport } from "./request-transport";
import type { ResponsesSidecarAuth } from "./request-sidecar-auth";
import type { ResponsesSendBudget } from "./request-send-budget";
import type { AdapterExchange } from "./adapter-dispatch";
import type { OcxParsedRequest, AdapterEvent, OcxUsage } from "../../types";
import type { AttemptRecoveryKind } from "../../usage/log";
import type { AdapterRequest } from "../../adapters/base";
import {
} from "../request-log";
import { noteAttemptRecoveryWithheld } from "../request-log";
import { withProviderRequestSlot } from "../../providers/request-pacing";
import { providerFetch, fetchWithHeaderTimeout, safeHostLabel } from "./fetch-helpers";
import { isLocalUpstream } from "../../lib/local-upstream";
import {
transientRetryPolicyFor,
rateLimitRetryDelayMs,
hasKeyPoolFailover,
rotateProviderTransportOn429,
} from "../../providers/key-failover";
import {
fetchWithTransientRetry,
fetchWithResetRetry,
applyUpstreamRecoveryInit,
isNonReplayableResponse, isSameTargetRetryHold,
prepareSameTarget429Wait,
SendBudgetExhaustedError, errorDestinationLabel,
} from "../../lib/upstream-retry";
import { unwrapUpstreamRetryEvidenceError } from "../../codex/auth-context";
import { AdmissionModelDeniedError, MODEL_NOT_ALLOWED_FOR_KEY } from "../admission-model-scope";
import { HostedAdvisorSpendRefusal } from "./policy-request-scope";
import { redactSecretString } from "../../lib/redact";
import { resolveWireProtocolOverride } from "../adapter-resolve";
import { bindRouteReasoningReplayScope } from "./core-replay";
import {
anthropicRoutingFor,
formatAnthropicProviderForLog,
} from "../../oauth/anthropic-routing";
import {
hasEligibleGenericOAuthFailoverTarget,
isGenericOAuthFailoverEnabled,
rotateGenericOAuthAccountOn429,
rotateGenericOAuthAccountOnRefusal,
quarantineKiroSuspendedAccount,
failoverAccountSnapshot,
} from "../../oauth/generic-account-failover";
import { classifyKiroRefusal } from "../../adapters/kiro-refusal";
import { normalizeFinalKiroHttpError } from "../../adapters/kiro-retry";
import { noteKiroMonthlyRefusal } from "../../providers/kiro-usage";
import { persistKiroAccountState } from "../../providers/kiro-account-state-disk";
import { shouldAttemptImageTierRetry } from "../image-retry";
import { readDisplaySafeErrorText, normalizeUpstreamErrorText } from "./core-errors";
import { isCyberPolicyCode, CYBER_POLICY_FALLBACK_MESSAGE, CYBER_POLICY_ERROR_CODE } from "../../lib/errors";
import {
readResponseBodyWithInactivity,
readResponseStreamWithInactivity,
ResponseBodyInactivityError,
} from "../../lib/response-body-inactivity";
import { providerStallTimeoutSec, resolveStallTimeoutMs } from "../../stall-timeout";
import { guardTerminalEventStream } from "./terminal-guard";
/** One responsibility of the Responses request pipeline; state owners are explicit. */
export function createAdapterContinuations(
requestContext: Pick<ResponsesRequestContext, "options" | "logCtx" | "config">,
requestState: Pick<
PreparedResponsesRequest,
| "route"
| "selectedForwardHeaders"
| "translatorBudget"
| "inboundWire"
| "parsed"
>,
transportState: Pick<
ResponsesTransport,
| "activeAdapter"
| "sameTargetRequest"
| "sameTargetParsed"
| "sameTargetToken"
| "transportToken"
| "imageTierBias"
| "oauthDispatch"
| "invalidateSameTargetRequest"
| "resolveSelectionAdapter"
| "resolveCopilotSelection"
| "anthropicInstance"
| "currentAnthropicRouteDecision"
| "anthropicRouteDecision"
| "anthropicPoolAccountId"
| "anthropicPoolFailovers"
| "anthropicSessionKey"
| "commitResolvedOAuthSelection"
| "genericFailoverAccountId"
| "genericFailovers"
| "genericFailoverLimit" | "anthropicFailoverLimit"
| "applyFailoverSnapshot"
| "replayOAuthCredentialSnapshot"
| "commitAdapterAttemptSend" | "createCredentialReplayAttempt" | "isCurrentInferenceResponse"
>,
sidecarState: Pick<ResponsesSidecarAuth, "routedCompaction">,
sendBudgetState: Pick<
ResponsesSendBudget,
| "adapterDispatchBudget"
| "noteAdapterPhysicalSend"
| "noteAdapterRecoveryWithheld"
| "remainingTransientSendBudget"
| "adapterSendBudget"
| "transientSendReporter"
| "reserveCredentialHop"
| "pendingHopPermit"
| "sendBudgetExhausted"
>,
adapterExchange: Pick<
AdapterExchange,
| "upstream"
| "connectMs"
| "rateLimitPolicy"
| "rateLimitRetries"
| "stallTimeoutMs"
| "localUpstream"
| "thinkingSignatureRecovery"
>,
) {
// A request-local symbol survives event spreads without being serialized or forgeable by a provider.
const continuationBodyStall = Symbol("continuation-body-stall");
const { options, logCtx, config } = requestContext;
const {
oauthDispatch,
invalidateSameTargetRequest,
resolveSelectionAdapter,
anthropicInstance,
currentAnthropicRouteDecision,
anthropicSessionKey,
commitResolvedOAuthSelection,
applyFailoverSnapshot,
} = transportState;
const { route, translatorBudget, inboundWire, parsed } = requestState;
const { routedCompaction } = sidecarState;
const { upstream, connectMs, rateLimitPolicy, stallTimeoutMs } = adapterExchange;
const {
adapterDispatchBudget,
noteAdapterPhysicalSend,
noteAdapterRecoveryWithheld,
remainingTransientSendBudget,
transientSendReporter,
reserveCredentialHop,
sendBudgetExhausted,
} = sendBudgetState;
const continuationFailureEvent = (error: unknown): Extract<AdapterEvent, { type: "error" }> => {
if (options.abortSignal?.aborted || upstream.signal.aborted) {
return { type: "error", message: "client closed request during terminal continuation", status: 499 };
}
const refusal = unwrapUpstreamRetryEvidenceError(error);
if (refusal instanceof HostedAdvisorSpendRefusal) return refusal.event();
if (refusal instanceof AdmissionModelDeniedError || refusal instanceof SendBudgetExhaustedError) rememberProxyErrorMessage(logCtx, refusal.message, { status: refusal instanceof AdmissionModelDeniedError ? 403 : 429,
type: refusal instanceof AdmissionModelDeniedError ? MODEL_NOT_ALLOWED_FOR_KEY : "rate_limit_error", code: refusal instanceof AdmissionModelDeniedError ? MODEL_NOT_ALLOWED_FOR_KEY : refusal.code, presentation: refusal instanceof AdmissionModelDeniedError ? "scope" : "send" });
if (refusal instanceof AdmissionModelDeniedError) {
return {
type: "error", message: refusal.message, status: 403,
errorType: MODEL_NOT_ALLOWED_FOR_KEY, code: MODEL_NOT_ALLOWED_FOR_KEY, retryable: false,
};
}
if (refusal instanceof SendBudgetExhaustedError) return {
type: "error", status: 429, errorType: "rate_limit_error", code: refusal.code,
message: refusal.message, retryable: false,
};
return {
type: "error",
message: `Provider continuation failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}`,
};
};
// One bounded internal continuation re-ask for clean end_turn turns that announced an edit
// without emitting a tool call. Anthropic gets this by default; openai-chat providers opt in
// per-provider via `terminalContinuationGuard` (the heuristic was tuned on Anthropic turns,
// so it stays off for the shared openai-chat adapter unless a provider enables it).
const terminalGuardEnabled = (transportState.activeAdapter.name === "anthropic"
|| (transportState.activeAdapter.name === "openai-chat" && route.provider.terminalContinuationGuard === true))
&& !options.comboAttempt && !routedCompaction;
/**
* One bounded internal re-ask for Anthropic end_turn-without-tool-call turns. Replays the
* continuation on a 429 with the same-key retry budget (hoisted per request), then falls
* back to key/account failover, rebuilding Copilot model, wire, and credential negotiation.
* A failure becomes an in-stream adapter error so the client
* never sees a second hidden HTTP response or an unbounded retry loop.
*/
const fetchTerminalGuardContinuation = async function* (
nextParsed: OcxParsedRequest,
initialRecoveryKind?: AttemptRecoveryKind,
// Ordinary terminal continuations have already delivered output; empty retries opt in.
allowAccountRefusal = false,
): AsyncGenerator<AdapterEvent> {
let response: Response | undefined;
let refusalPendingReplay: Response | undefined;
let replaySetupFailed = false;
// Refused account's request attribution, restored when its replacement fails before sending.
let refusalAttribution: { provider: string; label?: string; ordinal?: number; sends?: number;
attemptLabel?: NonNullable<typeof logCtx.activeAttempt>["accountLogLabel"] } | undefined;
const captureRefusalAttribution = () => {
const attempt = logCtx.activeAttempt;
refusalAttribution = { provider: logCtx.provider, label: logCtx.accountLogLabel, ordinal: attempt?.ordinal,
sends: attempt?.sendCount, attemptLabel: attempt?.accountLogLabel };
};
// One-shot recovery label for the next top-of-loop continuation send after a failover rotation.
let nextContinuationRecoveryKind: AttemptRecoveryKind | undefined = initialRecoveryKind;
/**
* Build and fetch one terminal-guard continuation. `recoveryKind` tags same-target and
* failover sends (`empty-completion`, `rate-limit-429`, `key-429`,
* `anthropic-oauth-429`, `image-413`); the
* adapter rebuild is deterministic for the same parsed request (tests assert byte-identical
* replays).
*/
const fetchContinuation = async (recoveryKind?: AttemptRecoveryKind): Promise<Response> => {
const replay = recoveryKind && credentialRecoveryStatus(recoveryKind) !== undefined && response && logCtx.activeAttempt?.sendCount
? transportState.createCredentialReplayAttempt(response.status, recoveryKind) : undefined;
let continuationRequest: AdapterRequest | undefined;
if (transportState.sameTargetRequest !== undefined && transportState.sameTargetParsed === nextParsed && transportState.sameTargetToken === transportState.transportToken) {
// Same target (key/adapter/parsed/tier unchanged): replay the exact cached request.
continuationRequest = transportState.sameTargetRequest;
} else {
try {
continuationRequest = await transportState.activeAdapter.buildRequest(nextParsed, {
headers: requestState.selectedForwardHeaders,
providerName: route.providerName,
translatorBudget,
abortSignal: upstream.signal,
...(transportState.imageTierBias > 0 ? { imageTierBias: transportState.imageTierBias } : {}),
});
} catch (err) {
// The main body is already streaming, so there is no HTTP error surface: release
// any partial body observation and surface the failure as an in-stream error via
// the outer catch (no upstream.abort() — that would kill the live body stream).
continuationRequest?.releaseBodyObservation?.();
throw err;
}
transportState.sameTargetRequest = continuationRequest;
transportState.sameTargetParsed = nextParsed;
transportState.sameTargetToken = transportState.transportToken;
}
// Both branches assign the request (the build catch rethrows), so capture it in a
// const for the fetch callback and finally below — a `let` read inside a nested
// function keeps its undefined half, which would break the byte-identical replay.
const builtContinuationRequest = continuationRequest;
const continuationEstimate = typeof builtContinuationRequest.usageLog?.inputTokens === "number"
? builtContinuationRequest.usageLog.inputTokens
: undefined;
const commit = (physicalRecovery?: AttemptRecoveryKind): void => {
const adapter = transportState.activeAdapter.name;
if (replay) replay(builtContinuationRequest, adapter, physicalRecovery);
else transportState.commitAdapterAttemptSend(builtContinuationRequest, adapter, physicalRecovery ?? recoveryKind);
};
try {
if (transportState.activeAdapter.fetchResponse) {
let published = false;
const publish = (): void => {
if (published) return;
commit();
published = true;
};
const producer = adapterDispatchBudget?.beginSpendProducer?.();
try {
return await withProviderRequestSlot(route.providerName, route.provider, nextParsed.modelId, upstream.signal, pacingSlot =>
transportState.activeAdapter.fetchResponse!(builtContinuationRequest, {
kiroPreferAccountFailover: route.providerName === "kiro" && isGenericOAuthFailoverEnabled(config, "kiro"),
abortSignal: upstream.signal,
timeoutMs: route.provider.authMode === "forward" ? connectMs : 0,
sendBudget: adapterDispatchBudget,
onPhysicalSend: send => {
if (transportState.activeAdapter.reportsPhysicalSends) publish();
noteAdapterPhysicalSend(continuationEstimate, send);
},
onRecoveryWithheld: noteAdapterRecoveryWithheld,
stream: nextParsed.stream,
executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, {
pacingSlotAcquired: true,
pacingSlot,
dispatchOverride: oauthDispatch(builtContinuationRequest, nextParsed, "adapter", true),
onPhysicalDispatch: publish,
providerName: route.providerName,
modelId: nextParsed.modelId,
}),
}));
} finally { producer?.close(); }
}
// Same #1851 scope guard as the initial send: transient-5xx retry only for direct
// Google AI Studio; every other adapter keeps reset-only semantics here.
const continuationTransientPolicy = transientRetryPolicyFor(route.provider);
const fetchContinuationWithRetryPolicy = (route.provider.adapter === "google" || continuationTransientPolicy)
? fetchWithTransientRetry
: fetchWithResetRetry;
const continuationReporter = transientSendReporter();
const countsContinuationSends = continuationTransientPolicy || anthropicInstance || sendBudgetState.adapterSendBudget?.spendEnforced;
const deferredContinuation = countsContinuationSends && (hostedAdvisorCredentials.has(builtContinuationRequest)
|| ((continuationTransientPolicy || sendBudgetState.adapterSendBudget?.spendEnforced)
&& route.providerName === "github-copilot" && route.provider.authMode !== "forward"))
? deferPhysicalSendReporter(continuationReporter) : undefined;
const continuationResponse = await fetchContinuationWithRetryPolicy(
recovery => {
sendBudgetState.adapterSendBudget?.startRequest?.(responsesSpendTarget(logCtx, route.providerName));
return fetchWithHeaderTimeout(
builtContinuationRequest.url,
applyUpstreamRecoveryInit({
method: builtContinuationRequest.method,
headers: builtContinuationRequest.headers,
body: builtContinuationRequest.body,
}, recovery),
upstream.signal,
route.provider.authMode === "forward" ? connectMs : 0,
nextParsed.stream,
providerFetch(route.provider, options.codexWsRuntimeIdentity, {
dispatchOverride: oauthDispatch(builtContinuationRequest, nextParsed, "adapter", true),
onPhysicalDispatch: () => {
if (deferredContinuation && !deferredContinuation.beforeDispatch()) throw new SendBudgetExhaustedError(errorDestinationLabel(builtContinuationRequest.url));
commit(recovery);
},
providerName: route.providerName,
modelId: nextParsed.modelId,
}),
);
},
{
abortSignal: upstream.signal,
label: safeHostLabel(builtContinuationRequest.url),
// Same request-scoped budget as the initial send and the 429/rotation refetches:
// a terminal-guard continuation is another leg of ONE request, so handing it a
// fresh `attempts` would let one request exceed the configured total-send ceiling.
...(countsContinuationSends
? {
attempts: Math.min(continuationTransientPolicy?.attempts ?? 1,
remainingTransientSendBudget(continuationTransientPolicy?.attempts ?? 1)
+ (sendBudgetState.pendingHopPermit ? 1 : 0)),
onSendsConsumed: deferredContinuation?.report ?? continuationReporter,
}
: {}),
},
);
if (nextParsed._anthropicImageOwner && sendBudgetState.adapterSendBudget?.spendEnforced) {
bindAdapterResponseSpend(continuationResponse, continuationReporter.recordUsage);
}
return continuationResponse;
} finally {
builtContinuationRequest.releaseBodyObservation?.();
}
};
while (true) {
try {
const recoveryKind = nextContinuationRecoveryKind;
nextContinuationRecoveryKind = undefined;
response = await fetchContinuation(recoveryKind);
bindAnthropicRejectionResponse(transportState.sameTargetRequest, response);
if (initialRecoveryKind === "empty-completion" && await adapterExchange.thinkingSignatureRecovery.prepareHttp(
response, transportState.activeAdapter.name, nextParsed,
)) {
try { void response.body?.cancel().catch(() => {}); } catch { /* closed */ }
invalidateSameTargetRequest();
response = await fetchContinuation("opaque-blob-rejection");
}
if (refusalPendingReplay) {
try { void refusalPendingReplay.body?.cancel().catch(() => {}); } catch { /* already closed */ }
refusalPendingReplay = undefined;
sendBudgetState.pendingHopPermit = undefined;
}
} catch (error) {
if (refusalPendingReplay && !options.abortSignal?.aborted && !upstream.signal.aborted) {
sendBudgetState.pendingHopPermit?.release();
sendBudgetState.pendingHopPermit = undefined;
response = refusalPendingReplay;
refusalPendingReplay = undefined;
replaySetupFailed = true;
const attempt = logCtx.activeAttempt;
// Same attempt and send count: the replacement never dispatched, so the refusal stays the refused account's.
if (attempt && refusalAttribution && attempt.ordinal === refusalAttribution.ordinal && attempt.sendCount === refusalAttribution.sends) {
logCtx.provider = refusalAttribution.provider;
if (refusalAttribution.label === undefined) delete logCtx.accountLogLabel;
else logCtx.accountLogLabel = refusalAttribution.label;
if (refusalAttribution.attemptLabel === undefined) delete attempt.accountLogLabel;
else attempt.accountLogLabel = refusalAttribution.attemptLabel;
}
} else {
if (refusalPendingReplay) {
sendBudgetState.pendingHopPermit?.release();
sendBudgetState.pendingHopPermit = undefined;
try { void refusalPendingReplay.body?.cancel().catch(() => {}); } catch { /* closed */ }
}
yield continuationFailureEvent(error); return;
}
}
// Same-target 429 wait-and-retry (opt-in `retryOn429`) before key/account failover:
// a primary-key rate-limit blip replays on the SAME key, matching the main recovery
// loop; only after the attempts are exhausted does the continuation fail over.
while (
response.status === 429
&& transportState.isCurrentInferenceResponse(response)
&& !replaySetupFailed
// A synthesized replay refusal is not a rate limit; replaying the continuation on
// it would re-send a turn whose first send may already have been processed.
&& !isNonReplayableResponse(response) && !isSameTargetRetryHold(response)
&& rateLimitPolicy !== null && providerAffinityGenerationIsCurrent(config, route.providerName, route.provider)
&& adapterExchange.rateLimitRetries < rateLimitPolicy.attempts
// The main recovery loop and the passthrough ladder both consult the shared remainder
// here; this loop did not, so a request whose budget was already spent could still
// same-key replay on a live stream. Checked BEFORE the wait below cancels the body, so
// a refusal keeps the real upstream 429 -- status, Retry-After, quota evidence -- intact.
&& !sendBudgetExhausted()
) {
adapterExchange.rateLimitRetries += 1;
// Release unread body + heartbeat-fed wait via the shared same-target helper.
const retryAfterHeader = response.headers.get("retry-after");
try {
yield* prepareSameTarget429Wait({
body: response.body,
// Listen on the upstream signal: once the SSE body is being streamed, a client
// cancel aborts `upstream` through the bridge, and upstream is also linked from
// options.abortSignal — so this covers both cancellation paths.
signal: upstream.signal,
delayMs: rateLimitRetryDelayMs(rateLimitPolicy, retryAfterHeader, Date.now()),
heartbeatIntervalMs: Math.min(10_000, Math.max(250, stallTimeoutMs / 2)),
});
} catch {
if (options.abortSignal?.aborted || upstream.signal.aborted) {
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
} else {
yield { type: "error", message: "Provider continuation failed: retry wait interrupted" };
}
return;
}
// Client cancellation wins over any stale timer edge: re-check before dispatching the
// replay so the continuation never starts work for a request the client abandoned.
if (options.abortSignal?.aborted || upstream.signal.aborted) {
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
return;
}
try {
response = await fetchContinuation("rate-limit-429");
} catch (error) {
yield continuationFailureEvent(error); return;
}
}
if (response.status === 429 && !transportState.isCurrentInferenceResponse(response)) break;
if (response.status === 429 && !isNonReplayableResponse(response) && hasKeyPoolFailover(route.provider)) {
const rotated = rotateProviderTransportOn429(config, route.providerName, route.provider, {
retryAfter: response.headers.get("retry-after"),
now: Date.now(),
attemptedKey: route.provider.apiKey,
promptCacheKey: nextParsed.options.promptCacheKey,
});
if (rotated) {
sealCredentialRefusal(logCtx, response.status, "key-429");
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
route.provider = rotated;
try { await transportState.resolveCopilotSelection(nextParsed); }
catch (error) {
yield continuationFailureEvent(error); return;
}
invalidateSameTargetRequest();
transportState.activeAdapter = resolveSelectionAdapter(
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy),
config.cacheRetention,
);
bindRouteReasoningReplayScope({
parsed: nextParsed,
providerName: route.providerName,
provider: route.provider,
adapterName: transportState.activeAdapter.name,
});
// Response persistence closes over the outer parsed request; keep its owner binding in
// sync with the terminal-guard clone that builds the rotated continuation request.
bindRouteReasoningReplayScope({
parsed,
providerName: route.providerName,
provider: route.provider,
adapterName: transportState.activeAdapter.name,
});
nextContinuationRecoveryKind = "key-429";
continue;
}
}
if (
(response.status === 429 || response.status === 403 || response.status === 401)
&& anthropicInstance
&& transportState.anthropicPoolAccountId
&& !replaySetupFailed && !isNonReplayableResponse(response)
) {
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
const hop = reserveCredentialHop("auth-recovery", `${route.providerName}|${route.modelId}|continuation-anthropic-429`,
!adapterOwnsDispatch);
let nextAccountId: string | null;
try {
nextAccountId = await rotateAnthropicAccountOnResponseForInstance(anthropicInstance, response, {
config, accountId: transportState.anthropicPoolAccountId, sessionKey: anthropicSessionKey,
model: route.modelId, requestKey: transportState, decision: transportState.anthropicRouteDecision, currentDecision: currentAnthropicRouteDecision, signal: upstream.signal,
canRetry: hop.allowed && transportState.anthropicPoolFailovers < transportState.anthropicFailoverLimit,
allowAccountRefusal, allow429Recovery: allowAccountRefusal,
});
} catch (error) { hop.permit?.release(); throw error; }
if (!nextAccountId) hop.permit?.release();
if (nextAccountId) {
captureRefusalAttribution();
const recovery = response.status === 401 ? "oauth-401"
: response.status === 403 ? "oauth-account-403" : "anthropic-oauth-429";
try {
const admitted = await commitResolvedOAuthSelection(await anthropicRoutingFor(anthropicInstance).getAnthropicPoolAccessSnapshot(nextAccountId));
if (!admitted) throw new Error("OAuth selection changed during recovery");
const changed = anthropicRefusalCredentialChanged(response, admitted);
if (changed) sealCredentialRefusal(logCtx, response.status, recovery);
transportState.anthropicPoolAccountId = admitted.accountId;
transportState.anthropicPoolFailovers += 1;
route.provider = { ...route.provider, apiKey: admitted.accessToken };
invalidateSameTargetRequest();
logCtx.provider = formatAnthropicProviderForLog(anthropicInstance, admitted.accountId, config);
transportState.activeAdapter = resolveSelectionAdapter(
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy),
config.cacheRetention,
);
// Rebind both the continuation clone and the outer request that persists the response.
for (const requestParsed of [nextParsed, parsed]) {
bindRouteReasoningReplayScope({
parsed: requestParsed,
providerName: route.providerName,
provider: route.provider,
adapterName: transportState.activeAdapter.name,
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
});
}
nextContinuationRecoveryKind = changed ? recovery : "rate-limit-429";
sendBudgetState.pendingHopPermit = hop.permit;
refusalPendingReplay = response;
continue;
} catch {
hop.permit?.release();
// fall through to emit continuation error below
}
}
}
// Generic OAuth rotation for the continuation loop. The streaming loop grew this arm with
// #2568 and this one did not, so an xAI/Cursor/Kimi/Copilot/Antigravity/Nous continuation
// 429 stayed terminal even with failover fully active -- the same class of divergence the
// two sidecars already produced once. Request-local state is shared with the other arms so
// the per-request bound cannot be silently re-armed by reaching a different loop.
if (route.providerName === "kiro") {
if (
(response.status === 429 || response.status === 400 || response.status === 403)
&& transportState.genericFailoverAccountId
&& !isNonReplayableResponse(response)
&& !replaySetupFailed
) {
const refusal = classifyKiroRefusal(response.status,
await readDisplaySafeErrorText(response.clone(), upstream.signal, ""));
if (refusal.kind === "other") break;
const sent = transportState.replayOAuthCredentialSnapshot;
const monthlyCooldownMs = refusal.kind === "monthly_quota" && sent
? noteKiroMonthlyRefusal(sent.accountId, sent.generation, Date.now()) : undefined;
if (monthlyCooldownMs !== undefined) persistKiroAccountState();
if (refusal.kind === "suspended")
quarantineKiroSuspendedAccount(transportState.genericFailoverAccountId, sent?.generation);
if (transportState.genericFailovers >= transportState.genericFailoverLimit
|| !isGenericOAuthFailoverEnabled(config, "kiro")) break;
// Intersection with the shared request budget. The continuation loop re-sends the
// turn, so without this the per-request bound could be re-armed simply by reaching a
// different loop -- which is the divergence the comment above already warns about.
//
// Who settles the reservation depends on who sends the replay (#4709). An adapter that
// owns its ladder reserves once per physical send and would charge this replay twice;
// the helper path reports it back instead, which is what `countedExternally` names.
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
const hop = reserveCredentialHop(
"auth-recovery",
`${route.providerName}|${route.modelId}|continuation-oauth-429`,
!adapterOwnsDispatch && transientRetryPolicyFor(route.provider) !== null,
);
const nextAccountId = hop.allowed
? rotateGenericOAuthAccountOnRefusal(
config,
route.providerName,
transportState.genericFailoverAccountId,
refusal.kind,
response.headers.get("retry-after"),
Date.now(),
route.modelId,
monthlyCooldownMs,
)
: null;
// A roster quorum ignores cooldowns, so only attribute a budget refusal when the
// non-mutating selector confirms that an alternate account could serve this model now.
if (!hop.allowed && hasEligibleGenericOAuthFailoverTarget(
route.providerName, transportState.genericFailoverAccountId, Date.now(), route.modelId,
)) noteAttemptRecoveryWithheld(logCtx.activeAttempt, "rotation-send-budget");
if (!nextAccountId) hop.permit?.release();
if (nextAccountId) {
sealCredentialRefusal(logCtx, response.status, "oauth-account-429");
try {
// The FULL snapshot through the shared helper, never a bare bearer: Antigravity
// pairs an account-matched projectId with its token and Kiro carries routing
// metadata, so a token-only swap would mix one account's credential with another's
// routing data.
const snapshot = await failoverAccountSnapshot(route.providerName, nextAccountId);
captureRefusalAttribution();
const applied = await applyFailoverSnapshot(snapshot, nextParsed);
if (!applied) hop.permit?.release();
if (applied) {
transportState.genericFailovers += 1;
invalidateSameTargetRequest();
transportState.activeAdapter = resolveSelectionAdapter(
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy),
config.cacheRetention,
);
bindRouteReasoningReplayScope({
parsed: nextParsed,
providerName: route.providerName,
provider: route.provider,
adapterName: transportState.activeAdapter.name,
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
});
// Response persistence closes over the outer parsed request; keep its owner binding in
// sync with the terminal-guard clone that builds the rotated continuation request.
bindRouteReasoningReplayScope({
parsed,
providerName: route.providerName,
provider: route.provider,
adapterName: transportState.activeAdapter.name,
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
});
// The replay goes out on the next iteration. An adapter that owns its ladder
// reserves for that send itself, so hand this reservation down rather than let it
// take a second one for the same replay. A helper-routed replay carries the
// same permit so its reporter settles only this booking.
sendBudgetState.pendingHopPermit = hop.permit;
nextContinuationRecoveryKind = "oauth-account-429";
refusalPendingReplay = response;
continue;
}
} catch {
// Everything in this try runs before the replay: the send happens on the next
// iteration, after `continue`. A throw here therefore leaves a reservation that
// never dispatched, and holding it would refuse a later recovery in this same
// request for a send that never left the process.
hop.permit?.release();
// fall through to emit continuation error below
}
}
}
} else {
if (
response.status === 429
&& transportState.genericFailoverAccountId
&& !isNonReplayableResponse(response)
&& transportState.genericFailovers < transportState.genericFailoverLimit
&& isGenericOAuthFailoverEnabled(config, route.providerName)
) {
// Intersection with the shared request budget. The continuation loop re-sends the
// turn, so without this the per-request bound could be re-armed simply by reaching a
// different loop -- which is the divergence the comment above already warns about.
//
// Who settles the reservation depends on who sends the replay (#4709). An adapter that
// owns its ladder reserves once per physical send and would charge this replay twice;
// the helper path reports it back instead, which is what `countedExternally` names.
const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined;
const hop = reserveCredentialHop(
"auth-recovery",
`${route.providerName}|${route.modelId}|continuation-oauth-429`,
!adapterOwnsDispatch && transientRetryPolicyFor(route.provider) !== null,
);
const nextAccountId = hop.allowed
? rotateGenericOAuthAccountOn429(
config,
route.providerName,
transportState.genericFailoverAccountId,
response.headers.get("retry-after"),
Date.now(),
route.modelId,
)
: null;
// A roster quorum ignores cooldowns, so only attribute a budget refusal when the
// non-mutating selector confirms that an alternate account could serve this model now.
if (!hop.allowed && hasEligibleGenericOAuthFailoverTarget(
route.providerName, transportState.genericFailoverAccountId, Date.now(), route.modelId,
)) noteAttemptRecoveryWithheld(logCtx.activeAttempt, "rotation-send-budget");
if (!nextAccountId) hop.permit?.release();
if (nextAccountId) {
sealCredentialRefusal(logCtx, response.status, "oauth-account-429");
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
try {
// The FULL snapshot through the shared helper, never a bare bearer: Antigravity
// pairs an account-matched projectId with its token and Kiro carries routing
// metadata, so a token-only swap would mix one account's credential with another's
// routing data.
const snapshot = await failoverAccountSnapshot(route.providerName, nextAccountId);
transportState.genericFailovers += 1;
const applied = await applyFailoverSnapshot(snapshot, nextParsed);
if (!applied) hop.permit?.release();
if (applied) {
invalidateSameTargetRequest();
transportState.activeAdapter = resolveSelectionAdapter(
resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy),
config.cacheRetention,
);
bindRouteReasoningReplayScope({
parsed: nextParsed,
providerName: route.providerName,
provider: route.provider,
adapterName: transportState.activeAdapter.name,
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
});
// Response persistence closes over the outer parsed request; keep its owner binding in
// sync with the terminal-guard clone that builds the rotated continuation request.
bindRouteReasoningReplayScope({
parsed,
providerName: route.providerName,
provider: route.provider,
adapterName: transportState.activeAdapter.name,
oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
});
// The replay goes out on the next iteration. An adapter that owns its ladder
// reserves for that send itself, so hand this reservation down rather than let it
// take a second one for the same replay. A helper-routed replay carries the
// same permit so its reporter settles only this booking.
sendBudgetState.pendingHopPermit = hop.permit;
nextContinuationRecoveryKind = "oauth-account-429";
continue;
}
} catch (error) {
// Everything in this try runs before the replay: the send happens on the next
// iteration, after `continue`. A throw here therefore leaves a reservation that
// never dispatched, and holding it would refuse a later recovery in this same
// request for a send that never left the process.
hop.permit?.release();
if (unwrapUpstreamRetryEvidenceError(error) instanceof AdmissionModelDeniedError) {
yield continuationFailureEvent(error);
return;
}
// Other negotiation failures preserve the original provider refusal.
}
}
}
}
if (shouldAttemptImageTierRetry({
status: response.status,
adapterName: transportState.activeAdapter.name,
parsed: nextParsed,
alreadyAttempted: transportState.imageTierBias > 0,
})) {
transportState.imageTierBias = 1;
invalidateSameTargetRequest();
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
nextContinuationRecoveryKind = "image-413";
continue;
}
break;
}
if (!response.ok) {
if (route.providerName === "kiro") response = carryProxyError(response, await normalizeFinalKiroHttpError(response, upstream.signal));
const errorText = await readDisplaySafeErrorText(response, upstream.signal, "unknown error");
const normalized = normalizeUpstreamErrorText(errorText, "unknown error");
const message = normalized.cyberPolicy
? normalized.message ?? (isCyberPolicyCode(normalized.code) ? CYBER_POLICY_FALLBACK_MESSAGE : normalized.safeText)
: `Provider continuation error ${response.status}: ${normalized.safeText}`;
if (isProxyError(response)) rememberProxyErrorMessage(logCtx, message, { status: normalized.cyberPolicy ? 400 : response.status,
type: normalized.cyberPolicy ? normalized.type ?? CYBER_POLICY_ERROR_CODE : undefined, code: normalized.cyberPolicy ? CYBER_POLICY_ERROR_CODE : undefined, presentation: proxyErrorPresentation(response) });
yield {
type: "error",
status: normalized.cyberPolicy ? 400 : response.status,
message,
...(normalized.cyberPolicy
? {
errorType: normalized.type ?? CYBER_POLICY_ERROR_CODE,
code: CYBER_POLICY_ERROR_CODE,
retryable: false,
}
: {}),
};
return;
}
try {
// Each successful continuation owns a fresh pending-read deadline and an abort
// listener that can cancel its reader even while the parser holds the body lock.
// The shared signal is not aborted on timeout: this generator must still emit
// the error that the already-live Responses bridge turns into a terminal.
const parse = async function* (current: Response): AsyncGenerator<AdapterEvent> {
const bodyInactivityMs = resolveStallTimeoutMs(providerStallTimeoutSec(config, responsesTimeoutProvider(options, route.provider)), { localUpstream: isLocalUpstream(transportState.sameTargetRequest?.url ?? route.provider.baseUrl) });
if (!current.ok) {
yield await adapterExchange.thinkingSignatureRecovery.projectHttpFailure(current, upstream.signal);
return;
}
if (!nextParsed.stream && !transportState.activeAdapter.parseResponse) {
try { void current.body?.cancel().catch(() => {}); } catch { /* closed */ }
yield { type: "error", message: "Provider continuation does not support response parsing" };
return;
}
const observeSpend = adapterResponseSpendObserver(current);
const events = nextParsed.stream
? readResponseStreamWithInactivity(current, upstream.signal, bodyInactivityMs,
guarded => transportState.activeAdapter.parseStream(guarded, translatorBudget, logCtx.activeTierMetadata))
: await readResponseBodyWithInactivity(current, upstream.signal, bodyInactivityMs,
guarded => transportState.activeAdapter.parseResponse!(guarded, translatorBudget, logCtx.activeTierMetadata));
for await (const event of events) {
observeSpend?.(event);
yield event;
}
};
yield* adapterExchange.thinkingSignatureRecovery.wrap(parse(response), async function* () {
invalidateSameTargetRequest();
yield* parse(await fetchContinuation("opaque-blob-rejection"));
}, () => transportState.activeAdapter.name,
initialRecoveryKind === "empty-completion" && !isNonReplayableResponse(response), response, nextParsed);
} catch (error) {
// Classify the thrown error before reading signal state: cancelling the stalled
// source can synchronously abort the shared signal, which would otherwise report
// this timeout as a client cancellation.
if (error instanceof ResponseBodyInactivityError) {
const event: AdapterEvent = { type: "error", message: error.message, status: 504,
errorType: "upstream_error", code: "upstream_stall_timeout" };
Object.defineProperty(event, continuationBodyStall, { value: true, enumerable: true });
yield event;
} else if (options.abortSignal?.aborted || upstream.signal.aborted) {
yield { type: "error", message: "client closed request during terminal continuation", status: 499 };
} else {
yield { type: "error", message: `Provider continuation parse failed: ${redactSecretString(error instanceof Error ? error.message : String(error))}` };
}
}
};
// The delivery's onHiddenUsage, so a re-ask inside the retry keeps its legs off the client terminal (#8877).
const fetchGuardedEmptyCompletionRetry = (onHiddenUsage?: (usage: OcxUsage) => void): AsyncIterable<AdapterEvent> => {
const retryEvents = fetchTerminalGuardContinuation(parsed, "empty-completion", true);
const terminalGuardedRetry = terminalGuardEnabled
? guardTerminalEventStream({
parsed,
firstEvents: retryEvents,
adapterName: transportState.activeAdapter.name,
maxAutoContinuations: 1,
continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream),
onHiddenUsage,
})
: retryEvents;
return parsed._advisorGuard
? parsed._advisorGuard({
parsed,
translatorBudget,
firstEvents: terminalGuardedRetry,
continuation: fetchTerminalGuardContinuation,
})
: terminalGuardedRetry;
};
return {
terminalGuardEnabled,
fetchTerminalGuardContinuation,
fetchGuardedEmptyCompletionRetry,
isContinuationBodyStall: (event: AdapterEvent): boolean => continuationBodyStall in event,
};
}
export type AdapterContinuations = Exclude<ReturnType<typeof createAdapterContinuations>, Response>;