@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
1,043 lines (979 loc) • 48.3 kB
text/typescript
import type { OcxConfig, OcxProviderConfig, OcxParsedRequest } from "../../types";
import type { CodexAuthContext, CodexAuthPolicyConfig } from "../../codex/auth-context";
import type { CodexUpstreamOutcome } from "../../codex/routing";
import {
recordCodexUpstreamOutcome,
computeQuotaCooldown,
formatCodexProviderForLog,
} from "../../codex/routing";
import type { CodexWsQuotaObserver } from "./codex-ws-metadata";
import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
import { isCodexAccountGenerationLive } from "../../codex/account-store";
import { applyAccountQuotaFromUpstreamHeaders as applyCapturedCodexQuota } from "../../codex/quota";
import type { RouteResult } from "../../router";
import {
normalizeUpstreamHostCircuitThreshold,
upstreamHostHealthKey,
resetUpstreamHostHealth,
} from "../../codex/upstream-host-health";
import { safeOriginLabel, fetchWithHeaderTimeout, providerFetch } from "./fetch-helpers";
import { classifyPoolRecoveryDispatch } from "../../routing/probe-lease";
import { formatErrorResponse } from "../../bridge";
import { readBoundedResponseBody } from "../../lib/bounded-body";
import { upstreamErrorMessageFromPayload, isRateLimitOrQuotaFailureMessage } from "../../lib/errors";
import { isNonReplayableResponse, isTransientUpstreamStatus } from "../../lib/upstream-retry";
import type { RequestLogContext } from "../request-log";
import type { DataPlaneAdmission } from "../auth-cors";
import type { InboundWire } from "../../providers/registry";
import type { BunRuntimeGateInput } from "./ws-upstream";
import type { TranslatorBudget } from "../../lib/translator-budget";
import type { AdmissionLease } from "../../lib/admission";
import {
resolveCodexModelEntitlements,
invalidateCodexModelEntitlementsForAccount,
entitledCodexAccountIdsForModel,
recordCodexModelDenialEvidence,
} from "../../codex/model-entitlements";
import type { TransientSendBudget } from "../../lib/upstream-retry";
import { resolveAdapter, resolveWireProtocolOverride } from "../adapter-resolve";
import { codexAccountSelectionForTurn } from "../lifecycle";
import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup";
import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/main-account";
import { slugsEquivalent } from "../../providers/slug-codec";
import {
callerCodexWorkspaceAccountId,
codexProbeLeaseId,
codexTransientProbeGrant,
codexProbeQuotaScope,
releaseCodexAuthContextProbeLease,
resolveCodexAuthContext,
CodexPoolAuthenticationError,
CodexAuthContextError,
CodexAccountCooldownError,
CodexMainProfileDrainingError,
headersForCodexAuthContext,
applyCodexAuthContextToProvider,
stripCodexRuntimeProviderFields,
createCodexReserveDispatchGuard,
} from "../../codex/auth-context";
import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS } from "../../codex/catalog/native-models";
import { isRequestExecutionBudget } from "../../lib/request-execution-budget";
import type { SingleUseDispatchPermit } from "../../lib/request-execution-budget";
import { hasForwardableCodexBearer } from "../auth-cors";
import { bindRouteReasoningReplayScope } from "./core-replay";
import {
conversationStateBindingFromAuth,
applyAccountChangeConversationStateScrub,
conversationCarriesUploadedFiles,
} from "./account-change-state";
import {
recordAdapterReasoning,
recordAdapterTier,
sealRequestAttemptIdentity,
recordAttemptCredentialSource,
noteProviderAttemptSend,
} from "../request-log";
import { codexAuthContextLogLabel } from "../../codex/account-label";
import { chargeWorkflowSends } from "../../lib/workflow-budget";
import type { ResponsesTerminalStatus } from "../../bridge";
export function sidecarOutcomeRecorder(
config: OcxConfig,
authCtx: CodexAuthContext,
): ((outcome: CodexUpstreamOutcome) => void) | undefined {
return authCtx.kind === "pool" || authCtx.kind === "main-pool"
? outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, {
threadId: authCtx.affinityKey,
fixedAccount: authCtx.fixedAccount,
probeLeaseId: authCtx.probeLeaseId,
probeQuotaScope: authCtx.probeQuotaScope,
writerGeneration: authCtx.writerGeneration,
// A vision or web-search sidecar can return 401/403, and that is evidence about the exact
// stored credential it used. Without the generation it becomes an account-wide quarantine
// that a replacement inherits (#2892 gap 4). `main-pool` has no stored-record generation, so
// it keeps the unfenced account-wide semantics.
...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}),
})
: undefined;
}
export function codexLogAccountId(authCtx: CodexAuthContext): string | null {
return authCtx.kind === "pool" || authCtx.kind === "main-pool" ? authCtx.accountId : null;
}
export function isFixedCodexAccount(authCtx: CodexAuthContext): boolean {
return (authCtx.kind === "pool" || authCtx.kind === "main-pool")
&& authCtx.fixedAccount === true;
}
export function usesCodexForwardPoolAuth(
authCtx: CodexAuthContext,
provider: OcxProviderConfig,
): authCtx is Extract<CodexAuthContext, { kind: "pool" | "main-pool" }> {
return (authCtx.kind === "pool" || authCtx.kind === "main-pool")
&& provider.authMode === "forward" && provider.adapter === "openai-responses";
}
export function codexWsQuotaObserver(authCtx: CodexAuthContext, provider: OcxProviderConfig, modelId?: string): CodexWsQuotaObserver | undefined {
if (!isCanonicalOpenAiForwardProvider(provider) || !usesCodexForwardPoolAuth(authCtx, provider)) return undefined;
const { accountId, writerGeneration } = authCtx;
const credentialGeneration = authCtx.kind === "pool" ? authCtx.generation : undefined;
const mainWriter = authCtx.kind === "main-pool" ? authCtx.mainQuotaWriter : undefined;
return headers => {
if (credentialGeneration !== undefined && !isCodexAccountGenerationLive(accountId, credentialGeneration)) return;
applyCapturedCodexQuota(accountId, headers, writerGeneration, mainWriter, { modelId, poolWriter: authCtx.kind === "pool" ? authCtx.poolQuotaWriter : undefined,
poolResponse: authCtx.kind === "pool" });
};
}
export function preAuthUpstreamHostCircuitKey(
route: Pick<RouteResult, "provider" | "providerName" | "codexAccountMode" | "codexAccountId">,
config: OcxConfig,
options: { requireResponsesAdapter?: boolean } = {},
): string | null {
if (
normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) === 0
|| route.codexAccountMode !== "pool"
|| route.codexAccountId !== undefined
|| route.provider.authMode !== "forward"
|| (options.requireResponsesAdapter !== false && route.provider.adapter !== "openai-responses")
) return null;
return upstreamHostHealthKey(route.providerName, safeOriginLabel(route.provider.baseUrl ?? ""));
}
export function upstreamHostCircuitOpenResponse(retryAfterSeconds: number): Response {
return formatErrorResponse(
503,
"upstream_host_circuit_open",
"Provider host is temporarily unavailable",
{ retryAfter: String(retryAfterSeconds) },
);
}
export function normalizeCodexUnsupportedModelDetail(value: string): string {
return value.trim().replace(/\s+/gu, " ").toLocaleLowerCase("en-US");
}
/**
* The model id an authenticated Codex refusal names, or `undefined` when the body is not that
* refusal.
*
* Extracted rather than string-compared so the caller can learn WHICH model was refused. The
* route and the wire can legitimately disagree: `applyCodexAccountGatedWireNormalization`
* rewrites `gpt-daybreak-blue-latest` to `gpt-5.6-sol` before dispatch, so upstream names the
* model it was actually sent. Building the expected sentence from `route.modelId` alone made
* that comparison fail for the one model that is still account-gated, which silently disabled
* both the alternate-account retry and the same-account ladder built for exactly that case.
*
* Accept the HTTP `detail` envelope and the `error.message` envelope emitted by the
* WebSocket refused-create projection. Both must match the whole sentence, whitespace-collapsed
* and case-folded, with nothing before or after it. Competing envelopes are ambiguous. No prose is
* inferred and no other 400 shape is admitted, because a 400 is also what a malformed request
* earns and that must never read as an entitlement fact.
*/
export function codexUnsupportedModelFromDetail(
status: number,
bodyText: string,
): string | undefined {
if (status !== 400) return undefined;
try {
const payload = JSON.parse(bodyText) as unknown;
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined;
const record = payload as Record<string, unknown>;
const hasDetail = Object.hasOwn(record, "detail");
const hasError = Object.hasOwn(record, "error");
if (hasDetail === hasError) return undefined;
let detail: unknown = record.detail;
if (hasError) {
const error = record.error;
if (!error || typeof error !== "object" || Array.isArray(error)) return undefined;
const fields = error as Record<string, unknown>;
if ([fields.type, fields.code].some(value => value != null && typeof value !== "string")) return undefined;
detail = fields.message;
}
if (typeof detail !== "string") return undefined;
const matched = /^the '([^']{1,256})' model is not supported when using codex with a chatgpt account\.$/u
.exec(normalizeCodexUnsupportedModelDetail(detail));
return matched?.[1];
} catch {
return undefined;
}
}
/**
* The refused model id when this response is the exact unsupported-model refusal for this
* request, read from a bounded clone.
*
* Same admission rules {@link shouldRetryCodexPoolAccountModel400} always applied, which is now
* a predicate over this: a truncated or non-display-safe body proves nothing and is refused.
* Returning the id lets the caller record the denial against the model upstream actually named.
*/
export async function codexPoolAccountModel400Denial(
response: Response,
modelId: string,
signal?: AbortSignal,
wireModelId?: string,
): Promise<string | undefined> {
if (response.status !== 400) return undefined;
// A response that must not be sent again cannot open an alternate-account retry either. The
// reset helper marks the answer to a spent operator replacement this way, and that turn may
// already have run on the first send. Same rule as the quota and transient ladders below.
if (isNonReplayableResponse(response)) return undefined;
try {
const body = await readBoundedResponseBody(response.clone(), { signal });
if (!body.displaySafe || body.truncated) return undefined;
return isAllowListedCodexAccountModel400(response.status, body.text, modelId, wireModelId)
? codexUnsupportedModelFromDetail(response.status, body.text)
: undefined;
} catch {
return undefined;
}
}
export function isAllowListedCodexAccountModel400(
status: number,
bodyText: string,
modelId: string,
wireModelId?: string,
): boolean {
const refused = codexUnsupportedModelFromDetail(status, bodyText);
if (refused === undefined) return false;
return [modelId, wireModelId].some(candidate => (
candidate !== undefined
&& refused === normalizeCodexUnsupportedModelDetail(candidate)
));
}
export async function shouldRetryCodexPoolAccountModel400(
response: Response,
modelId: string,
signal?: AbortSignal,
wireModelId?: string,
): Promise<boolean> {
return await codexPoolAccountModel400Denial(response, modelId, signal, wireModelId) !== undefined;
}
/** Pre-stream quota/billing rejections that warrant one alternate-account attempt (#584). */
export function codexQuotaFailureMessage(body: string): string | undefined {
try {
const payload = JSON.parse(body) as unknown;
const canonical = upstreamErrorMessageFromPayload(payload);
if (canonical !== undefined) return canonical;
if (typeof payload === "string") return payload;
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined;
const record = payload as Record<string, unknown>;
if (typeof record.message === "string") return record.message;
return typeof record.error === "string" ? record.error : undefined;
} catch {
// Plain-text gateways remain supported. Valid JSON is inspected only at recognized
// message fields so echoed request content elsewhere cannot trigger account cooldown.
return body;
}
}
export async function shouldRetryCodexPoolAccountQuota(
response: Response,
signal?: AbortSignal,
): Promise<boolean> {
// A post-send WebSocket gateway status must not become a second account's send; the
// body carries no quota evidence either, but the marker is the contract, not the prose.
if (isNonReplayableResponse(response)) return false;
if (response.status === 402 || response.status === 429) {
// The response does not identify the organization or project whose quota was exhausted.
// Resolve the alternate before deciding whether its known workspace identity proves that an
// organization-scoped retry would be futile. Until then, preserve the broad #584 behaviour.
void signal;
return true;
}
if (response.status < 500 || response.status >= 600) return false;
try {
// Reject malformed UTF-8 instead of matching quota words around replacement characters.
const body = await readBoundedResponseBody(response.clone(), { signal, fatalUtf8: true });
const message = body.displaySafe && !body.truncated
? codexQuotaFailureMessage(body.text)
: undefined;
return message !== undefined
&& isRateLimitOrQuotaFailureMessage(message);
} catch {
return false;
}
}
export async function shouldRetryCodexScopedQuotaOnAlternate(
response: Response,
firstWorkspaceAccountId: string,
alternateWorkspaceAccountId: string | undefined,
signal?: AbortSignal,
): Promise<boolean> {
if (!firstWorkspaceAccountId || firstWorkspaceAccountId !== alternateWorkspaceAccountId) return true;
const { codexScopedExhaustionCode } = await import("../../codex/quota-rejection");
const code = await codexScopedExhaustionCode(response, { signal });
// Workspace identity binds organization-level limits, but the response supplies no project id.
return code === undefined || code === "project_spend_limit_exceeded";
}
/**
* A pre-stream upstream 5xx another Codex account may still be able to serve.
*
* `server_is_overloaded` is the shape this exists for. The ChatGPT backend refuses in a few
* hundred milliseconds, the body carries no quota evidence, and nothing in that exchange is
* account health — so the pool keeps choosing the same account and every request fails on it
* while the other accounts sit idle. That is what an operator sees as the pool refusing to move.
*
* The status stays exactly as upstream sent it. `classifyCodexUpstreamOutcome` maps 5xx to the
* transient class, so the account earns an ordinary failure streak and `upstreamFailoverThreshold`
* decides when it is soft-avoided, rather than a quota cooldown it never earned.
*
* Deliberately narrow. {@link isNonReplayableResponse} still refuses: a post-send WebSocket
* gateway status means the body already reached the origin, so sending it from a second account
* could duplicate a turn the origin may still be running. A 5xx whose body confirms quota is not
* routed here either — {@link shouldRetryCodexPoolAccountQuota} classifies that one first and
* carries the cooldown with it.
*/
export function shouldRetryCodexPoolAccountTransient(response: Response): boolean {
return !isNonReplayableResponse(response) && isTransientUpstreamStatus(response.status);
}
export interface CodexPoolAccountRetryArgs {
/** Sanitized caller input, before any selected Pool credential was materialized. */
callerAuthHeaders: Headers;
config: OcxConfig;
/** Actual routed result narrowed to the fields this retry consumes. */
route: Pick<RouteResult, "providerName" | "modelId" | "provider" | "staticPolicy">;
parsed: OcxParsedRequest;
logCtx: RequestLogContext;
options: {
admission?: DataPlaneAdmission;
codexAuthPolicy?: CodexAuthPolicyConfig;
visionDescribeTerminal?: boolean;
abortSignal?: AbortSignal;
onCodexAuthContextResolved?: (ctx: CodexAuthContext) => void;
deferCodexResetDerivedCooldown?: boolean;
// Narrowed subset of HandleResponsesOptions: the retry rebuilds the adapter, so it
// needs the inbound scope or the retry could land on a different wire than the
// first attempt.
inboundWire?: InboundWire;
codexWsRuntimeIdentity?: BunRuntimeGateInput;
translatorBudget: TranslatorBudget;
turnAdmissionLease?: AdmissionLease;
resolveCodexModelEntitlements?: typeof resolveCodexModelEntitlements;
/** The logical request's execution budget: the account move is its fourth send. */
sendBudget?: TransientSendBudget;
/** Root workflow this turn belongs to, so the move is charged there as well. */
workflowRootId?: string;
};
firstAuthCtx: Extract<CodexAuthContext, { kind: "pool" | "main-pool" }>;
firstResponse: Response;
outcomeStatus: number;
/**
* Forbid resolving a DIFFERENT account for this retry.
*
* Set when a stored Pool 401 already spent this logical request's account budget on its own
* refresh and replay. The same-account gated-model retry above stays available, because it
* sends to the account that was already paying; only the alternate-account resolution below is
* out of budget.
*/
sameAccountOnly?: boolean;
upstream: AbortController;
connectMs: number;
passthroughEstimate?: number;
stream: boolean;
/** Keep a buffered canonical client on HTTP/SSE after moving to another Pool account. */
httpOnly?: boolean;
onResponse?: (
response: Response,
authCtx: CodexAuthContext,
request: Awaited<ReturnType<ReturnType<typeof resolveAdapter>["buildRequest"]>>,
) => void;
}
export type CodexPoolAccountRetryResult =
| {
kind: "retried";
authCtx: CodexAuthContext;
request: Awaited<ReturnType<ReturnType<typeof resolveAdapter>["buildRequest"]>>;
upstreamResponse: Response;
selectedForwardHeaders: Headers;
}
| { kind: "no-alternate" }
| {
kind: "transport";
error: unknown;
authCtx: CodexAuthContext;
};
/** Keep retry-stage entitlement snapshots inside the native-main selection fence. */
export async function resolveCodexRetryModelEntitlements(
config: OcxConfig,
resolver: typeof resolveCodexModelEntitlements,
turnAdmissionLease?: AdmissionLease,
): Promise<Awaited<ReturnType<typeof resolveCodexModelEntitlements>>> {
// The initial auth selection has already released its admission before the first
// response arrives. Re-enter for every refresh so profile switching cannot overlap
// credential discovery, and omit main entirely when a drain or recovery owns it.
const selectionAdmission = codexAccountSelectionForTurn(turnAdmissionLease)?.();
const nativeMainReadsForbidden = isNativeMainTrafficBlocked()
|| selectionAdmission?.mainProfileDraining === true;
try {
return await resolver(config, {
excludeAccountIds: nativeMainReadsForbidden
? new Set([MAIN_CODEX_ACCOUNT_ID])
: undefined,
});
} finally {
selectionAdmission?.release();
}
}
export const CODEX_ACCOUNT_GATED_CANONICAL_WIRE_MODELS: ReadonlyMap<string, string> = new Map([
// The authenticated catalog currently advertises Daybreak Blue, while successful responses
// identify the serving model as gpt-5.6-sol. Sending the selector itself is shard-dependent:
// live traffic can receive the exact unsupported-model 400 repeatedly from the same entitled
// account. Keep Daybreak as the admission/catalog identity, but use the stable serving id on
// the credential-bearing wire after entitlement selection has completed.
["gpt-daybreak-blue-latest", "gpt-5.6-sol"],
]);
export function codexAccountGatedCanonicalWireModel(modelId: string): string | undefined {
const exact = CODEX_ACCOUNT_GATED_CANONICAL_WIRE_MODELS.get(modelId);
if (exact) return exact;
for (const [selector, wireModel] of CODEX_ACCOUNT_GATED_CANONICAL_WIRE_MODELS) {
if (slugsEquivalent(modelId, selector)) return wireModel;
}
return undefined;
}
export function applyCodexAccountGatedWireNormalization(parsed: OcxParsedRequest, route: RouteResult, logCtx?: RequestLogContext): void {
if (!isCanonicalOpenAiForwardProvider(route.provider)) return;
const wireModel = codexAccountGatedCanonicalWireModel(route.modelId);
if (!wireModel) return;
if (logCtx) {
logCtx.preserveResolvedModelFromRoute = true;
delete logCtx.resolvedModel;
logCtx.wireModel = wireModel;
}
parsed.modelId = wireModel;
if (!parsed._rawBody || typeof parsed._rawBody !== "object") return;
const raw = parsed._rawBody as Record<string, unknown>;
raw.model = wireModel;
// Daybreak's authenticated catalog does not advertise retention support, and the upstream
// rejects this optional Codex hint before model execution. Removing it preserves request
// semantics while avoiding an otherwise terminal pre-stream 400.
delete raw.prompt_cache_retention;
}
/**
* Workspace-denial evidence for a 403, read from the upstream body.
*
* #1789: a valid K12 credential gets 403 `codex_workspace_access_denied` on a routed prompt.
* Without this the account is quarantined for reauthentication, which cannot fix a workspace
* grant and loops forever. Fails closed: an unreadable body keeps the historical handling.
*/
export async function codexDenialOutcomeMeta(response: Response): Promise<{ denial?: "workspace" | "entitlement" }> {
if (response.status !== 403) return {};
const { classifyCodexPreStreamRejection } = await import("../../codex/quota-rejection");
const rejection = await classifyCodexPreStreamRejection(response);
return rejection.denial ? { denial: rejection.denial } : {};
}
export function codexQuotaOutcomeMeta(response: Response): {
retryAfter: string | null;
resetAt: string[];
} {
return {
retryAfter: response.headers.get("retry-after"),
resetAt: [
response.headers.get("x-codex-primary-reset-at"),
response.headers.get("x-codex-secondary-reset-at"),
response.headers.get("x-codex-tertiary-reset-at"),
].filter((value): value is string => !!value),
};
}
/**
* A reset timestamp describes a quota window, not an explicit instruction to
* stop using the whole account. A combo may therefore try a later model in the
* same request, while Retry-After and headerless quota failures remain blocking.
*/
export function shouldDeferCodexResetDerivedCooldown(response: Response, enabled?: boolean): boolean {
return enabled === true
&& (response.status === 429 || response.status === 402)
&& computeQuotaCooldown(codexQuotaOutcomeMeta(response)).source === "reset-derived";
}
/**
* One bounded alternate-account retry for Codex pool auth. Used for allow-listed
* model-400 and for pre-stream 429/402 quota failures (#584).
*/
export async function retryCodexPoolOnAlternateAccount(
args: CodexPoolAccountRetryArgs,
): Promise<CodexPoolAccountRetryResult> {
const {
callerAuthHeaders, config, route, parsed, logCtx, options, firstAuthCtx, firstResponse,
outcomeStatus, upstream, connectMs, passthroughEstimate, stream, httpOnly,
} = args;
const inboundWire = options.inboundWire ?? "responses";
const entitlementResolver = options.resolveCodexModelEntitlements ?? resolveCodexModelEntitlements;
let retryAuthCtx: CodexAuthContext | undefined;
// A transient 5xx must record even when this request cannot move: the ordinary terminal
// recorder only fires for an OK event-stream body, so a pre-stream refusal would otherwise
// leave the account looking healthy no matter how many times it refused, and the pool would
// keep handing it the next request.
const recordUnmovedTransientOutcome = (): void => {
if (!isTransientUpstreamStatus(outcomeStatus)) return;
recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, {
threadId: firstAuthCtx.affinityKey,
fixedAccount: firstAuthCtx.fixedAccount,
modelId: route.modelId,
probeLeaseId: codexProbeLeaseId(firstAuthCtx),
probeQuotaScope: codexProbeQuotaScope(firstAuthCtx),
transientProbe: codexTransientProbeGrant(firstAuthCtx),
writerGeneration: firstAuthCtx.writerGeneration,
});
};
// A body-confirmed quota response may arrive under HTTP 5xx. A path that returns the
// first response without a move must still record the NORMALIZED outcome: the ordinary
// terminal recorder sees only that wire status and would misclassify it as transient,
// leaving the exhausted account immediately selectable next turn.
const recordWrappedQuotaOutcome = (): void => {
if (outcomeStatus === firstResponse.status || (outcomeStatus !== 429 && outcomeStatus !== 402)) return;
recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, {
...codexQuotaOutcomeMeta(firstResponse),
threadId: firstAuthCtx.affinityKey,
modelId: route.modelId,
probeLeaseId: codexProbeLeaseId(firstAuthCtx),
probeQuotaScope: codexProbeQuotaScope(firstAuthCtx),
transientProbe: codexTransientProbeGrant(firstAuthCtx),
writerGeneration: firstAuthCtx.writerGeneration,
});
};
if (outcomeStatus === 400 && ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId)) {
invalidateCodexModelEntitlementsForAccount(firstAuthCtx.accountId);
let refreshed;
try {
refreshed = await resolveCodexRetryModelEntitlements(
config,
entitlementResolver,
options.turnAdmissionLease,
);
} catch (error) {
await firstResponse.body?.cancel().catch(() => undefined);
releaseCodexAuthContextProbeLease(firstAuthCtx);
throw error;
}
if (entitledCodexAccountIdsForModel(refreshed, route.modelId)?.has(firstAuthCtx.accountId)) {
// The authenticated roster still grants this exact model. Retry on the same account:
// upstream shards can briefly disagree during a gated-model rollout, but a pre-stream 400
// proves no output was committed and keeps this replay bounded.
retryAuthCtx = firstAuthCtx;
}
}
// Exact account selectors may retry the same confirmed account above, but must never resolve
// an alternate. Quota failures and a refreshed entitlement miss remain terminal.
if (!retryAuthCtx && (firstAuthCtx.fixedAccount || args.sameAccountOnly === true)) {
recordUnmovedTransientOutcome();
return { kind: "no-alternate" };
}
// An uploaded file is readable only by the account it was sent to, so NO alternate can serve
// this body. Which account would be chosen does not change that, which is why this asks before
// the resolution rather than after it: refusing here reserves no send, cancels no response, and
// leaves the caller holding the first account's rejection to return unchanged (#4710). The
// initial-dispatch sites answer with a 400 instead, because there is no earlier response there
// to fall back to. A same-account replay -- the gated-model 400 ladder above -- is unaffected,
// since it never leaves the issuing account.
if (!retryAuthCtx && conversationCarriesUploadedFiles(parsed._rawBody)) {
recordUnmovedTransientOutcome();
return { kind: "no-alternate" };
}
// An account move is the guarded profile's fourth send and draws the single shared
// final-recovery reserve. Nothing bounded it per request before: `excludeAccountId` excludes
// only the account that just failed, and the caller's recovery loop can return here after the
// alternate fails too, so one request could walk the pool an account at a time. The permit is
// consumed immediately before the physical send, so a resolution that finds no alternate
// costs nothing.
const executionBudget = isRequestExecutionBudget(args.options.sendBudget)
? args.options.sendBudget
: undefined;
let accountMovePermit: SingleUseDispatchPermit | undefined;
if (!retryAuthCtx && executionBudget) {
const decision = executionBudget.reserveDispatch({
sendClass: "account-failover",
targetKey: `${route.providerName}|${route.modelId}|alternate-account`,
});
if (!decision.allowed) {
recordUnmovedTransientOutcome();
return { kind: "no-alternate" };
}
accountMovePermit = decision.permit;
}
try {
retryAuthCtx ??= await resolveCodexAuthContext(
callerAuthHeaders,
config,
"pool",
{
excludeAccountId: firstAuthCtx.accountId,
admission: options.admission,
codexAuthPolicy: options.codexAuthPolicy,
modelId: route.modelId,
requestScopedMainCredential: hasForwardableCodexBearer(callerAuthHeaders, config),
beginCodexAccountSelection: codexAccountSelectionForTurn(options.turnAdmissionLease),
resolveCodexModelEntitlements: entitlementResolver,
},
);
} catch (error) {
const unexpectedRetryError =
!(error instanceof CodexPoolAuthenticationError)
&& !(error instanceof CodexAuthContextError)
&& !(error instanceof CodexAccountCooldownError)
&& !(error instanceof CodexMainProfileDrainingError);
if (unexpectedRetryError) {
// The reservation is the charge now, so an abandoned move has to hand its send back.
accountMovePermit?.release();
await firstResponse.body?.cancel().catch(() => undefined);
releaseCodexAuthContextProbeLease(firstAuthCtx);
throw error;
}
}
// A validated request-owned main bearer is a real alternate when the failed credential was a
// stored Pool account. It has no Pool account id to promote or cool, but it can own this one
// bounded replay. The resolver already refuses it when main itself is the excluded credential.
if (
retryAuthCtx?.kind !== "pool"
&& retryAuthCtx?.kind !== "main-pool"
&& retryAuthCtx?.kind !== "main"
) {
recordWrappedQuotaOutcome();
// No usable alternate was resolved, so the reserved move never becomes a send.
accountMovePermit?.release();
recordUnmovedTransientOutcome();
return { kind: "no-alternate" };
}
if (
(outcomeStatus === 429 || outcomeStatus === 402)
&& !await shouldRetryCodexScopedQuotaOnAlternate(
firstResponse,
firstAuthCtx.chatgptAccountId,
retryAuthCtx.kind === "pool" || retryAuthCtx.kind === "main-pool"
? retryAuthCtx.chatgptAccountId
// A request-owned `main` alternate has no stored account id; its workspace
// identity is what the caller's own credential materializes upstream.
: callerCodexWorkspaceAccountId(callerAuthHeaders),
options.abortSignal,
)
) {
// Suppressing the move is not suppressing the evidence: a same-workspace refusal
// still records its normalized quota outcome on the account that produced it.
recordWrappedQuotaOutcome();
accountMovePermit?.release();
releaseCodexAuthContextProbeLease(retryAuthCtx);
return { kind: "no-alternate" };
}
// The scope classification above reads the rejection body asynchronously, so the
// request may have been cancelled while it ran. Re-check before the send below
// mutates routing state or spends the alternate on a caller that is gone.
if (options.abortSignal?.aborted) {
recordWrappedQuotaOutcome();
recordUnmovedTransientOutcome();
accountMovePermit?.release();
releaseCodexAuthContextProbeLease(retryAuthCtx);
return { kind: "no-alternate" };
}
const quotaMeta = { ...codexQuotaOutcomeMeta(firstResponse), ...(await codexDenialOutcomeMeta(firstResponse)) };
if (outcomeStatus === 429 || outcomeStatus === 402) {
const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api");
applyAccountQuotaFromUpstreamHeaders(
firstAuthCtx.accountId,
firstResponse.headers,
firstAuthCtx.writerGeneration,
firstAuthCtx.kind === "main-pool" ? firstAuthCtx.mainQuotaWriter : undefined,
{ modelId: route.modelId, poolWriter: firstAuthCtx.kind === "pool" ? firstAuthCtx.poolQuotaWriter : undefined,
poolResponse: firstAuthCtx.kind === "pool" },
);
}
const deferFirstOutcome = shouldDeferCodexResetDerivedCooldown(
firstResponse,
options.deferCodexResetDerivedCooldown,
);
const recordFirstOutcome = (): void => {
recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, {
...quotaMeta,
threadId: firstAuthCtx.affinityKey,
modelId: route.modelId,
probeLeaseId: codexProbeLeaseId(firstAuthCtx),
probeQuotaScope: codexProbeQuotaScope(firstAuthCtx),
transientProbe: codexTransientProbeGrant(firstAuthCtx),
writerGeneration: firstAuthCtx.writerGeneration,
// Retry already advanced the RR ring via excludeAccountId — reuse for promotion.
...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}),
});
};
// Only a combo reset-derived outcome is deferred. Retry-After, defaults, and
// ordinary requests must block the first account before the alternate send.
if (!deferFirstOutcome) recordFirstOutcome();
const retryHeaders = headersForCodexAuthContext(callerAuthHeaders, retryAuthCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission);
const retryProvider = applyCodexAuthContextToProvider(
stripCodexRuntimeProviderFields(route.provider),
retryAuthCtx,
"pool",
);
const retryAdapter = resolveAdapter(
resolveWireProtocolOverride(route.providerName, route.modelId, retryProvider, inboundWire, route.staticPolicy),
config.cacheRetention,
route.providerName,
);
bindRouteReasoningReplayScope({
parsed,
providerName: route.providerName,
provider: retryProvider,
adapterName: retryAdapter.name,
codexAuthContext: retryAuthCtx,
forwardHeaders: retryHeaders,
});
{
const binding = conversationStateBindingFromAuth(
retryAuthCtx,
firstAuthCtx.kind === "pool" || firstAuthCtx.kind === "main-pool"
? firstAuthCtx.affinityKey
: undefined,
);
if (binding) {
applyAccountChangeConversationStateScrub({
body: parsed._rawBody,
parsed,
bindingKey: binding.bindingKey,
servingAccountId: binding.accountId,
priorAccountId: firstAuthCtx.accountId,
logCtx,
});
}
}
const request = await retryAdapter.buildRequest(parsed, {
headers: retryHeaders,
providerName: route.providerName,
translatorBudget: options.translatorBudget,
});
recordAdapterReasoning(logCtx, request);
recordAdapterTier(logCtx, request);
await firstResponse.body?.cancel().catch(() => undefined);
options.onCodexAuthContextResolved?.(retryAuthCtx);
route.provider = retryProvider;
logCtx.provider = formatCodexProviderForLog(
route.providerName,
retryAuthCtx.accountId,
config,
);
logCtx.accountLogLabel = codexAuthContextLogLabel(retryAuthCtx, config);
sealRequestAttemptIdentity(
logCtx.activeAttempt,
logCtx.provider,
retryAdapter.name,
logCtx.accountLogLabel,
);
recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, retryAdapter.name);
const retrySameConfirmedAccount = outcomeStatus === 400
&& ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId)
&& retryAuthCtx.accountId === firstAuthCtx.accountId;
// Live Daybreak traffic has produced long runs of unsupported-model 400s from different
// upstream shards even while the authenticated roster continues to grant the model. Permit
// seven additional same-account sends (eight total including the original), re-checking the
// exact allow-listed body and fresh entitlement before every later send. Alternate-account and
// quota recovery retain their historical one-send bound.
//
// Two different bounds, and the effective one is the smaller. `maxRetrySends` answers "how
// many times is it worth re-asking THIS account for a model its roster still grants"; the
// shared budget answers "how many times may this LOGICAL REQUEST reach upstream in total,
// across every layer that can re-send". A ladder of eight layered on sends the request had
// already made is exactly the per-request multiplication #4546 is about, so the ladder is
// capped at what the request has left. The floor of one keeps the single retry this function
// was called to make -- the move already paid for itself with its own permit -- and each rung
// past the first reserves its own send below, so a refusal stops the ladder with the last
// upstream answer intact.
// The ladder replays to the SAME account, so it must reserve under the same target key the
// other legs use. Folding the account id in made every rung read as a target change, which
// spent the one cross-account slot a real move needs on a same-account replay.
const ladderTargetKey = `${route.providerName}|${route.modelId}`;
// The ladder keeps its OWN bound rather than drawing on what the request has left. Clamping it
// to the shared total looked right and broke a working, pinned path: #2097 fixes this recovery
// at eight same-account dispatches (tests/server/server-auth.test.ts), and a request that has
// already spent sends would silently stop short of it. Reconciling an eight-send same-account
// ladder with a four-send request total is a policy decision, not a clamp to add in passing.
// What this diff does fix is that the rungs are now CHARGED instead of free.
const maxRetrySends = retrySameConfirmedAccount ? 7 : 1;
let retrySendCount = 0;
let upstreamResponse: Response;
try {
while (true) {
// The same-account gated-model 400 ladder below keeps its own `maxRetrySends` bound and
// does not take the reserve again; only the move itself does.
if (accountMovePermit) {
// The pool-wide recovery window is consulted BEFORE the request-local permit is used.
// `reserveDispatch` charges at reservation time and `release()` is the only way back, so
// using the permit first and refusing afterwards would spend a send the request never
// made. An account move is recovery traffic like any other: one request's own budget
// cannot see that a thousand other requests are moving at the same moment, which is
// precisely the amplification this window exists to bound (#4701).
//
// A refusal here is not a new failure mode: "no alternate was available" is already the
// outcome when the pool has nowhere to move this request to, and it is handled.
if (!classifyPoolRecoveryDispatch("retry").admitted) {
accountMovePermit.release();
accountMovePermit = undefined;
// The alternate context was resolved and will not send. Hand back whatever recovery
// lease it is holding rather than leaving that account unprobeable.
releaseCodexAuthContextProbeLease(retryAuthCtx);
recordUnmovedTransientOutcome();
return { kind: "no-alternate" };
}
const charged = accountMovePermit.use();
accountMovePermit = undefined;
if (!charged) {
releaseCodexAuthContextProbeLease(retryAuthCtx);
recordUnmovedTransientOutcome();
return { kind: "no-alternate" };
}
// The move is a physical send like any other, so the root workflow is charged too.
chargeWorkflowSends(args.options.workflowRootId, 1);
}
noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate);
try {
upstreamResponse = await fetchWithHeaderTimeout(
request.url,
{
method: request.method,
headers: request.headers,
body: request.body,
},
upstream.signal,
connectMs,
stream,
providerFetch(route.provider, options.codexWsRuntimeIdentity, {
httpOnly,
providerName: route.providerName,
modelId: route.modelId,
onCodexWsQuota: codexWsQuotaObserver(retryAuthCtx, route.provider, route.modelId),
beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider)
? createCodexReserveDispatchGuard(retryAuthCtx, options.codexAuthPolicy ?? config, route.modelId, options.admission, options.visionDescribeTerminal === true) : undefined,
}),
// Credential-bearing forward send: never follow a redirect into a
// dead-host rejection after the credential was seen (#914).
route.provider.authMode === "forward",
);
} catch (error) {
// Only the forward send is a transport boundary. Entitlement resolver throws below are
// deliberately outside this catch so programming errors retain their original path.
return { kind: "transport", error, authCtx: retryAuthCtx };
}
retrySendCount += 1;
args.onResponse?.(upstreamResponse, retryAuthCtx, request);
// The alternate account can refuse the same model, and that refusal is evidence about the
// account that produced it. Read BEFORE the ladder's own break, so the ordinary
// single-retry path -- every flagship model, which is the #4906 case -- records it too
// rather than only the gated ladder below. Without this the pool learns nothing from a
// refusal and the next request repeats the same selection.
const retryModelDenial = await codexPoolAccountModel400Denial(
upstreamResponse,
route.modelId,
options.abortSignal,
parsed.modelId,
);
if (retryModelDenial !== undefined) {
recordCodexModelDenialEvidence(
retryAuthCtx.accountId,
retryModelDenial,
retryAuthCtx.kind === "pool" ? retryAuthCtx.generation : undefined,
);
}
if (!retrySameConfirmedAccount || retrySendCount >= maxRetrySends) break;
// Caller-owned main is an alternate-account replay and can never enter the bounded
// same-stored-account 400 loop above. Keep that invariant explicit for the account-id reads.
if (retryAuthCtx.kind === "main") break;
if (retryModelDenial === undefined) break;
invalidateCodexModelEntitlementsForAccount(retryAuthCtx.accountId);
let refreshed: Awaited<ReturnType<typeof resolveCodexModelEntitlements>>;
try {
refreshed = await resolveCodexRetryModelEntitlements(
config,
entitlementResolver,
options.turnAdmissionLease,
);
} catch (error) {
await upstreamResponse.body?.cancel().catch(() => undefined);
await firstResponse.body?.cancel().catch(() => undefined);
releaseCodexAuthContextProbeLease(firstAuthCtx);
releaseCodexAuthContextProbeLease(retryAuthCtx);
throw error;
}
if (!entitledCodexAccountIdsForModel(refreshed, route.modelId)?.has(retryAuthCtx.accountId)) break;
// The next rung is another physical send of this logical request: a same-account,
// same-target replay, charged as an ordinary transient send rather than as a move.
// Reserved here, immediately before looping back, so a refusal stops the ladder with the
// last upstream 400 intact instead of spending a send it cannot make.
// Every rung is CHARGED, and a refusal does not end the ladder. That asymmetry is
// deliberate and it is the one place the shared cap yields. This is a same-account,
// same-target replay of a model-gating 400 whose own bound is eight dispatches, pinned by
// #2097; letting a spent request budget cut it to four would break a recovery that works
// today, which is precisely the mistake 040_send_budget.md warns a flat ceiling makes.
// The request total still governs everything that changes target or credential.
if (executionBudget) {
const rung = executionBudget.reserveDispatch({
sendClass: "transient",
targetKey: ladderTargetKey,
});
if (rung.allowed) rung.permit.use();
chargeWorkflowSends(args.options.workflowRootId, 1);
}
await upstreamResponse.body?.cancel().catch(() => undefined);
}
} finally {
request.releaseBodyObservation?.();
}
// A real HTTP response proves the host was reached (#914).
const retryHostKey = upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url));
if (normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0) {
resetUpstreamHostHealth(retryHostKey, null);
} else {
resetUpstreamHostHealth(retryHostKey);
}
if (deferFirstOutcome && upstreamResponse.ok) {
// Deferral keeps the first account eligible for a later combo model while an
// alternate attempt is still fallible. Commit its quota outcome only once the
// alternate account returns a successful HTTP response; otherwise the combo may
// still need the first account for its next target.
recordFirstOutcome();
}
return {
kind: "retried",
authCtx: retryAuthCtx,
request,
upstreamResponse,
selectedForwardHeaders: retryHeaders,
};
}
export function codexForwardTerminalOutcomeRecorder(
config: OcxConfig,
authCtx: CodexAuthContext,
provider: OcxProviderConfig,
modelId?: string,
logCtx?: RequestLogContext,
): ((status: ResponsesTerminalStatus, httpStatusOverride?: number) => void) | undefined {
if (!usesCodexForwardPoolAuth(authCtx, provider)) return undefined;
return (status, httpStatusOverride) => {
const quotaStatus = [httpStatusOverride, logCtx?.terminalHttpStatus]
.find(value => value === 429 || value === 402);
if (status === "incomplete" && quotaStatus === undefined) {
// Normal limit/content-filter/stall terminal — the account served the
// request. Don't penalize account health; record success to clear any
// prior soft-avoid so a healthy account isn't stuck avoided.
recordCodexUpstreamOutcome(config, authCtx.accountId, 200, {
threadId: authCtx.affinityKey,
fixedAccount: authCtx.fixedAccount,
modelId,
probeLeaseId: codexProbeLeaseId(authCtx),
probeQuotaScope: codexProbeQuotaScope(authCtx),
transientProbe: codexTransientProbeGrant(authCtx),
writerGeneration: authCtx.writerGeneration,
});
return;
}
// status === "completed" or "failed": use the semantic HTTP status derived
// from the terminal SSE error payload (httpStatusFromTerminalError in
// request-log inspection) instead of collapsing every non-completed terminal
// to 502. A 400 invalid_request_error must not soft-avoid the account or
// rebind threads — only genuine transport/5xx failures should trigger
// transient health recording.
// httpStatusOverride: the combo WS path inspects SSE payloads into the parent
// logCtx, but this recorder closes over the child logCtx. The caller passes
// the parent's terminalHttpStatus so the semantic status is not lost.
const outcome = status === "completed"
? 200
: (quotaStatus ?? httpStatusOverride ?? logCtx?.terminalHttpStatus ?? 502);
recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, {
threadId: authCtx.affinityKey,
fixedAccount: authCtx.fixedAccount,
modelId,
probeLeaseId: codexProbeLeaseId(authCtx),
probeQuotaScope: codexProbeQuotaScope(authCtx),
transientProbe: codexTransientProbeGrant(authCtx),
writerGeneration: authCtx.writerGeneration,
// A mid-stream terminal can carry a semantic 401 long after the credential was
// replaced. It is never replayed — the client already saw output — but it must
// not retire the replacement either (#2887).
...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}),
});
};
}