@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
1,019 lines • 51.9 kB
text/typescript
import { sealCredentialRefusal } from "./credential-recovery-attempt";
import { recordUpstreamProviderErrorDetail } from "./request-log-failure-detail";
import { normalizeNativeChatContent } from "../adapters/openai-chat/native-content";
import { attachComboFailureEvidence, ownedChatErrorPayload, type ClientErrorContract } from "../lib/client-error-body";
import type { FailureVerdictSink } from "../chat/failure-verdict";
import { normalizeChatChoiceErrors } from "./chat-native-sse";
import { captureAccountAutoPauseAttempt, wasAccountAutoPaused } from "../oauth/account-auto-pause";
import { createPhysicalSendReporter } from "../lib/request-execution-budget";
import { createInferenceSendBudget } from "./inference/context";
import { observeAdvisorNativeReasoning } from "./inference/advisor-diagnostics";
import { buildOpenAIChatPassthroughRequest, createOpenAIChatAdapter } from "../adapters/openai-chat";
import { affinityLaneFromRequest } from "./request-log-conversation";
import type { AdapterRequest, ProviderAdapter } from "../adapters/base";
import { isNativeChatRouteEligible } from "./chat-native-eligibility";
import {
chatCompletionsErrorBody,
ChatCompletionsCollectorSource,
chatCompletionsErrorResponse,
collectChatCompletion,
isChatCompletionsStreamError,
} from "../chat/outbound";
import { applyChatEffortCap, chatCollabSurface, effortCapAppliesTo, resolvePinnedEffort, supportedLadderFor } from "./effort-policy";
import { mapReasoningEffort } from "../reasoning-effort";
import {
classifyError,
cyberPolicyErrorType,
CYBER_POLICY_ERROR_CODE,
isCyberPolicyCode,
isCyberPolicyMessage,
SEND_BUDGET_EXHAUSTED_CODE,
UNDECLARED_TOOL_CALL_ERROR_CODE,
} from "../lib/errors";
import { reportDispatchSends, type RequestExecutionBudget, type SingleUseDispatchPermit } from "../lib/request-execution-budget";
import type { AdmissionLease } from "../lib/admission";
import { resolveContextPrincipal, type DataPlaneAdmission } from "./auth-cors";
import { readBoundedResponseBody } from "../lib/bounded-body";
import { redactSecretString } from "../lib/redact";
import { formatClientRetryAfter, resolveClientRetryAfter } from "../lib/retry-after";
import {
applyUpstreamRecoveryInit,
fetchWithResetRetry,
fetchWithTransientRetry,
isNonReplayableResponse, isSameTargetRetryHold,
isReplayRefusalCode,
isReplayRefusalResponse,
prepareSameTarget429Wait,
REPLAY_REFUSAL_CLIENT_HEADERS,
REPLAY_REFUSED_STATUS,
retainReplayRefusal,
SendBudgetExhaustedError, errorDestinationLabel,
TRANSIENT_RETRY_MAX_ATTEMPTS,
UpstreamRetryEvidenceError,
type UpstreamSendRecovery,
UPSTREAM_RESET_REPLAY_REFUSED_CODE,
wrapWithZeroOutputRefetch,
} from "../lib/upstream-retry";
import { isTranslatorBudgetExceededError, type TranslatorBudget } from "../lib/translator-budget";
import { authorizeResendForRecovery } from "../lib/request-resend-gate";
import { ambiguousResendAllowanceFor, selfContainedChatBody } from "./responses/reset-replay";
import {
hasKeyPoolFailover,
selectProactiveApiKeyTransport,
rateLimitRetryDelayMs,
rateLimitRetryPolicyFor,
rotateProviderTransportOn429,
rotateProviderTransportOn401,
transientRetryPolicyFor,
} from "../providers/key-failover";
import { fastPolicyForModel } from "../providers/service-tier";
import { RequestPacingQueueOverloadError } from "../providers/request-pacing";
import { stampApiKeyAccountLabel } from "../providers/label";
import { distributedKeyPhysicalFetch, distributedKeyResponseOwner } from "../providers/api-key-response-ownership";
import { passiveKeyPhysicalFetch } from "../providers/passive-key-observation";
import { isProviderApiKeyPaused, ProviderApiKeyPausedError } from "../providers/api-key-pause";
import { providerAffinityGenerationIsCurrent, providerApiKeySelectionIsCurrent, resolveCurrentProviderApiKeyTransport } from "../providers/api-key-selection";
import { isXaiGrokOAuth, isXaiLegacyPrivacyError, xaiPrivacyModeRefusal } from "../providers/xai-privacy-mode";
import type { OcxProviderTransport } from "../providers/xai-transport";
import type { RouteResult } from "../router";
import type { OcxConfig, OcxProviderConfig } from "../types";
import { fetchWithHeaderTimeout, physicalSendObserverInit, providerFetch, safeHostLabel, sendWithConnectionPolicy } from "./responses/fetch-helpers";
import { isLocalUpstream } from "../lib/local-upstream";
import { resolveConnectTimeoutMs, resolveJsonBodyReadTimeoutMs } from "../stall-timeout";
import { linkAbortSignal } from "./responses";
import {
noteProviderAttemptSend,
recordKeyWireAttemptUsage,
recordFirstOutput,
type RequestLogContext,
} from "./request-log";
import { recordKeyAttemptFailure } from "./request-log-key-failure";
import { chatResponseHasUndeclaredToolCalls, UNDECLARED_CHAT_TOOL_MESSAGE, jsonCompletionSse, nativeChatSse, structuredError, usageFromChat } from "./chat-native-sse";
import { beginInferenceAttempt, type InferenceAttempt } from "./inference/attempt";
import { createFinalRequestLog } from "./inference/final-log";
import { registerTurn, unregisterTurn } from "./lifecycle";
import { attachRequestSpendTracker } from "./responses/request-spend";
import { workflowRefusalResponse, workflowSendCeilingRefusalResponse } from "./workflow-refusal";
import type { ComboProtocolSource } from "./responses/core-combo-native";
import type { ProtocolEnvelope } from "../protocols/envelope";
import { protocolTraceSuppressed } from "../protocols/trace";
import { nativeChatReplayScope, bindNativeChatReplayScope, nativeChatReasoningCapture } from "./chat-native-reasoning";
import { resolveMistralTransport } from "../providers/mistral-transport";
export { isNativeChatRouteEligible, nativeChatDeclineReason } from "./chat-native-eligibility";
type Rec = Record<string, unknown>;
const MAX_NATIVE_CHAT_JSON_BYTES = 32 * 1024 * 1024;
const MAX_NATIVE_CHAT_ERROR_BYTES = 64 * 1024;
class NativeChatSpendRefusal extends Error {}
const chatEffortSnapshots = new WeakMap<Rec, {
inputModel: string;
providerName: string;
modelId: string;
present: boolean;
value: unknown;
annotation: string | undefined;
}>();
function normalizePinnedChatEffort(options: HandleNativeChatOptions): void {
const { chatBody, route, config, req, logCtx, requestedModel } = options;
let snapshot = chatEffortSnapshots.get(chatBody);
const inputModel = typeof chatBody.model === "string" ? chatBody.model : requestedModel;
let selector = inputModel;
if (snapshot) {
if (snapshot.providerName === route.providerName && snapshot.modelId === route.modelId) {
logCtx.requestedEffort = snapshot.annotation;
return;
}
if (snapshot.present) chatBody.reasoning_effort = snapshot.value;
else delete chatBody.reasoning_effort;
if (selector === snapshot.inputModel || selector === snapshot.modelId) {
selector = `${route.providerName}/${route.modelId}`;
}
} else {
snapshot = {
inputModel,
providerName: route.providerName,
modelId: route.modelId,
present: Object.hasOwn(chatBody, "reasoning_effort"),
value: chatBody.reasoning_effort,
annotation: undefined,
};
chatEffortSnapshots.set(chatBody, snapshot);
}
snapshot.inputModel = inputModel;
snapshot.providerName = route.providerName;
snapshot.modelId = route.modelId;
const from = typeof chatBody.reasoning_effort === "string" ? chatBody.reasoning_effort : undefined;
logCtx.requestedEffort = from;
// Compaction is normally excluded by native-route eligibility; preserve that boundary here too.
const compaction = chatBody.compaction_trigger !== undefined;
const pinned = !compaction
? resolvePinnedEffort(route, selector, config)
: undefined;
let normalizeForWire = false;
if (pinned !== undefined) {
logCtx.requestedEffort = from ? `${from}->${pinned}` : pinned;
if (pinned === "none") delete chatBody.reasoning_effort;
else chatBody.reasoning_effort = pinned;
normalizeForWire = true;
}
// A qualifying turn's ceiling is independent of whether an operator pin resolved.
if (effortCapAppliesTo(chatCollabSurface(chatBody), req.headers, config, compaction)) {
const capped = applyChatEffortCap(chatBody, req.headers, config, supportedLadderFor(route));
if (capped) {
logCtx.requestedEffort = `${logCtx.requestedEffort ?? capped.from}->${capped.to}`;
normalizeForWire = true;
}
}
// Normalize operator-pinned values and cap rewrites; otherwise preserve caller spelling.
if (normalizeForWire) {
const effort = typeof chatBody.reasoning_effort === "string" ? chatBody.reasoning_effort : undefined;
const wireEffort = mapReasoningEffort(route.provider, route.modelId, effort);
if (wireEffort === undefined) delete chatBody.reasoning_effort;
else chatBody.reasoning_effort = wireEffort;
}
snapshot.annotation = logCtx.requestedEffort;
}
function isRec(value: unknown): value is Rec {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function chatCompletionJson(value: unknown): Rec | null {
if (!isRec(value) || !Array.isArray(value.choices) || value.choices.length === 0) return null;
return value;
}
export interface HandleNativeChatOptions {
req: Request;
config: OcxConfig;
logCtx: RequestLogContext;
logIds?: { requestId: string; start: number; turnAdmissionLease?: AdmissionLease; admission?: DataPlaneAdmission };
route: RouteResult;
chatBody: Rec;
requestedModel: string;
requestedStream: boolean;
translatorBudget: TranslatorBudget;
}
/** Records the outcome on whichever final row owns this attempt; the first call wins. */
export type NativeChatFinishLog = (
status: number,
message?: string,
closeReason?: "non_stream" | "terminal" | "client_cancel",
) => void;
export interface NativeChatExecution extends HandleNativeChatOptions {
finishLog: NativeChatFinishLog;
/**
* A combo child's per-target budget (PF-07). With it the attempt opens no spend tracker of its
* own: the combo's hop reservation already booked the first send on the request's shared
* counter, and every later physical send is reported to that counter, whose observer is the
* request's one spend tracker. A second tracker would book each send twice and, merged into
* the parent row, replace the one the final log settles.
*/
sendBudget?: RequestExecutionBudget;
comboDispatchPermit?: SingleUseDispatchPermit;
/** The shared observer records a Combo child's refusal on its parent's context. */
workflowRefusalContext?: Pick<RequestLogContext, "localTerminalReason">;
/** Replaces the request-relative first-output mark; a combo child records its own. */
onFirstOutput?: () => void;
/** The lease a streamed body holds; defaults to `logIds.turnAdmissionLease`. */
turnAdmissionLease?: AdmissionLease;
/**
* The request's resolved data-plane admission, when the caller carries it without `logIds`.
* A combo child runs on the parent's log row, so it receives the admitted identity here or
* its reasoning replay scope can never establish a caller principal (PF-07).
*/
admission?: DataPlaneAdmission;
}
/**
* The native Chat ingress: opens the attempt on the request's log context and owns the
* request's final log row, then runs the attempt.
*/
export async function handleNativeChatCompletions(options: HandleNativeChatOptions): Promise<Response> {
const { logCtx, logIds, route } = options;
if (!protocolTraceSuppressed(logCtx)) logCtx.inboundProtocol = "chat";
const attemptHandle = beginInferenceAttempt(logCtx, {
provider: route.providerName,
model: route.modelId,
adapter: "openai-chat",
});
attemptHandle.seal(logCtx.accountLogLabel);
const finalLog = createFinalRequestLog(logIds, logCtx);
const finishLog: NativeChatFinishLog = (status, message, closeReason = "non_stream") => {
if (finalLog.finished()) return;
// The failure text lands on the context before the row is written, and only once.
if (message) logCtx.upstreamError = redactSecretString(message).slice(0, 500);
finalLog.finish(status, { closeReason });
};
return runNativeChatAttempt({ ...options, finishLog }, attemptHandle);
}
/**
* The Chat source a combo hands its native children (PF-07). A child runs on the attempt the
* combo opened and reports through the combo's callbacks, so the final row stays the parent's.
*/
export function createNativeChatComboSource(input: {
req: Request;
config: OcxConfig;
envelope: ProtocolEnvelope;
requestedModel: string;
requestedStream: boolean;
translatorBudget: TranslatorBudget;
workflowRefusalContext?: Pick<RequestLogContext, "localTerminalReason">;
/** The request's data-plane admission; native children inherit it for replay scope. */
admission?: DataPlaneAdmission;
}): ComboProtocolSource {
return {
inbound: "chat",
envelope: input.envelope,
dispatchNativeChild: child => runNativeChatAttempt({
req: input.req,
config: input.config,
logCtx: child.childLog,
route: child.route,
chatBody: child.body,
requestedModel: input.requestedModel,
requestedStream: input.requestedStream,
translatorBudget: input.translatorBudget,
finishLog: child.finishLog,
sendBudget: child.sendBudget,
comboDispatchPermit: child.comboDispatchPermit,
workflowRefusalContext: input.workflowRefusalContext,
onFirstOutput: child.onFirstOutput,
...(child.turnAdmissionLease ? { turnAdmissionLease: child.turnAdmissionLease } : {}),
...(input.admission ? { admission: input.admission } : {}),
}, child.attemptHandle),
};
}
/**
* One native Chat attempt on an already-open attempt row: effort normalization, the send
* loop with key failover and 429 replay, relay and usage. Every outcome is reported through
* `execution.finishLog`, so the caller decides which final row it lands on.
*/
export async function runNativeChatAttempt(
execution: NativeChatExecution,
attemptHandle: InferenceAttempt,
): Promise<Response> {
const { req, config, logCtx, logIds, route, requestedModel, requestedStream, translatorBudget, finishLog } = execution;
const { sendBudget } = execution;
const clientDeclaredEmptyToolCatalog = Array.isArray(execution.chatBody.tools) && execution.chatBody.tools.length === 0;
const declaredLegacyFunctionNames = new Set<string>(clientDeclaredEmptyToolCatalog && Array.isArray(execution.chatBody.functions)
? execution.chatBody.functions.flatMap(fn => isRec(fn) && typeof fn.name === "string" && fn.name.length > 0 ? [fn.name] : [])
: []);
const { attempt } = attemptHandle;
let outputDelivered = false;
const notifyFirstOutput = execution.onFirstOutput
?? (logIds ? () => recordFirstOutput(logCtx, logIds.start) : undefined);
const onFirstOutput = () => { outputDelivered = true; notifyFirstOutput?.(); };
const failLocal = (status: number, message: string, type?: string, code?: string | null): Response => {
const safeMessage = redactSecretString(message);
finishLog(status, safeMessage);
return chatCompletionsErrorResponse(status, safeMessage, type, code);
};
const failUpstream = (status: number, message: string, type?: string, code?: string | null, contract?: ClientErrorContract): Response => {
const legacy = chatCompletionsErrorBody(status, redactSecretString(message), type, code);
finishLog(status, redactSecretString(message));
const error = legacy.error as { type: string; code: string | null };
const finalStatus = error.code === CYBER_POLICY_ERROR_CODE ? 400 : status;
return attachComboFailureEvidence(new Response(JSON.stringify(ownedChatErrorPayload({ status: finalStatus, type: error.type, code: error.code, contract,
evidence: { status, type, code: code ?? undefined, message } })),
{ status: finalStatus, headers: { "Content-Type": "application/json" } }), JSON.stringify(legacy));
};
normalizePinnedChatEffort(execution);
logCtx.requestedServiceTier = typeof execution.chatBody.service_tier === "string"
? execution.chatBody.service_tier
: undefined;
const upstream = new AbortController();
const cleanupAbort = linkAbortSignal(upstream, req.signal);
// nativeChatSse already owns the translated stream's async pull/cancel path.
// Bind the lease to that same controller and its terminal callbacks instead
// of adding another trackStreamLifetime wrapper (unsafe on bundled Bun#32111).
let streamTurnRegistered = false;
const transferTurnToStream = () => {
const lease = execution.turnAdmissionLease ?? logIds?.turnAdmissionLease;
if (!lease || typeof (lease as { bindAbortController?: unknown }).bindAbortController !== "function") return;
registerTurn(upstream, lease);
streamTurnRegistered = true;
};
const releaseStreamTurn = () => {
if (!streamTurnRegistered) return;
streamTurnRegistered = false;
unregisterTurn(upstream);
};
// Set only at the final send boundary; queued refresh can replace the destination.
let localUpstream = false;
// Native chat is its own entry path -- chat-completions.ts routes here directly and never
// through the Responses core -- so the pre-dispatch key preference is applied again here
// rather than inherited. Assigned before the adapter binds below, for the same reason it is
// assigned before the transport pin in core.ts.
// Transport variant: the bare picker answers with the persisted row, which for a built-in
// provider carries no adapter id or base URL until routedProviderConfig backfills it.
try {
const affinityLane = affinityLaneFromRequest(req, logCtx, config, execution.admission ?? logIds?.admission);
if (affinityLane) logCtx.affinityLane = affinityLane;
const chatMessages = Array.isArray(execution.chatBody.messages) ? execution.chatBody.messages : [];
const proactiveKeyProvider = selectProactiveApiKeyTransport(config, route.providerName, route.provider, undefined, Date.now(), { lane: affinityLane, model: route.modelId, log: logCtx, continuation: (chatMessages.at(-1) as { role?: unknown } | undefined)?.role === "tool" });
if (proactiveKeyProvider) route.provider = proactiveKeyProvider;
if (isProviderApiKeyPaused(route.provider)) throw new ProviderApiKeyPausedError();
} catch (error) {
cleanupAbort(); upstream.abort();
if (error instanceof ProviderApiKeyPausedError) {
return failLocal(403, error.message, "permission_error");
}
if (!(error instanceof RequestPacingQueueOverloadError)) throw error;
logCtx.terminalSource = "synthetic";
const refusal = failLocal(429, error.message, "rate_limit_error");
const advice = formatClientRetryAfter(error.retryAfterSeconds);
if (advice !== undefined) refusal.headers.set("Retry-After", advice);
return refusal;
}
// Key selection re-resolves the transport; a single-key route still needs canonical Mistral pacing.
let activeProvider: OcxProviderConfig = resolveMistralTransport(route.provider);
stampApiKeyAccountLabel(logCtx, route.providerName, activeProvider);
const spendTracker = sendBudget ? undefined : attachRequestSpendTracker(req, logCtx);
const physicalBudget = sendBudget ?? createInferenceSendBudget(req, logCtx);
let activeAdapter: ProviderAdapter = createOpenAIChatAdapter(activeProvider);
let activeRequest: AdapterRequest;
const replayScope = nativeChatReplayScope(req.headers,
resolveContextPrincipal(req, config, execution.admission ?? logIds?.admission));
let retainedRequestBytes = 0;
const releaseRetainedRequest = () => {
if (retainedRequestBytes === 0) return;
translatorBudget.releaseRetained(retainedRequestBytes, { kind: "request_copies" });
retainedRequestBytes = 0;
};
const retainRequest = (request: AdapterRequest) => {
const bytes = Buffer.byteLength(request.body);
translatorBudget.chargeRetained(bytes, { kind: "request_copies" });
retainedRequestBytes = bytes;
};
const buildActiveRequest = () => {
bindNativeChatReplayScope(replayScope, activeProvider, route.providerName, route.modelId);
const request = buildOpenAIChatPassthroughRequest(
activeProvider,
execution.chatBody,
route.modelId,
requestedStream,
fastPolicyForModel(activeProvider, route.modelId, route.providerName, "chat"),
config.fastMode,
route.providerName,
replayScope,
undefined,
translatorBudget,
);
observeAdvisorNativeReasoning(req.headers, logCtx, request.body);
return request;
};
try {
activeRequest = buildActiveRequest();
retainRequest(activeRequest);
} catch (error) {
releaseRetainedRequest();
cleanupAbort();
upstream.abort();
if (isTranslatorBudgetExceededError(error)) {
return failLocal(413, "request translation buffer exceeded the safe limit", "request_too_large", "translation_buffer_limit");
}
return failLocal(400, error instanceof Error ? error.message : String(error), "invalid_request_error");
}
// One inbound request owns one transient send allowance. Capture the policy before any
// key rotation so recovery cannot replace the ceiling along with the active credential.
const requestTransientPolicy = transientRetryPolicyFor(activeProvider);
let transientSendsUsed = 0;
// A Combo child's initial send is prepaid, not exempt from the shared ceiling. Only its
// owning permit exposes that one booking in addition to the still-unspent allowance, and only
// until this child's first physical send has started and settled the booking.
const sharedSendCap = requestTransientPolicy?.attempts ?? TRANSIENT_RETRY_MAX_ATTEMPTS;
let physicalSends = 0;
let initialSendStarted = false;
const remainingSharedSends = (): number => {
if (!sendBudget) return Number.POSITIVE_INFINITY;
const remaining = sendBudget.remainingBaseSends(sharedSendCap);
return Math.min(sharedSendCap, remaining + (!initialSendStarted && execution.comboDispatchPermit ? 1 : 0));
};
const remainingTransientSends = (): number => Math.min(
requestTransientPolicy
? Math.max(0, requestTransientPolicy.attempts - transientSendsUsed)
: Number.POSITIVE_INFINITY,
remainingSharedSends(),
);
const transientSendAvailable = (): boolean => remainingTransientSends() > 0;
/**
* The operator's replacement grant for THIS logical request, read per leg because
* `activeProvider` is reassigned by credential rotation inside the send loop.
*
* A combo child already holds the request's send ledger, so the grant comes from it and every
* ambiguous stage of the request draws on the same counter. A direct Chat request opens no such
* ledger -- it accounts through a spend tracker instead -- so it keeps the single grant locally,
* which is the same ceiling for the same one attempt.
*/
const requestIsSelfContained = (() => {
let memo: boolean | undefined;
return (): boolean => memo ??= selfContainedChatBody(execution.chatBody);
})();
let localAmbiguousResendSpent = false;
const claimAmbiguousResend = (limit: number): boolean => {
if (sendBudget?.claimAmbiguousResend) return sendBudget.claimAmbiguousResend(limit);
if (localAmbiguousResendSpent || limit <= 0) return false;
localAmbiguousResendSpent = true;
return true;
};
const ambiguousResend = () => ambiguousResendAllowanceFor(
activeProvider,
requestIsSelfContained,
claimAmbiguousResend,
);
// Non-enforced accounting for one physical send. Enforced spend is owned by the reporter.
const comboReceipt = Boolean(sendBudget && execution.comboDispatchPermit);
const settleNonEnforcedSend = (url: string): void => {
if (sendBudget) {
// Backstop for sends the helper cannot see coming (a reset replay). The first
// report settles the combo's booking; each later one is charged and booked.
if (physicalSends > 0 && sendBudget.remainingBaseSends(sharedSendCap) <= 0) {
throw new SendBudgetExhaustedError(errorDestinationLabel(url));
}
physicalSends += 1;
reportDispatchSends(sendBudget, 1, execution.comboDispatchPermit, true);
initialSendStarted = true;
} else if (!spendTracker?.charge({ workflowAlreadyCounted: true })) throw new NativeChatSpendRefusal();
};
let responseOwnsDistributedKey = true;
const send = async (
request: AdapterRequest,
recovery?: "rate-limit-429" | "key-401" | "key-429" | UpstreamSendRecovery,
singleShot = false,
): Promise<Response> => {
try {
// #2643: opted-in key-auth openai-chat providers retry pre-stream transient statuses on
// the native chat lane too; everyone else keeps reset-only semantics.
const remaining = remainingTransientSends();
if (sendBudget && remaining <= 0) throw new SendBudgetExhaustedError(errorDestinationLabel(request.url));
if (requestTransientPolicy && remaining <= 0) {
throw new Error("native Chat transient send budget exhausted before recovery dispatch");
}
const dispatch = (transportRecovery?: UpstreamSendRecovery) => {
const effectiveRecovery = transportRecovery
?? (recovery === "connection-reset" ? "connection-reset" : undefined);
return fetchWithHeaderTimeout(
request.url,
applyUpstreamRecoveryInit({
method: request.method,
headers: request.headers,
body: request.body,
}, effectiveRecovery),
upstream.signal,
0, // Final physical send owns the header clock after queued credential/destination refresh.
requestedStream,
providerFetch(activeProvider, undefined, {
providerName: route.providerName,
modelId: route.modelId,
onPhysicalDispatch: comboReceipt
? () => { if (!physicalBudget.spendEnforced) settleNonEnforcedSend(request.url); }
: undefined,
dispatchOverride: async (_input, init, execute, reacquirePacing) => {
for (let attempt = 0; !providerApiKeySelectionIsCurrent(config, route.providerName, activeProvider); attempt++) {
if (outputDelivered) throw new Error("Provider key selection is unavailable during the native Chat turn");
if (!providerAffinityGenerationIsCurrent(config, route.providerName, activeProvider)) throw new Error("Provider affinity selection is no longer available");
if (attempt >= 3) throw new Error("Provider key selection changed repeatedly before native Chat dispatch");
let current = resolveCurrentProviderApiKeyTransport(config, route.providerName, activeProvider);
if (!current && activeProvider.requestPacing?.enabled
&& (activeProvider.requestPacing.keys || activeProvider.requestPacing.keySelection === "round-robin")) {
const fresh = resolveCurrentProviderApiKeyTransport(config, route.providerName, { ...activeProvider, _apiKeyAttempt: undefined });
if (fresh) current = selectProactiveApiKeyTransport(config, route.providerName, fresh) ?? fresh;
}
if (!current || !isNativeChatRouteEligible({ ...route, provider: current }, execution.chatBody, config)) {
if (!current && isProviderApiKeyPaused(config.providers[route.providerName])) throw new ProviderApiKeyPausedError();
throw new Error("Provider key selection is no longer available for native Chat");
}
activeProvider = current;
stampApiKeyAccountLabel(logCtx, route.providerName, activeProvider);
activeAdapter = createOpenAIChatAdapter(current);
activeRequest.releaseBodyObservation?.();
releaseRetainedRequest();
activeRequest = buildActiveRequest();
try { retainRequest(activeRequest); }
catch (error) { activeRequest.releaseBodyObservation?.(); throw error; }
if (current.requestPacing?.enabled
&& (current.requestPacing.keys || current.requestPacing.keySelection === "round-robin")) {
if (!reacquirePacing) throw new Error("Refreshed native Chat dispatch requires pacing readmission");
await reacquirePacing(current, route.modelId);
}
}
// The retry closure may still hold a pre-pacing request. Replace its entire
// wire shape, not just Authorization, and retain transport recovery flags.
request = activeRequest;
const headers = new Headers(request.headers);
const encoding = new Headers(init.headers).get("accept-encoding");
if (!headers.has("accept-encoding") && encoding) headers.set("accept-encoding", encoding);
if (init.signal?.aborted) throw init.signal.reason;
const spendReport = createPhysicalSendReporter(physicalBudget, () => ({
poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel,
}), execution.comboDispatchPermit);
const observeAutoPause = captureAccountAutoPauseAttempt(route.providerName, activeProvider, { config, oauth: undefined }, headers);
let dispatched: Response;
try {
// Capture this request's spend policy at its first physical boundary, including
// observe-only starts; later configuration changes belong to later requests.
if (spendReport.beforeSend && !spendReport.beforeSend()) {
if ((logCtx.spendTracker as { refusals?: number } | undefined)?.refusals) throw new NativeChatSpendRefusal();
throw new SendBudgetExhaustedError();
}
// Enforced spend started this send at `beforeSend`. A Combo child's non-enforced
// send is booked by the physical-dispatch receipt instead, so a local refusal
// before the wire refunds the booking rather than charging it.
if (physicalBudget.spendEnforced) initialSendStarted = true;
else if (!comboReceipt) settleNonEnforcedSend(request.url);
// A reselected provider transport is still a physical send: the connection policy
// and manual-redirect ownership wrap the selected implementation (#4992).
const physicalFetch = ((input, physicalInit) => sendWithConnectionPolicy(
passiveKeyPhysicalFetch(distributedKeyPhysicalFetch((activeProvider as OcxProviderTransport).fetch ?? execute,
route.providerName, activeProvider, request.url), config, route.providerName, activeProvider),
String(input), physicalInit, { providerName: route.providerName, provider: activeProvider },
)) as typeof fetch;
localUpstream = isLocalUpstream(request.url);
dispatched = await fetchWithHeaderTimeout(
request.url,
applyUpstreamRecoveryInit({
...init, method: request.method, headers, body: request.body,
...physicalSendObserverInit(observeAutoPause, request.url, () => {
noteProviderAttemptSend(logCtx, route.providerName, activeProvider, logCtx.usageLogInputTokens, transportRecovery ?? recovery);
}),
}, transportRecovery),
init.signal ?? upstream.signal,
resolveConnectTimeoutMs(config.connectTimeoutMs, { localUpstream }),
requestedStream, physicalFetch,
);
} finally {
try { if (physicalBudget.spendEnforced) spendReport(1); }
finally { spendReport.close?.(); }
}
responseOwnsDistributedKey = !activeProvider._apiKeyAttempt?.distributed
|| distributedKeyResponseOwner(dispatched, config, route.providerName) !== null;
if (!dispatched.ok) await recordKeyAttemptFailure(logCtx, dispatched, init.signal ?? upstream.signal);
return observeAutoPause ? observeAutoPause(dispatched) : dispatched;
},
}),
);
};
// A replacement bought by an ambiguous reset is ONE send. Routing it back through a retry
// helper would let that helper spend sends the operator allowance never granted, which is
// the duplicated inference the allowance exists to bound.
if (singleShot) return await dispatch("connection-reset");
const fetchWithPolicy = requestTransientPolicy ? fetchWithTransientRetry : fetchWithResetRetry;
return await fetchWithPolicy(
dispatch,
{
abortSignal: upstream.signal,
label: safeHostLabel(request.url), destination: errorDestinationLabel(request.url),
...(requestTransientPolicy
? {
attempts: remaining,
onSendsConsumed: (sends: number) => { transientSendsUsed += Math.max(0, sends); },
}
: {}),
},
);
} finally {
request.releaseBodyObservation?.();
}
};
let response: Response;
try {
response = await send(activeRequest);
const retryPolicy = rateLimitRetryPolicyFor(activeProvider);
let retries = 0;
let autoPauseMoves = 0;
for (;;) {
const priorResponse = response;
for (; wasAccountAutoPaused(response) && config.providers[route.providerName]?.apiKey && autoPauseMoves < 3 && transientSendAvailable(); autoPauseMoves++) {
const fresh = resolveCurrentProviderApiKeyTransport(config, route.providerName, { ...activeProvider, _apiKeyAttempt: undefined });
const next = fresh && (selectProactiveApiKeyTransport(config, route.providerName, fresh) ?? fresh);
if (!next) break;
sealCredentialRefusal(logCtx, response.status, "key-401");
try { void response.body?.cancel().catch(() => {}); } catch { /* closed */ }
activeProvider = next;
stampApiKeyAccountLabel(logCtx, route.providerName, activeProvider);
activeAdapter = createOpenAIChatAdapter(activeProvider);
releaseRetainedRequest();
activeRequest = buildActiveRequest();
retainRequest(activeRequest);
response = await send(activeRequest, "key-401");
}
while (
response.status === 429
// A 429 this proxy synthesized for a refused reset replay is not a provider rate
// limit: waiting and re-sending here is exactly the duplicate inference the refusal
// exists to stop. It kept the same shape under the old 502 only because 502 never
// matched this branch.
&& !isNonReplayableResponse(response) && !isSameTargetRetryHold(response)
&& retryPolicy && providerAffinityGenerationIsCurrent(config, route.providerName, activeProvider)
&& retries < retryPolicy.attempts
&& transientSendAvailable()
) {
retries += 1;
for await (const _ of prepareSameTarget429Wait({
body: response.body,
signal: upstream.signal,
delayMs: rateLimitRetryDelayMs(retryPolicy, response.headers.get("retry-after"), Date.now()),
})) { /* pre-stream wait */ }
if (upstream.signal.aborted) throw upstream.signal.reason;
response = await send(activeRequest, "rate-limit-429");
}
// Same reason as above, plus a second one: rotating here would write a cooldown against
// a key that rate-limited nothing, and that false signal outlives the request.
while ((response.status === 429 || (response.status === 401 && activeProvider._apiKeyAttempt?.distributed))
&& responseOwnsDistributedKey && !isNonReplayableResponse(response) && hasKeyPoolFailover(activeProvider)) {
const failureStatus = response.status;
const rotate = failureStatus === 401 ? rotateProviderTransportOn401 : rotateProviderTransportOn429;
const rotated = rotate(config, route.providerName, activeProvider, {
retryAfter: response.headers.get("retry-after"),
now: Date.now(),
attemptedKey: activeProvider.apiKey,
attemptedSelection: activeProvider._apiKeyAttempt,
promptCacheKey: typeof execution.chatBody.prompt_cache_key === "string" ? execution.chatBody.prompt_cache_key : undefined,
});
if (!rotated) break;
// Rotation also records the failed key's cooldown and persists the next healthy key.
// Keep that bookkeeping when this request has spent its final send, but preserve the
// terminal 429 body and do not dispatch with the replacement credential.
if (!transientSendAvailable()) break;
const recovery = failureStatus === 401 ? "key-401" : "key-429";
sealCredentialRefusal(logCtx, failureStatus, recovery);
try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
activeProvider = rotated;
stampApiKeyAccountLabel(logCtx, route.providerName, activeProvider);
activeAdapter = createOpenAIChatAdapter(activeProvider);
releaseRetainedRequest();
activeRequest = buildActiveRequest();
retainRequest(activeRequest);
response = await send(activeRequest, recovery);
}
if (response !== priorResponse && wasAccountAutoPaused(response)) continue;
break;
}
} catch (error) {
releaseRetainedRequest();
cleanupAbort();
upstream.abort();
if (req.signal.aborted) return failLocal(499, "Client cancelled request", "client_cancelled");
const sendError = error instanceof UpstreamRetryEvidenceError ? error.cause : error;
if (sendError instanceof ProviderApiKeyPausedError) {
return failLocal(403, sendError.message, "permission_error");
}
if (sendError instanceof RequestPacingQueueOverloadError) {
logCtx.terminalSource = "synthetic";
const refusal = failLocal(429, sendError.message, "rate_limit_error");
const advice = formatClientRetryAfter(sendError.retryAfterSeconds);
if (advice !== undefined) refusal.headers.set("Retry-After", advice);
return refusal;
}
if (sendError instanceof SendBudgetExhaustedError) {
logCtx.terminalSource = "synthetic";
const refusal = workflowSendCeilingRefusalResponse(logCtx, req.headers.get("x-codex-parent-thread-id")?.trim() || undefined,
execution.workflowRefusalContext);
if (refusal) { finishLog(429); return refusal; }
// A decision this process made, answered as the Responses path answers it: 429, not 502.
return failLocal(429, sendError.message, SEND_BUDGET_EXHAUSTED_CODE, SEND_BUDGET_EXHAUSTED_CODE);
}
if (sendError instanceof NativeChatSpendRefusal) {
const refusal = workflowRefusalResponse(logCtx.errorCode === "workflow_sends_exhausted"
? "workflow-sends-exhausted" : "workflow-spend-exhausted", logCtx);
finishLog(429);
return refusal;
}
if (isTranslatorBudgetExceededError(error)) {
return failLocal(413, "request translation buffer exceeded the safe limit", "request_too_large", "translation_buffer_limit");
}
return failUpstream(502, error instanceof Error ? error.message : String(error), "server_error", undefined, "transport_failed");
}
releaseRetainedRequest();
if (!response.ok) {
let bodyText = "";
try {
const body = await readBoundedResponseBody(response, {
signal: upstream.signal,
maxBytes: MAX_NATIVE_CHAT_ERROR_BYTES,
});
if (body.displaySafe) bodyText = body.text;
} catch { /* status-only fallback */ }
cleanupAbort();
if (req.signal.aborted) {
upstream.abort();
return failLocal(499, "Client cancelled request", "client_cancelled");
}
if (isXaiGrokOAuth(route.providerName, route.provider) && isXaiLegacyPrivacyError(response.status, bodyText)) {
const refusal = xaiPrivacyModeRefusal(logCtx);
finishLog(refusal.status);
return refusal;
}
const detail = activeAdapter.formatErrorBody?.(response.status, response.headers, bodyText) ?? "";
let upstreamType: string | undefined;
let upstreamCode: string | null | undefined;
let upstreamMessage: string | undefined;
try {
const parsedError = JSON.parse(bodyText) as Rec;
recordUpstreamProviderErrorDetail(logCtx, response.status, parsedError);
const nested = isRec(parsedError.error) ? parsedError.error : undefined;
const details = nested ?? parsedError;
if (typeof details.type === "string") upstreamType = details.type;
if (details.code === null || typeof details.code === "string") upstreamCode = details.code;
const rawMessage = typeof details.message === "string"
? details.message
: typeof parsedError.error === "string" ? parsedError.error : undefined;
if (rawMessage?.trim()) {
upstreamMessage = redactSecretString(rawMessage.trim());
}
} catch { /* keep generic classification */ }
const message = upstreamMessage
&& (isCyberPolicyCode(upstreamCode) || isCyberPolicyMessage(upstreamMessage))
? upstreamMessage
: detail ? `Provider error ${response.status}: ${detail}` : `Provider error ${response.status}`;
const classified = classifyError(
response.status,
upstreamType ?? (response.status === 401 ? "authentication_error"
: response.status === 429 ? "rate_limit_error"
: response.status >= 500 ? "server_error" : "invalid_request_error"),
message,
);
// The verdict is read once, from the response that carries it and from the code a
// re-wrapped body kept -- never from the status, which a real rate limit shares.
const replayRefusal = isReplayRefusalResponse(response) || isReplayRefusalCode(upstreamCode);
if (isCyberPolicyCode(upstreamCode) || classified.code === CYBER_POLICY_ERROR_CODE) {
classified.code = CYBER_POLICY_ERROR_CODE;
classified.type = cyberPolicyErrorType(upstreamType);
} else if (replayRefusal) {
// 429 classifies as a rate limit and a rate limit already carries a code, so the branch
// below -- which only fills an EMPTY code -- could never restore this one. Without it the
// client is told the provider throttled the turn, when what happened is that this proxy
// declined to send it a second time.
classified.code = UPSTREAM_RESET_REPLAY_REFUSED_CODE;
} else if (upstreamCode === "model_not_found") {
classified.code = "model_not_found";
classified.type = "invalid_request_error";
} else if (upstreamCode !== undefined && upstreamCode !== null && classified.code == null) {
classified.code = upstreamCode;
}
const status = isCyberPolicyCode(classified.code) ? 400
: replayRefusal ? REPLAY_REFUSED_STATUS
: response.status;
// A refusal this proxy made has no wait to report. Synthesizing one here would hand the
// client the default two-second retry for a rate limit that never happened, which is the
// duplicate send the refusal exists to prevent.
const retryAfter = isCyberPolicyCode(classified.code) || replayRefusal
? undefined
: resolveClientRetryAfter({
status: response.status,
message: classified.message,
upstreamRetryAfter: response.headers.get("retry-after"),
});
finishLog(status, classified.message);
const rewritten = new Response(JSON.stringify(ownedChatErrorPayload({ status, type: classified.type, code: classified.code,
evidence: { status: response.status, type: upstreamType, code: upstreamCode ?? undefined, message: upstreamMessage ?? detail },
localReason: wasAccountAutoPaused(response) ? "credentials_paused" : undefined })), {
status,
headers: {
"Content-Type": "application/json",
...(formatClientRetryAfter(retryAfter) ? { "Retry-After": formatClientRetryAfter(retryAfter)! } : {}),
...(replayRefusal ? REPLAY_REFUSAL_CLIENT_HEADERS : {}),
},
});
attachComboFailureEvidence(rewritten, JSON.stringify(chatCompletionsErrorBody(status, classified.message, classified.type, classified.code)));
return replayRefusal ? retainReplayRefusal(rewritten) : rewritten;
}
const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
const reasoningCapture = nativeChatReasoningCapture(activeProvider, route.modelId, replayScope, translatorBudget);
if (contentType.includes("text/event-stream") && response.body) {
if (requestedStream) transferTurnToStream();
let terminalStatus: number | undefined;
const resilientBody = wrapWithZeroOutputRefetch(
response.body,
// One physical send, not one more trip through a retry helper: the allowance below buys a
// single replacement, so the send must not be able to spend more than it granted.
async () => {
try {
return await send(activeRequest, "connection-reset", true);
} finally {
// Reselection can charge the rebuilt request after the initial send settled.
releaseRetainedRequest();
}
},
{
abortSignal: upstream.signal,
label: safeHostLabel(activeRequest.url), destination: errorDestinationLabel(activeRequest.url),
attempts: remainingTransientSends(),
// The origin returned a head and may already be running the turn, so this is the
// ambiguous row of the stage table. Only the operator allowance the Responses stream
// also draws on can authorise it; without one the original failure stands.
authorize: () => authorizeResendForRecovery("headers-only", "connection-reset", ambiguousResend()).allowed,
// A replacement must satisfy the contract already promised to the client. A 200 that is
// not an event stream would reach the SSE translator as a non-SSE body and surface as a
// malformed-stream error instead of the reset that actually happened.
acceptResponse: replacement =>
(replacement.headers.get("content-type") ?? "").toLowerCase().includes("text/event-stream"),
},
);
const failureVerdictSink: FailureVerdictSink = {};
const collectorSource = requestedStream ? undefined : new ChatCompletionsCollectorSource(translatorBudget);
const stream = nativeChatSse(resilientBody, {
provider: activeProvider,
failureVerdictSink,
collectorSource,
requestedModel,
translatorBudget,
signal: upstream.signal,
stallTimeoutSec: config.stallTimeoutSec,
localUpstream,
clientDeclaredEmptyToolCatalog,
declaredLegacyFunctionNames,
onFirstOutput,
onChunk: reasoningCapture.observe,
onUsage: usage => {
if (!recordKeyWireAttemptUsage(logCtx, usage)) {
logCtx.usage = usage;
attempt.usage = usage;
}
},
onTerminal: (status: number, message?: string) => {
reasoningCapture.finish(status < 400);
terminalStatus = status;
if (!requestedStream) return;
try {
cleanupAbort();
finishLog(status, message, "terminal");
if (status >= 400) upstream.abort();
} finally {
releaseStreamTurn();
}
},
onCancel: () => {
reasoningCapture.finish(false);
if (requestedStream) {
try {
cleanupAbort();
upstream.abort();
finishLog(499, undefined, "client_cancel");
} finally {
releaseStreamTurn();
}
}
},
});
if (requestedStream) {
return new Response(stream, {
status: 200,
headers: {
"Content-Type": "text/event-stream; charset=utf-8",
"Cache-Control": "no-cache",
Connection: "keep-alive",
},
});
}
try {
const completion = await collectChatCompletion(stream, requestedModel, translatorBudget, failureVerdictSink, collectorSource);
cleanupAbort();
// A cancelled native relay closes its downstream body. EOF alone must not
// promote the buffered prefix to a successful Chat completion. A terminal
// already accepted by the relay retains precedence over a later abort.
if (req.signal.aborted && terminalStatus === undefined) {
return failLocal(499, "Client cancelled request", "client_cancelled");
}
finishLog(200);
return Response.json(completion);
} catch (error) {
cleanupAbort();
upstream.abort();
if (req.signal.aborted && terminalStatus === undefined) {
return failLocal(499, "Client cancelled request", "client_cancelled");
}
if (isChatCompletionsStreamError(error)) {
return failUpstream(error.status, error.message, error.type, error.code);
}
return failUpstream(502, error instanceof Error ? error.message : String(error), "upstream_error", undefined, "transport_failed");
}
}
let body;
try {
const jsonBodyReadMs = resolveJsonBodyReadTimeoutMs(config.stallTimeoutSec, { localUpstream }, config.connectTimeoutMs);
body = await readBoundedResponseBody(response, {
signal: upstream.signal,
maxBytes: MAX_NATIVE_CHAT_JSON_BYTES,
totalTimeoutMs: localUpstream ? 0 : jsonBodyReadMs,
inactivityTimeoutMs: jsonBodyReadMs,
});
} catch (error) {
cleanupAbort();
upstream.abort();
if (req.signal.aborted) return failLocal(499, "Client cancelled request", "client_cancelled");
return failUpstream(502, error instanceof Error ? error.message : String(error), "upstream_error", undefined, "transport_failed");
}
cleanupAbort();
if (body.oversized) {
upstream.abort();
return failUpstream(502, "upstream response exceeded the safe limit", "upstream_error", "translation_buffer_limit");
}
let parsedJson: unknown;
try {
parsedJson = JSON.parse(body.text);
} catch {
return failUpstream(502, "upstream returned malformed Chat Completions JSON", "upstream_error");
}
const usage = isRec(parsedJson) ? usageFromChat(parsedJson.usage) : undefined;
if (usage) {
if (!recordKeyWireAttemptUsage(logCtx, usage)) {
logCtx.usage = usage;
attempt.usage = usage;
}
}
if (isRec(parsedJson)) parsedJson = normalizeNativeChatContent(parsedJson, activeProvider);
const error = structuredError(parsedJson);
if (error) return failUpstream(error.status ?? 502, error.message, error.type, error.code);
const originalCompletion = chatCompletionJson(parsedJson);
const completion = originalCompletion ? normalizeChatChoiceErrors(originalCompletion) : null;
if (!completion) return failUpstream(502, "upstream response contained no choices", "upstream_error");
reasoningCapture.json(completion);
if (clientDeclaredEmptyToolCatalog && chatResponseHasUndeclaredToolCalls(completion, declaredLegacyFunctionNames)) {
return failUpstream(502, UNDECLARED_CHAT_TOOL_MESSAGE, "upstream_error", UNDECLARED_TOOL_CALL_ERROR_CODE);
}
onFirstOutput?.();
try {
const serialized = requestedStream
? jsonCompletionSse(completion, requestedModel, translatorBudget)
: JSON.stringify(completion);
if (!requestedStream) translatorBudget.chargeRetained(Buffer.byteLength(serialized) * 2, { kind: "live_transient" });
finishLog(200);
return new Response(serialized, {
status: 200,
headers: requestedStream
? { "Content-Type": "text/event-stream; charset=utf-8", "Cache-Control": "no-cache" }
: { "Content-Type": "application/json" },
});
} catch (error) {
if (isTranslatorBudgetExceededError(error)) {
return failUpstream(502, "upstream translation buffer exceeded the safe limit", "upstream_error", "translation_buffer_limit");
}
throw error;
}
}