@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
451 lines (442 loc) • 25 kB
text/typescript
import { proxyErrorPresentation, isProxyError, rememberProxyErrorMessage } from "../../lib/client-error-body";
import { responsesTimeoutProvider, type ResponsesRequestContext } from "./core-options";
import type { PreparedResponsesRequest } from "./request-prepare";
import type { ResponsesTransport } from "./request-transport";
import type { ResponsesSidecarAuth } from "./request-sidecar-auth";
import type { ResponsesEffects } from "./response-effects";
import type { ResponsesCompletionPolicy } from "./completion-policy";
import type { AdapterExchange } from "./adapter-dispatch";
import type { AdapterContinuations } from "./adapter-continuation";
import { guardTerminalEventStream } from "./terminal-guard";
import { guardEmptyCompletionEventStream, mergeUsage } from "./empty-completion-guard";
import { UNDECLARED_TOOL_CALL_ERROR_CODE } from "../../lib/errors";
import { bridgeToResponsesSSE, buildResponseJSON, formatErrorResponse } from "../../bridge";
import type { OcxProviderContinuationState, AdapterEvent, OcxUsage } from "../../types";
import { rememberResponseState } from "../../responses/state";
import { trackStreamLifetime } from "../lifecycle";
import { awaitThoughtSignatureDurability } from "../../responses/thought-signature-replay";
import { adapterResponseReachedServingTerminal } from "./core-replay";
import {
readResponseBodyWithInactivity,
readResponseStreamWithInactivity,
ResponseBodyInactivityError,
} from "../../lib/response-body-inactivity";
import { providerStallTimeoutSec, resolveStallTimeoutMs } from "../../stall-timeout";
import { clientEncoderForDelivery, deliverClientEncodedResponse } from "../inference/client-encoder-delivery";
import { noteKiroServedSuccess } from "../../providers/kiro-usage";
import { persistKiroAccountState } from "../../providers/kiro-account-state-disk";
import { readDisplaySafeErrorText, normalizeUpstreamErrorText } from "./core-errors";
import { sanitizeNonReplayableUpstreamError } from "./non-replayable-error";
import { isNonReplayableResponse } from "../../lib/upstream-retry";
import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
import { currentTurnToolBridgeMaps, guardEmptyToolCatalogEvents, guardEmptyToolCatalogStream } from "../../lib/empty-tool-catalog";
import { guardCompactionSummary } from "../../responses/compaction-summary";
import { settleAnthropicPerMessageEffort } from "../../adapters/anthropic/per-message-effort";
import { adapterResponseSpendObserver } from "./request-spend";
/** One responsibility of the Responses request pipeline; state owners are explicit. */
export async function deliverAdapterResponse(
requestContext: Pick<ResponsesRequestContext, "logCtx" | "options" | "config">,
requestState: Pick<
PreparedResponsesRequest,
| "parsed"
| "route"
| "translatorBudget"
| "toolBridgeMaps"
| "clientToolCatalogRestriction"
| "rememberKiroDeliveredFinalAnswer"
| "responseStateOptions"
| "turnMembership"
| "completedToolBlocks"
>,
transportState: Pick<ResponsesTransport, "activeAdapter" | "bindKeyUsageFromBridge" | "sentOAuthSnapshot" | "trackGuardedOAuthAttemptUsage">,
sidecarState: Pick<ResponsesSidecarAuth, "routedCompaction">,
responseEffects: Pick<
ResponsesEffects,
| "cancelResponseCompletion"
| "commitReasoningReplayServingRoute"
| "continuationStateForResponse"
| "notifyResponseComplete"
>,
completionPolicy: Pick<ResponsesCompletionPolicy, "emptyCompletionGuardEnabled">,
adapterExchange: Pick<AdapterExchange, "upstreamResponse" | "upstream" | "cleanupUpstreamAbort" | "localUpstream" | "thinkingSignatureRecovery" | "refetchThinkingSignature">,
continuationState: Pick<AdapterContinuations, "terminalGuardEnabled" | "fetchTerminalGuardContinuation" | "fetchGuardedEmptyCompletionRetry" | "isContinuationBodyStall">,
): Promise<Response> {
const { logCtx, options, config } = requestContext;
const {
parsed,
route,
translatorBudget,
toolBridgeMaps,
rememberKiroDeliveredFinalAnswer,
responseStateOptions,
} = requestState;
const { upstreamResponse, upstream, cleanupUpstreamAbort, localUpstream, thinkingSignatureRecovery, refetchThinkingSignature } = adapterExchange;
const {
terminalGuardEnabled,
fetchTerminalGuardContinuation,
fetchGuardedEmptyCompletionRetry,
isContinuationBodyStall,
} = continuationState;
const preserveEmptyCompletionRetryError = (event: AdapterEvent) =>
(event.type === "error" && event.code === UNDECLARED_TOOL_CALL_ERROR_CODE) || isContinuationBodyStall(event);
const { emptyCompletionGuardEnabled } = completionPolicy;
const {
cancelResponseCompletion,
commitReasoningReplayServingRoute,
continuationStateForResponse,
notifyResponseComplete,
} = responseEffects;
const { routedCompaction } = sidecarState;
const stallTimeoutSec = () => providerStallTimeoutSec(config, responsesTimeoutProvider(options, route.provider));
const bodyInactivityMs = resolveStallTimeoutMs(stallTimeoutSec(), { localUpstream });
const upstreamRequestsStream = parsed.stream || isCanonicalOpenAiForwardProvider(route.provider);
const observeInitialSpend = adapterResponseSpendObserver(upstreamResponse);
const recoverThinkingSignature = (events: AsyncIterable<AdapterEvent>): AsyncIterable<AdapterEvent> =>
thinkingSignatureRecovery.wrap(events, async function* () {
const result = await refetchThinkingSignature();
let response = "failed" in result ? result.failed : result;
// A replacement can be an HTTP/auth/budget refusal; never feed its JSON body to an SSE parser.
if (!response.ok) {
// A local refusal closes the upstream signal before returning its generated JSON.
// Reading that bounded local body still honors caller cancellation.
const errorSignal = "failed" in result ? options.abortSignal ?? new AbortController().signal : upstream.signal;
const nonReplayable = isNonReplayableResponse(response);
if (nonReplayable) response = await sanitizeNonReplayableUpstreamError(response, errorSignal);
const body = await readDisplaySafeErrorText(response, errorSignal, "Upstream recovery failed");
const error = normalizeUpstreamErrorText(body, "Upstream recovery failed");
if (isProxyError(response)) rememberProxyErrorMessage(logCtx, error.message ?? error.safeText, { status: response.status, type: error.type ?? "upstream_error", code: error.code, presentation: proxyErrorPresentation(response) });
yield { type: "error", status: response.status, message: error.message ?? error.safeText,
errorType: error.type ?? "upstream_error", ...(error.code ? { code: error.code } : {}),
...(nonReplayable ? { retryable: false } : {}) };
return;
}
try {
const observeRetrySpend = adapterResponseSpendObserver(response);
const bodyInactivityMs = resolveStallTimeoutMs(stallTimeoutSec(), { localUpstream: adapterExchange.localUpstream });
const retryEvents = upstreamRequestsStream
? readResponseStreamWithInactivity(response, upstream.signal, bodyInactivityMs,
guarded => transportState.activeAdapter.parseStream(guarded, translatorBudget, logCtx.activeTierMetadata))
: (async function* () {
yield* await readResponseBodyWithInactivity(response, upstream.signal, bodyInactivityMs,
guarded => transportState.activeAdapter.parseResponse!(guarded, translatorBudget, logCtx.activeTierMetadata));
})();
for await (const event of retryEvents) {
observeRetrySpend?.(event);
options.onCompactionRecoveryAdapterEvent?.(event);
yield event;
}
} catch (error) {
if (!(error instanceof ResponseBodyInactivityError)) throw error;
yield { type: "error", message: error.message,
status: 504, errorType: "upstream_error", code: "upstream_stall_timeout" };
}
}, () => transportState.activeAdapter.name, !isNonReplayableResponse(upstreamResponse), upstreamResponse);
const clientToolCatalog = requestState.clientToolCatalogRestriction && route.provider.authMode !== "forward"
? requestState.clientToolCatalogRestriction : undefined;
const deliveryToolBridgeMaps = currentTurnToolBridgeMaps(toolBridgeMaps, clientToolCatalog);
if (terminalGuardEnabled) transportState.trackGuardedOAuthAttemptUsage();
// Legs the terminal guard re-asked: the client terminal carries only the final leg's usage
// (clients size their context window from it, #8877), while the request log meters every leg
// that was sent. Hidden legs add counters only: their context checkpoint and raw wire usage
// are not the final terminal's (same split as the web-search loop).
let terminalGuardHiddenUsage: OcxUsage | undefined;
const onHiddenUsage = ({ contextTotalTokens: _context, rawUsage: _raw, ...counters }: OcxUsage): void => {
terminalGuardHiddenUsage = mergeUsage(terminalGuardHiddenUsage, counters);
};
const bindDeliveredUsage = (usage: OcxUsage | undefined): void => {
// A re-asked leg was dispatched, so a delivered leg without a credit reading leaves the credits incomplete (#8452).
transportState.bindKeyUsageFromBridge(mergeUsage(terminalGuardHiddenUsage, usage, terminalGuardHiddenUsage !== undefined));
};
if (parsed.stream) {
const clientEncoder = clientToolCatalog?.bareCustomToolNames.has("exec") ? undefined
: clientEncoderForDelivery(options, logCtx, !!routedCompaction, transportState.activeAdapter.name);
// Sign only the negotiated canonical Chat delivery. Native/sidecar/opaque item identities
// retain their existing owners; structural membership grants no supplier-state authority.
const outputItemIds = requestState.completedToolBlocks && transportState.activeAdapter.name === "openai-chat"
&& options.inboundWire !== "chat" && options.inboundWire !== "anthropic"
&& !routedCompaction && !parsed._advisorGuard && !clientEncoder && upstreamResponse.headers.get("x-opencodex-tool-completion") === "block-stop-v1"
? requestState.turnMembership?.begin() : undefined;
// The continuation legs classify a stalled body themselves; the initial stream needs the
// same mapping or the bridge catch reports this upstream timeout as a 500 proxy_error.
const initialEventStream = (async function* (): AsyncGenerator<AdapterEvent> {
try {
for await (const event of readResponseStreamWithInactivity(
upstreamResponse,
upstream.signal,
bodyInactivityMs,
response => transportState.activeAdapter.parseStream(response, translatorBudget, logCtx.activeTierMetadata, { completedToolBlocks: outputItemIds !== undefined }),
)) {
observeInitialSpend?.(event);
options.onCompactionRecoveryAdapterEvent?.(event);
yield event;
}
} catch (error) {
if (error instanceof ResponseBodyInactivityError) {
yield {
type: "error",
message: error.message,
status: 504,
errorType: "upstream_error",
code: "upstream_stall_timeout",
};
return;
}
throw error;
}
})();
const recoveredEvents = recoverThinkingSignature(initialEventStream);
const eventStream = terminalGuardEnabled
? guardTerminalEventStream({
parsed,
firstEvents: recoveredEvents,
adapterName: transportState.activeAdapter.name,
maxAutoContinuations: 1,
continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream),
onHiddenUsage,
})
: recoveredEvents;
// Optional advisor guard (registered per request by the sidecar planner; absent unless the
// user enabled the advisor): holds synthetic `advisor` tool calls, consults the configured
// expert model, and re-dispatches the worker. Sits inside the empty-completion guard so an
// advisor-only turn never reads as an empty completion, and outside the bridge so the
// synthetic call never reaches the client.
const advisorStream = parsed._advisorGuard
? parsed._advisorGuard({
parsed,
translatorBudget,
firstEvents: eventStream,
continuation: fetchTerminalGuardContinuation,
})
: eventStream;
// The empty-completion guard sits OUTSIDE the terminal guard: a completed
// turn with no text and no tool call is retried with the IDENTICAL request
// (fetchTerminalGuardContinuation(parsed) replays the cached byte-identical
// request — same body, same headers, same signal).
const completionGuardedEventStream = emptyCompletionGuardEnabled
? guardEmptyCompletionEventStream({
firstEvents: advisorStream,
continuation: () => fetchGuardedEmptyCompletionRetry(onHiddenUsage),
preserveRetryError: preserveEmptyCompletionRetryError,
})
: advisorStream;
const catalogGuardedEventStream = guardEmptyToolCatalogStream(completionGuardedEventStream, clientToolCatalog, translatorBudget);
const guardedEventStream = routedCompaction && transportState.activeAdapter.name === "anthropic"
? guardCompactionSummary(catalogGuardedEventStream, translatorBudget) : catalogGuardedEventStream;
const { toolNsMap, declaredToolNames, toolParameterSchemas, freeformToolNames, bareCustomToolNames, toolSearchToolNames } = deliveryToolBridgeMaps;
// One completion owner for both deliveries: the bridge calls it from its terminal, the
// direct client encoder from the fold of the same events.
const onCompletedResponse = (response: Record<string, unknown>, providerState?: OcxProviderContinuationState) => {
if (!options.abortSignal?.aborted) thinkingSignatureRecovery.rememberSuccess(response);
settleAnthropicPerMessageEffort(parsed, response, transportState.activeAdapter.name);
const served = transportState.sentOAuthSnapshot;
if (transportState.activeAdapter.name === "kiro" && response.status === "completed" && served
&& noteKiroServedSuccess(served.accountId, served.generation)) persistKiroAccountState();
commitReasoningReplayServingRoute();
rememberKiroDeliveredFinalAnswer(transportState.activeAdapter.name, response);
// Compaction turns must NOT enter the continuation cache: _rawBody still holds the full
// PRE-compaction history, and a later previous_response_id expansion would rehydrate the
// giant stale chain Codex just replaced.
if (!routedCompaction) {
rememberResponseState(
parsed._rawBody,
response,
continuationStateForResponse(providerState),
responseStateOptions(transportState.activeAdapter.name === "kiro"),
);
}
notifyResponseComplete(response);
};
// The Responses bridge owns code-mode helper compilation; the direct encoder only renames.
if (clientEncoder) {
return deliverClientEncodedResponse({
encoder: clientEncoder,
events: guardedEventStream,
logCtx,
translatorBudget,
responseModelId: parsed._responseModelId ?? parsed.modelId,
adapterName: transportState.activeAdapter.name,
fold: {
replayCacheScope: parsed._reasoningReplayScope,
hideThinkingSummary: parsed.options.hideThinkingSummary,
chatRawReasoningVisible: options.chatRawReasoningVisible,
hideRawReasoning: parsed.options.hideRawReasoning,
toolNsMap, declaredToolNames, toolParameterSchemas, freeformToolNames, bareCustomToolNames, toolSearchToolNames,
},
stallTimeoutSec: stallTimeoutSec(),
get localUpstream() { return adapterExchange.localUpstream; },
turnAdmissionLease: options.turnAdmissionLease,
...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}),
stopUpstream: () => { cancelResponseCompletion(); upstream.abort(); },
onStreamDone: cleanupUpstreamAbort,
onCompletedResponse,
bindUsage: bindDeliveredUsage,
});
}
const sseStream = bridgeToResponsesSSE(
guardedEventStream, parsed._responseModelId ?? parsed.modelId, toolNsMap, freeformToolNames, toolSearchToolNames,
() => { cancelResponseCompletion(); upstream.abort(); }, 2_000,
{
translatorBudget, proxyErrorContext: logCtx,
replayCacheScope: parsed._reasoningReplayScope,
...(outputItemIds ? { outputItemIds } : {}),
...(options.forceEmptyResponseId ? { responseId: "" } : {}),
stallTimeoutSec: stallTimeoutSec(),
get localUpstream() { return adapterExchange.localUpstream; },
hideThinkingSummary: parsed.options.hideThinkingSummary,
chatRawReasoningVisible: options.chatRawReasoningVisible,
hideRawReasoning: parsed.options.hideRawReasoning,
rawReasoningAsSummary: parsed.options.rawReasoningAsSummary,
declaredToolNames,
bareCustomToolNames,
enforceDeclaredToolNames: options.inboundWire !== "chat" && options.inboundWire !== "anthropic",
terminalRefusalAsFailed: options.inboundWire !== "chat" && options.inboundWire !== "anthropic",
escapeMemoryCitationTags: options.inboundWire !== "chat" && options.inboundWire !== "anthropic",
toolParameterSchemas,
...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}),
...(routedCompaction ? { compaction: true, compactionRetention: parsed._compactionRetention } : {}),
completeOnMaxOutputTokens: config.completeOnMaxOutputTokens === true && options.inboundWire !== "chat" && options.inboundWire !== "anthropic",
textFormat: parsed.options.textFormat,
// Same grok-surface split as the runTurn branch above.
...(logCtx.surface === "grok" ? { heartbeatStyle: "comment" as const } : {}),
// Raw adapter usage, pre wire-normalization (see the runTurn branch above).
onUsage: bindDeliveredUsage,
onCompletedResponse,
},
);
const bridgeTurnAc = new AbortController();
const trackedSse = trackStreamLifetime(sseStream, bridgeTurnAc, cleanupUpstreamAbort, options.turnAdmissionLease);
return new Response(trackedSse, {
headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no" },
});
}
if (transportState.activeAdapter.parseResponse
|| (upstreamRequestsStream && transportState.activeAdapter.parseStream)) {
let events: AdapterEvent[];
try {
const initialEvents: AdapterEvent[] = [];
if (upstreamRequestsStream && transportState.activeAdapter.parseStream) {
// The canonical ChatGPT adapter coerces its actual upstream request to SSE even for a
// JSON client. Routed compaction still owns the non-streaming client response, so fold
// that SSE through the adapter instead of handing it to parseResponse as JSON.
for await (const event of readResponseStreamWithInactivity(
upstreamResponse,
upstream.signal,
bodyInactivityMs,
response => transportState.activeAdapter.parseStream(response, translatorBudget, logCtx.activeTierMetadata),
)) initialEvents.push(event);
} else {
initialEvents.push(...await readResponseBodyWithInactivity(
upstreamResponse,
upstream.signal,
bodyInactivityMs,
response => transportState.activeAdapter.parseResponse!(response, translatorBudget, logCtx.activeTierMetadata),
));
}
for (const event of initialEvents) {
observeInitialSpend?.(event);
options.onCompactionRecoveryAdapterEvent?.(event);
}
const recoveredEvents: AdapterEvent[] = [];
for await (const event of recoverThinkingSignature((async function* () { yield* initialEvents; })())) {
recoveredEvents.push(event);
}
let guardedEvents: AdapterEvent[];
if (terminalGuardEnabled) {
guardedEvents = [];
for await (const event of guardTerminalEventStream({
parsed,
firstEvents: (async function* () { yield* recoveredEvents; })(),
adapterName: transportState.activeAdapter.name,
maxAutoContinuations: 1,
continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream),
onHiddenUsage,
})) guardedEvents.push(event);
} else {
guardedEvents = recoveredEvents;
}
// Optional advisor guard — same semantics as the streaming branch above.
if (parsed._advisorGuard) {
const advisorEvents: AdapterEvent[] = [];
for await (const event of parsed._advisorGuard({
parsed,
translatorBudget,
firstEvents: (async function* () { yield* guardedEvents; })(),
continuation: fetchTerminalGuardContinuation,
})) advisorEvents.push(event);
guardedEvents = advisorEvents;
}
if (emptyCompletionGuardEnabled) {
events = [];
for await (const event of guardEmptyCompletionEventStream({
firstEvents: (async function* () { yield* guardedEvents; })(),
continuation: () => fetchGuardedEmptyCompletionRetry(onHiddenUsage),
preserveRetryError: preserveEmptyCompletionRetryError,
})) events.push(event);
} else {
events = guardedEvents;
}
if (routedCompaction && transportState.activeAdapter.name === "anthropic") {
const summaryEvents: AdapterEvent[] = [];
for await (const event of guardCompactionSummary((async function* () { yield* events; })(), translatorBudget)) {
summaryEvents.push(event);
}
events = summaryEvents;
}
} catch (error) {
if (error instanceof ResponseBodyInactivityError) {
return formatErrorResponse(504, "upstream_error", error.message);
}
throw error;
} finally {
cleanupUpstreamAbort();
}
const { toolNsMap, declaredToolNames, toolParameterSchemas, freeformToolNames, bareCustomToolNames, toolSearchToolNames } = deliveryToolBridgeMaps;
let providerState: OcxProviderContinuationState | undefined;
events = guardEmptyToolCatalogEvents(events, clientToolCatalog, translatorBudget);
const json = buildResponseJSON(events, parsed._responseModelId ?? parsed.modelId, {
translatorBudget,
replayCacheScope: parsed._reasoningReplayScope,
hideThinkingSummary: parsed.options.hideThinkingSummary,
chatRawReasoningVisible: options.chatRawReasoningVisible,
hideRawReasoning: parsed.options.hideRawReasoning,
rawReasoningAsSummary: parsed.options.rawReasoningAsSummary,
toolNsMap,
declaredToolNames,
bareCustomToolNames,
enforceDeclaredToolNames: options.inboundWire !== "chat" && options.inboundWire !== "anthropic",
escapeMemoryCitationTags: options.inboundWire !== "chat" && options.inboundWire !== "anthropic",
toolParameterSchemas,
freeformToolNames,
toolSearchToolNames,
...(routedCompaction ? { compaction: true, compactionRetention: parsed._compactionRetention } : {}),
completeOnMaxOutputTokens: config.completeOnMaxOutputTokens === true && options.inboundWire !== "chat" && options.inboundWire !== "anthropic",
textFormat: parsed.options.textFormat,
onProviderState: state => { providerState = state; },
onUsage: bindDeliveredUsage,
});
// See the streaming branch: compaction turns skip the continuation cache.
if (!routedCompaction) {
rememberKiroDeliveredFinalAnswer(transportState.activeAdapter.name, json);
rememberResponseState(
parsed._rawBody,
json,
continuationStateForResponse(providerState),
responseStateOptions(transportState.activeAdapter.name === "kiro"),
);
}
// #1926 gap 2: same buffered-path durability bound as the primary branch.
await awaitThoughtSignatureDurability();
if (adapterResponseReachedServingTerminal(events, json)) {
if (!options.abortSignal?.aborted) thinkingSignatureRecovery.rememberSuccess(json);
const served = transportState.sentOAuthSnapshot;
if (transportState.activeAdapter.name === "kiro" && json.status === "completed" && served
&& noteKiroServedSuccess(served.accountId, served.generation)) persistKiroAccountState();
commitReasoningReplayServingRoute();
}
notifyResponseComplete(json);
const lastError = events.findLast(event => event.type === "error");
const status = json.status === "failed" && lastError && isContinuationBodyStall(lastError) ? 504 : 200;
return new Response(JSON.stringify(json), { status, headers: { "Content-Type": "application/json" } });
}
return formatErrorResponse(400, "invalid_request_error", "Non-streaming not supported by this adapter");
}