UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

451 lines (442 loc) • 25 kB
import { proxyErrorPresentation, isProxyError, rememberProxyErrorMessage } from "../../lib/client-error-body"; import { responsesTimeoutProvider, type ResponsesRequestContext } from "./core-options"; import type { PreparedResponsesRequest } from "./request-prepare"; import type { ResponsesTransport } from "./request-transport"; import type { ResponsesSidecarAuth } from "./request-sidecar-auth"; import type { ResponsesEffects } from "./response-effects"; import type { ResponsesCompletionPolicy } from "./completion-policy"; import type { AdapterExchange } from "./adapter-dispatch"; import type { AdapterContinuations } from "./adapter-continuation"; import { guardTerminalEventStream } from "./terminal-guard"; import { guardEmptyCompletionEventStream, mergeUsage } from "./empty-completion-guard"; import { UNDECLARED_TOOL_CALL_ERROR_CODE } from "../../lib/errors"; import { bridgeToResponsesSSE, buildResponseJSON, formatErrorResponse } from "../../bridge"; import type { OcxProviderContinuationState, AdapterEvent, OcxUsage } from "../../types"; import { rememberResponseState } from "../../responses/state"; import { trackStreamLifetime } from "../lifecycle"; import { awaitThoughtSignatureDurability } from "../../responses/thought-signature-replay"; import { adapterResponseReachedServingTerminal } from "./core-replay"; import { readResponseBodyWithInactivity, readResponseStreamWithInactivity, ResponseBodyInactivityError, } from "../../lib/response-body-inactivity"; import { providerStallTimeoutSec, resolveStallTimeoutMs } from "../../stall-timeout"; import { clientEncoderForDelivery, deliverClientEncodedResponse } from "../inference/client-encoder-delivery"; import { noteKiroServedSuccess } from "../../providers/kiro-usage"; import { persistKiroAccountState } from "../../providers/kiro-account-state-disk"; import { readDisplaySafeErrorText, normalizeUpstreamErrorText } from "./core-errors"; import { sanitizeNonReplayableUpstreamError } from "./non-replayable-error"; import { isNonReplayableResponse } from "../../lib/upstream-retry"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { currentTurnToolBridgeMaps, guardEmptyToolCatalogEvents, guardEmptyToolCatalogStream } from "../../lib/empty-tool-catalog"; import { guardCompactionSummary } from "../../responses/compaction-summary"; import { settleAnthropicPerMessageEffort } from "../../adapters/anthropic/per-message-effort"; import { adapterResponseSpendObserver } from "./request-spend"; /** One responsibility of the Responses request pipeline; state owners are explicit. */ export async function deliverAdapterResponse( requestContext: Pick<ResponsesRequestContext, "logCtx" | "options" | "config">, requestState: Pick< PreparedResponsesRequest, | "parsed" | "route" | "translatorBudget" | "toolBridgeMaps" | "clientToolCatalogRestriction" | "rememberKiroDeliveredFinalAnswer" | "responseStateOptions" | "turnMembership" | "completedToolBlocks" >, transportState: Pick<ResponsesTransport, "activeAdapter" | "bindKeyUsageFromBridge" | "sentOAuthSnapshot" | "trackGuardedOAuthAttemptUsage">, sidecarState: Pick<ResponsesSidecarAuth, "routedCompaction">, responseEffects: Pick< ResponsesEffects, | "cancelResponseCompletion" | "commitReasoningReplayServingRoute" | "continuationStateForResponse" | "notifyResponseComplete" >, completionPolicy: Pick<ResponsesCompletionPolicy, "emptyCompletionGuardEnabled">, adapterExchange: Pick<AdapterExchange, "upstreamResponse" | "upstream" | "cleanupUpstreamAbort" | "localUpstream" | "thinkingSignatureRecovery" | "refetchThinkingSignature">, continuationState: Pick<AdapterContinuations, "terminalGuardEnabled" | "fetchTerminalGuardContinuation" | "fetchGuardedEmptyCompletionRetry" | "isContinuationBodyStall">, ): Promise<Response> { const { logCtx, options, config } = requestContext; const { parsed, route, translatorBudget, toolBridgeMaps, rememberKiroDeliveredFinalAnswer, responseStateOptions, } = requestState; const { upstreamResponse, upstream, cleanupUpstreamAbort, localUpstream, thinkingSignatureRecovery, refetchThinkingSignature } = adapterExchange; const { terminalGuardEnabled, fetchTerminalGuardContinuation, fetchGuardedEmptyCompletionRetry, isContinuationBodyStall, } = continuationState; const preserveEmptyCompletionRetryError = (event: AdapterEvent) => (event.type === "error" && event.code === UNDECLARED_TOOL_CALL_ERROR_CODE) || isContinuationBodyStall(event); const { emptyCompletionGuardEnabled } = completionPolicy; const { cancelResponseCompletion, commitReasoningReplayServingRoute, continuationStateForResponse, notifyResponseComplete, } = responseEffects; const { routedCompaction } = sidecarState; const stallTimeoutSec = () => providerStallTimeoutSec(config, responsesTimeoutProvider(options, route.provider)); const bodyInactivityMs = resolveStallTimeoutMs(stallTimeoutSec(), { localUpstream }); const upstreamRequestsStream = parsed.stream || isCanonicalOpenAiForwardProvider(route.provider); const observeInitialSpend = adapterResponseSpendObserver(upstreamResponse); const recoverThinkingSignature = (events: AsyncIterable<AdapterEvent>): AsyncIterable<AdapterEvent> => thinkingSignatureRecovery.wrap(events, async function* () { const result = await refetchThinkingSignature(); let response = "failed" in result ? result.failed : result; // A replacement can be an HTTP/auth/budget refusal; never feed its JSON body to an SSE parser. if (!response.ok) { // A local refusal closes the upstream signal before returning its generated JSON. // Reading that bounded local body still honors caller cancellation. const errorSignal = "failed" in result ? options.abortSignal ?? new AbortController().signal : upstream.signal; const nonReplayable = isNonReplayableResponse(response); if (nonReplayable) response = await sanitizeNonReplayableUpstreamError(response, errorSignal); const body = await readDisplaySafeErrorText(response, errorSignal, "Upstream recovery failed"); const error = normalizeUpstreamErrorText(body, "Upstream recovery failed"); if (isProxyError(response)) rememberProxyErrorMessage(logCtx, error.message ?? error.safeText, { status: response.status, type: error.type ?? "upstream_error", code: error.code, presentation: proxyErrorPresentation(response) }); yield { type: "error", status: response.status, message: error.message ?? error.safeText, errorType: error.type ?? "upstream_error", ...(error.code ? { code: error.code } : {}), ...(nonReplayable ? { retryable: false } : {}) }; return; } try { const observeRetrySpend = adapterResponseSpendObserver(response); const bodyInactivityMs = resolveStallTimeoutMs(stallTimeoutSec(), { localUpstream: adapterExchange.localUpstream }); const retryEvents = upstreamRequestsStream ? readResponseStreamWithInactivity(response, upstream.signal, bodyInactivityMs, guarded => transportState.activeAdapter.parseStream(guarded, translatorBudget, logCtx.activeTierMetadata)) : (async function* () { yield* await readResponseBodyWithInactivity(response, upstream.signal, bodyInactivityMs, guarded => transportState.activeAdapter.parseResponse!(guarded, translatorBudget, logCtx.activeTierMetadata)); })(); for await (const event of retryEvents) { observeRetrySpend?.(event); options.onCompactionRecoveryAdapterEvent?.(event); yield event; } } catch (error) { if (!(error instanceof ResponseBodyInactivityError)) throw error; yield { type: "error", message: error.message, status: 504, errorType: "upstream_error", code: "upstream_stall_timeout" }; } }, () => transportState.activeAdapter.name, !isNonReplayableResponse(upstreamResponse), upstreamResponse); const clientToolCatalog = requestState.clientToolCatalogRestriction && route.provider.authMode !== "forward" ? requestState.clientToolCatalogRestriction : undefined; const deliveryToolBridgeMaps = currentTurnToolBridgeMaps(toolBridgeMaps, clientToolCatalog); if (terminalGuardEnabled) transportState.trackGuardedOAuthAttemptUsage(); // Legs the terminal guard re-asked: the client terminal carries only the final leg's usage // (clients size their context window from it, #8877), while the request log meters every leg // that was sent. Hidden legs add counters only: their context checkpoint and raw wire usage // are not the final terminal's (same split as the web-search loop). let terminalGuardHiddenUsage: OcxUsage | undefined; const onHiddenUsage = ({ contextTotalTokens: _context, rawUsage: _raw, ...counters }: OcxUsage): void => { terminalGuardHiddenUsage = mergeUsage(terminalGuardHiddenUsage, counters); }; const bindDeliveredUsage = (usage: OcxUsage | undefined): void => { // A re-asked leg was dispatched, so a delivered leg without a credit reading leaves the credits incomplete (#8452). transportState.bindKeyUsageFromBridge(mergeUsage(terminalGuardHiddenUsage, usage, terminalGuardHiddenUsage !== undefined)); }; if (parsed.stream) { const clientEncoder = clientToolCatalog?.bareCustomToolNames.has("exec") ? undefined : clientEncoderForDelivery(options, logCtx, !!routedCompaction, transportState.activeAdapter.name); // Sign only the negotiated canonical Chat delivery. Native/sidecar/opaque item identities // retain their existing owners; structural membership grants no supplier-state authority. const outputItemIds = requestState.completedToolBlocks && transportState.activeAdapter.name === "openai-chat" && options.inboundWire !== "chat" && options.inboundWire !== "anthropic" && !routedCompaction && !parsed._advisorGuard && !clientEncoder && upstreamResponse.headers.get("x-opencodex-tool-completion") === "block-stop-v1" ? requestState.turnMembership?.begin() : undefined; // The continuation legs classify a stalled body themselves; the initial stream needs the // same mapping or the bridge catch reports this upstream timeout as a 500 proxy_error. const initialEventStream = (async function* (): AsyncGenerator<AdapterEvent> { try { for await (const event of readResponseStreamWithInactivity( upstreamResponse, upstream.signal, bodyInactivityMs, response => transportState.activeAdapter.parseStream(response, translatorBudget, logCtx.activeTierMetadata, { completedToolBlocks: outputItemIds !== undefined }), )) { observeInitialSpend?.(event); options.onCompactionRecoveryAdapterEvent?.(event); yield event; } } catch (error) { if (error instanceof ResponseBodyInactivityError) { yield { type: "error", message: error.message, status: 504, errorType: "upstream_error", code: "upstream_stall_timeout", }; return; } throw error; } })(); const recoveredEvents = recoverThinkingSignature(initialEventStream); const eventStream = terminalGuardEnabled ? guardTerminalEventStream({ parsed, firstEvents: recoveredEvents, adapterName: transportState.activeAdapter.name, maxAutoContinuations: 1, continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream), onHiddenUsage, }) : recoveredEvents; // Optional advisor guard (registered per request by the sidecar planner; absent unless the // user enabled the advisor): holds synthetic `advisor` tool calls, consults the configured // expert model, and re-dispatches the worker. Sits inside the empty-completion guard so an // advisor-only turn never reads as an empty completion, and outside the bridge so the // synthetic call never reaches the client. const advisorStream = parsed._advisorGuard ? parsed._advisorGuard({ parsed, translatorBudget, firstEvents: eventStream, continuation: fetchTerminalGuardContinuation, }) : eventStream; // The empty-completion guard sits OUTSIDE the terminal guard: a completed // turn with no text and no tool call is retried with the IDENTICAL request // (fetchTerminalGuardContinuation(parsed) replays the cached byte-identical // request — same body, same headers, same signal). const completionGuardedEventStream = emptyCompletionGuardEnabled ? guardEmptyCompletionEventStream({ firstEvents: advisorStream, continuation: () => fetchGuardedEmptyCompletionRetry(onHiddenUsage), preserveRetryError: preserveEmptyCompletionRetryError, }) : advisorStream; const catalogGuardedEventStream = guardEmptyToolCatalogStream(completionGuardedEventStream, clientToolCatalog, translatorBudget); const guardedEventStream = routedCompaction && transportState.activeAdapter.name === "anthropic" ? guardCompactionSummary(catalogGuardedEventStream, translatorBudget) : catalogGuardedEventStream; const { toolNsMap, declaredToolNames, toolParameterSchemas, freeformToolNames, bareCustomToolNames, toolSearchToolNames } = deliveryToolBridgeMaps; // One completion owner for both deliveries: the bridge calls it from its terminal, the // direct client encoder from the fold of the same events. const onCompletedResponse = (response: Record<string, unknown>, providerState?: OcxProviderContinuationState) => { if (!options.abortSignal?.aborted) thinkingSignatureRecovery.rememberSuccess(response); settleAnthropicPerMessageEffort(parsed, response, transportState.activeAdapter.name); const served = transportState.sentOAuthSnapshot; if (transportState.activeAdapter.name === "kiro" && response.status === "completed" && served && noteKiroServedSuccess(served.accountId, served.generation)) persistKiroAccountState(); commitReasoningReplayServingRoute(); rememberKiroDeliveredFinalAnswer(transportState.activeAdapter.name, response); // Compaction turns must NOT enter the continuation cache: _rawBody still holds the full // PRE-compaction history, and a later previous_response_id expansion would rehydrate the // giant stale chain Codex just replaced. if (!routedCompaction) { rememberResponseState( parsed._rawBody, response, continuationStateForResponse(providerState), responseStateOptions(transportState.activeAdapter.name === "kiro"), ); } notifyResponseComplete(response); }; // The Responses bridge owns code-mode helper compilation; the direct encoder only renames. if (clientEncoder) { return deliverClientEncodedResponse({ encoder: clientEncoder, events: guardedEventStream, logCtx, translatorBudget, responseModelId: parsed._responseModelId ?? parsed.modelId, adapterName: transportState.activeAdapter.name, fold: { replayCacheScope: parsed._reasoningReplayScope, hideThinkingSummary: parsed.options.hideThinkingSummary, chatRawReasoningVisible: options.chatRawReasoningVisible, hideRawReasoning: parsed.options.hideRawReasoning, toolNsMap, declaredToolNames, toolParameterSchemas, freeformToolNames, bareCustomToolNames, toolSearchToolNames, }, stallTimeoutSec: stallTimeoutSec(), get localUpstream() { return adapterExchange.localUpstream; }, turnAdmissionLease: options.turnAdmissionLease, ...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}), stopUpstream: () => { cancelResponseCompletion(); upstream.abort(); }, onStreamDone: cleanupUpstreamAbort, onCompletedResponse, bindUsage: bindDeliveredUsage, }); } const sseStream = bridgeToResponsesSSE( guardedEventStream, parsed._responseModelId ?? parsed.modelId, toolNsMap, freeformToolNames, toolSearchToolNames, () => { cancelResponseCompletion(); upstream.abort(); }, 2_000, { translatorBudget, proxyErrorContext: logCtx, replayCacheScope: parsed._reasoningReplayScope, ...(outputItemIds ? { outputItemIds } : {}), ...(options.forceEmptyResponseId ? { responseId: "" } : {}), stallTimeoutSec: stallTimeoutSec(), get localUpstream() { return adapterExchange.localUpstream; }, hideThinkingSummary: parsed.options.hideThinkingSummary, chatRawReasoningVisible: options.chatRawReasoningVisible, hideRawReasoning: parsed.options.hideRawReasoning, rawReasoningAsSummary: parsed.options.rawReasoningAsSummary, declaredToolNames, bareCustomToolNames, enforceDeclaredToolNames: options.inboundWire !== "chat" && options.inboundWire !== "anthropic", terminalRefusalAsFailed: options.inboundWire !== "chat" && options.inboundWire !== "anthropic", escapeMemoryCitationTags: options.inboundWire !== "chat" && options.inboundWire !== "anthropic", toolParameterSchemas, ...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}), ...(routedCompaction ? { compaction: true, compactionRetention: parsed._compactionRetention } : {}), completeOnMaxOutputTokens: config.completeOnMaxOutputTokens === true && options.inboundWire !== "chat" && options.inboundWire !== "anthropic", textFormat: parsed.options.textFormat, // Same grok-surface split as the runTurn branch above. ...(logCtx.surface === "grok" ? { heartbeatStyle: "comment" as const } : {}), // Raw adapter usage, pre wire-normalization (see the runTurn branch above). onUsage: bindDeliveredUsage, onCompletedResponse, }, ); const bridgeTurnAc = new AbortController(); const trackedSse = trackStreamLifetime(sseStream, bridgeTurnAc, cleanupUpstreamAbort, options.turnAdmissionLease); return new Response(trackedSse, { headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", "Connection": "keep-alive", "X-Accel-Buffering": "no" }, }); } if (transportState.activeAdapter.parseResponse || (upstreamRequestsStream && transportState.activeAdapter.parseStream)) { let events: AdapterEvent[]; try { const initialEvents: AdapterEvent[] = []; if (upstreamRequestsStream && transportState.activeAdapter.parseStream) { // The canonical ChatGPT adapter coerces its actual upstream request to SSE even for a // JSON client. Routed compaction still owns the non-streaming client response, so fold // that SSE through the adapter instead of handing it to parseResponse as JSON. for await (const event of readResponseStreamWithInactivity( upstreamResponse, upstream.signal, bodyInactivityMs, response => transportState.activeAdapter.parseStream(response, translatorBudget, logCtx.activeTierMetadata), )) initialEvents.push(event); } else { initialEvents.push(...await readResponseBodyWithInactivity( upstreamResponse, upstream.signal, bodyInactivityMs, response => transportState.activeAdapter.parseResponse!(response, translatorBudget, logCtx.activeTierMetadata), )); } for (const event of initialEvents) { observeInitialSpend?.(event); options.onCompactionRecoveryAdapterEvent?.(event); } const recoveredEvents: AdapterEvent[] = []; for await (const event of recoverThinkingSignature((async function* () { yield* initialEvents; })())) { recoveredEvents.push(event); } let guardedEvents: AdapterEvent[]; if (terminalGuardEnabled) { guardedEvents = []; for await (const event of guardTerminalEventStream({ parsed, firstEvents: (async function* () { yield* recoveredEvents; })(), adapterName: transportState.activeAdapter.name, maxAutoContinuations: 1, continuation: next => fetchTerminalGuardContinuation(next, undefined, !parsed.stream), onHiddenUsage, })) guardedEvents.push(event); } else { guardedEvents = recoveredEvents; } // Optional advisor guard — same semantics as the streaming branch above. if (parsed._advisorGuard) { const advisorEvents: AdapterEvent[] = []; for await (const event of parsed._advisorGuard({ parsed, translatorBudget, firstEvents: (async function* () { yield* guardedEvents; })(), continuation: fetchTerminalGuardContinuation, })) advisorEvents.push(event); guardedEvents = advisorEvents; } if (emptyCompletionGuardEnabled) { events = []; for await (const event of guardEmptyCompletionEventStream({ firstEvents: (async function* () { yield* guardedEvents; })(), continuation: () => fetchGuardedEmptyCompletionRetry(onHiddenUsage), preserveRetryError: preserveEmptyCompletionRetryError, })) events.push(event); } else { events = guardedEvents; } if (routedCompaction && transportState.activeAdapter.name === "anthropic") { const summaryEvents: AdapterEvent[] = []; for await (const event of guardCompactionSummary((async function* () { yield* events; })(), translatorBudget)) { summaryEvents.push(event); } events = summaryEvents; } } catch (error) { if (error instanceof ResponseBodyInactivityError) { return formatErrorResponse(504, "upstream_error", error.message); } throw error; } finally { cleanupUpstreamAbort(); } const { toolNsMap, declaredToolNames, toolParameterSchemas, freeformToolNames, bareCustomToolNames, toolSearchToolNames } = deliveryToolBridgeMaps; let providerState: OcxProviderContinuationState | undefined; events = guardEmptyToolCatalogEvents(events, clientToolCatalog, translatorBudget); const json = buildResponseJSON(events, parsed._responseModelId ?? parsed.modelId, { translatorBudget, replayCacheScope: parsed._reasoningReplayScope, hideThinkingSummary: parsed.options.hideThinkingSummary, chatRawReasoningVisible: options.chatRawReasoningVisible, hideRawReasoning: parsed.options.hideRawReasoning, rawReasoningAsSummary: parsed.options.rawReasoningAsSummary, toolNsMap, declaredToolNames, bareCustomToolNames, enforceDeclaredToolNames: options.inboundWire !== "chat" && options.inboundWire !== "anthropic", escapeMemoryCitationTags: options.inboundWire !== "chat" && options.inboundWire !== "anthropic", toolParameterSchemas, freeformToolNames, toolSearchToolNames, ...(routedCompaction ? { compaction: true, compactionRetention: parsed._compactionRetention } : {}), completeOnMaxOutputTokens: config.completeOnMaxOutputTokens === true && options.inboundWire !== "chat" && options.inboundWire !== "anthropic", textFormat: parsed.options.textFormat, onProviderState: state => { providerState = state; }, onUsage: bindDeliveredUsage, }); // See the streaming branch: compaction turns skip the continuation cache. if (!routedCompaction) { rememberKiroDeliveredFinalAnswer(transportState.activeAdapter.name, json); rememberResponseState( parsed._rawBody, json, continuationStateForResponse(providerState), responseStateOptions(transportState.activeAdapter.name === "kiro"), ); } // #1926 gap 2: same buffered-path durability bound as the primary branch. await awaitThoughtSignatureDurability(); if (adapterResponseReachedServingTerminal(events, json)) { if (!options.abortSignal?.aborted) thinkingSignatureRecovery.rememberSuccess(json); const served = transportState.sentOAuthSnapshot; if (transportState.activeAdapter.name === "kiro" && json.status === "completed" && served && noteKiroServedSuccess(served.accountId, served.generation)) persistKiroAccountState(); commitReasoningReplayServingRoute(); } notifyResponseComplete(json); const lastError = events.findLast(event => event.type === "error"); const status = json.status === "failed" && lastError && isContinuationBodyStall(lastError) ? 504 : 200; return new Response(JSON.stringify(json), { status, headers: { "Content-Type": "application/json" } }); } return formatErrorResponse(400, "invalid_request_error", "Non-streaming not supported by this adapter"); }