UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

442 lines (419 loc) • 18.6 kB
import type { ResponsesTerminalStatus } from "../../bridge"; import { comboFailureDecision } from "../../combos"; import { httpStatusFromTerminalError } from "../../lib/errors"; import type { RequestFailureStage } from "../../lib/request-failure-model"; import type { RequestLogContext } from "../request-log"; import { createSseInspector } from "../relay"; import { MAX_CLIENT_SSE_FRAME_BYTES } from "../sse-frame-buffer"; const COMBO_STREAM_PREFLIGHT_MAX_BYTES = MAX_CLIENT_SSE_FRAME_BYTES; // Keep retained object count proportional to the same byte budget used by the // shared SSE framer. Tiny or empty upstream reads must not bypass the byte cap. const COMBO_STREAM_PREFLIGHT_MAX_CHUNKS = Math.max( 1, Math.ceil(COMBO_STREAM_PREFLIGHT_MAX_BYTES / 1024), ); const PRE_OUTPUT_CONTROL_EVENTS = new Set([ "response.created", "response.in_progress", "response.queued", "response.heartbeat", ]); const TERMINAL_EVENTS = new Set([ "response.completed", "response.failed", "response.incomplete", ]); const RETRYABLE_ZERO_OUTPUT_INCOMPLETE_REASONS = new Set([ "adapter_eof", "missing_terminal_event", "upstream_stall_timeout", ]); function bareErrorStatus(payload: unknown): number | undefined { if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined; const event = payload as Record<string, unknown>; if (event.type !== "error") return undefined; const nested = event.error; const error = nested && typeof nested === "object" && !Array.isArray(nested) ? nested as Record<string, unknown> : event; const explicitStatus = [ event.status, event.status_code, event.http_status, error.status, error.status_code, error.http_status, ] .map(value => typeof value === "number" && Number.isInteger(value) ? value : typeof value === "string" && /^\d{3}$/.test(value.trim()) ? Number(value) : undefined) .find(value => value !== undefined && value >= 400 && value <= 599); const code = typeof error.code === "string" ? error.code : typeof event.code === "string" ? event.code : null; if (explicitStatus === undefined && code === "invalid_request_error") return 400; return explicitStatus ?? httpStatusFromTerminalError({ type: typeof error.type === "string" && error.type !== "error" ? error.type : undefined, code, message: typeof error.message === "string" ? error.message : typeof event.message === "string" ? event.message : undefined, }); } function bareErrorIsRetryable(payload: unknown): boolean { const status = bareErrorStatus(payload); if (status === undefined || !payload || typeof payload !== "object" || Array.isArray(payload)) { return false; } const event = payload as Record<string, unknown>; const nested = event.error; const error = nested && typeof nested === "object" && !Array.isArray(nested) ? nested as Record<string, unknown> : event; const code = typeof error.code === "string" ? error.code : typeof event.code === "string" ? event.code : null; const message = typeof error.message === "string" ? error.message : typeof event.message === "string" ? event.message : ""; return comboFailureDecision(status, message, { code }) === "hop"; } function retryableZeroOutputTerminal(payload: unknown): boolean { if (!payload || typeof payload !== "object" || Array.isArray(payload)) return false; const event = payload as { type?: unknown; response?: { incomplete_details?: { reason?: unknown } }; }; if (bareErrorIsRetryable(event)) return true; if (event.type === "response.failed") return true; if (event.type !== "response.incomplete") return false; const reason = event.response?.incomplete_details?.reason; return typeof reason === "string" && RETRYABLE_ZERO_OUTPUT_INCOMPLETE_REASONS.has(reason); } /** * Decide when replaying the request on another combo target would risk duplicating * client-visible output or a tool-side effect. Unknown event types commit the child * conservatively; only the small Responses lifecycle preamble remains replayable. */ export function comboStreamPayloadCommitsOutput(payload: unknown): boolean { if (!payload || typeof payload !== "object" || Array.isArray(payload)) return true; const type = (payload as { type?: unknown }).type; if (typeof type !== "string") return true; if (type === "response.created") { // A created event is a control frame only while its snapshot is empty. An origin that // resumes a turn can put completed items in it, and treating that as a prelude would let // a replacement re-emit output the caller already received. const response = (payload as { response?: unknown }).response; if (response && typeof response === "object" && !Array.isArray(response)) { const output = (response as { output?: unknown }).output; if (Array.isArray(output) && output.length > 0) return true; } } return !PRE_OUTPUT_CONTROL_EVENTS.has(type) && !TERMINAL_EVENTS.has(type); } /** * How far this SSE body got, in the vocabulary of src/lib/request-failure-model.ts. * * The preflight cannot separate `semantic-output` from `side-effect`: it classifies any event * that is not a lifecycle control frame as committing, without reading item types. Both stages * refuse a resend, so the distinction would change no decision -- it is named here so a later * reader does not mistake the collapse for an omission. * * A terminal that settled carrying no output is `protocol-prelude`, not `terminal`. That is * the failure model's own rule: `terminal` means the answer was delivered, and an empty * completion delivered none. * * Only the two nothing-observed stages actually reach a read error today: the loop below hands * the body back as `accepted` the moment output commits or a terminal arrives, so a stream * that committed anything never reports a stage at all. The committed branches stay because * this has to be total for any other caller, and because a later change to that loop must not * be able to promote a committed stream into a replaceable one by omission. */ function observedResponsesStage(state: { readonly outputCommitted: boolean; readonly terminalStatus: ResponsesTerminalStatus | undefined; readonly responseCreated: boolean; }): RequestFailureStage { if (state.outputCommitted) return "semantic-output"; if (state.terminalStatus === "completed") return "terminal"; if (state.responseCreated || state.terminalStatus !== undefined) return "protocol-prelude"; return "headers-only"; } function replayBufferedResponse( response: Response, reader: ReadableStreamDefaultReader<Uint8Array>, buffered: Uint8Array[], ): Response { let index = 0; const body = new ReadableStream<Uint8Array>({ async pull(controller) { if (index < buffered.length) { controller.enqueue(buffered[index++]!); return; } try { const next = await reader.read(); if (next.done) controller.close(); else controller.enqueue(next.value); } catch (error) { try { controller.error(error); } catch { /* consumer already closed */ } } }, cancel(reason) { reader.cancel(reason).catch(() => undefined); }, }); return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers, }); } function failedTerminalResponse( response: Response, terminalPayload: Record<string, unknown>, logCtx: RequestLogContext, ): Response { const nested = terminalPayload.response; const terminalResponse = nested && typeof nested === "object" && !Array.isArray(nested) ? nested as Record<string, unknown> : {}; const nestedError = terminalResponse.error; const topLevelError = terminalPayload.error; const error = nestedError && typeof nestedError === "object" && !Array.isArray(nestedError) ? nestedError as Record<string, unknown> : topLevelError && typeof topLevelError === "object" && !Array.isArray(topLevelError) ? topLevelError as Record<string, unknown> : { type: "upstream_error", code: "upstream_server_error", message: typeof terminalPayload.message === "string" ? terminalPayload.message : logCtx.upstreamError ?? "Provider stream failed before producing output", }; const headers = new Headers(response.headers); headers.set("content-type", "application/json"); headers.delete("content-length"); headers.delete("content-encoding"); const usage = terminalResponse.usage; return new Response(JSON.stringify({ error, // The combo classifier needs only the error and optional usage. Do not carry // response ids, provider metadata, or future terminal fields into the client // error envelope merely because they shared the terminal snapshot. response: { error, ...(usage && typeof usage === "object" && !Array.isArray(usage) ? { usage } : {}), }, }), { status: logCtx.terminalHttpStatus ?? bareErrorStatus(terminalPayload) ?? 502, headers, }); } export type ComboStreamPreflightResult = | { kind: "accepted"; response: Response } | { kind: "failed"; response: Response } /** * The body errored mid-stream and `replayReadErrors` asked for the prefix back rather than * a rethrow. `stage` is how far the inspection actually got; whether that permits a * replacement is the resend gate's decision, not this function's. Callers that only act on * a projected terminal can treat this exactly as `accepted`, which is what it was before * the stage became observable. */ | { kind: "read-error"; response: Response; error: unknown; stage: RequestFailureStage }; /** * Buffer a Responses SSE only until the request becomes unsafe to replay or reaches a * terminal. Combo failover and native post-header reset recovery share this protocol * boundary, because they are asking the same question about the same bytes. This owns exactly * one body reader. The aggregate buffer is capped by bytes and retained chunks; hitting either * cap commits the current target instead of growing memory or guessing that replay is safe. */ export async function preflightComboStreamResponse( response: Response, logCtx: RequestLogContext, retryableTerminal: (payload: unknown) => boolean = retryableZeroOutputTerminal, options?: { allowMissingContentType?: boolean; replayReadErrors?: boolean }, ): Promise<ComboStreamPreflightResult> { const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; const isEventStream = contentType.includes("text/event-stream") || (!contentType && options?.allowMissingContentType === true); if (!response.ok || !response.body || !isEventStream) { return { kind: "accepted", response }; } const reader = response.body.getReader(); const buffered: Uint8Array[] = []; let bufferedBytes = 0; let outputCommitted = false; let responseCreated = false; let terminalStatus: ResponsesTerminalStatus | undefined; let retryableTerminalPayload: Record<string, unknown> | undefined; const inspector = createSseInspector({ logCtx, // A payload the inspector could not parse still reached this proxy, and it may be output. // Committing on it is what keeps an unreadable frame from reading as an empty prelude. onOpaquePayload: () => { outputCommitted = true; }, onParsedPayload: payload => { if (terminalStatus !== undefined || outputCommitted || retryableTerminalPayload) return; if (payload !== null && typeof payload === "object" && !Array.isArray(payload) && (payload as { type?: unknown }).type === "response.created") responseCreated = true; const retryable = retryableTerminal(payload); const matchedBareError = retryable && payload !== null && typeof payload === "object" && !Array.isArray(payload) && (payload as { type?: unknown }).type === "error"; // A zero-output bare error is terminal evidence. Explicit client errors stay // committed; unknown and retryable upstream failures may advance the combo. if (comboStreamPayloadCommitsOutput(payload) && !matchedBareError) outputCommitted = true; if (!payload || typeof payload !== "object" || Array.isArray(payload)) return; if (retryable) retryableTerminalPayload = payload as Record<string, unknown>; }, onTerminal: status => { terminalStatus = status; }, }); try { for (;;) { let next: Awaited<ReturnType<typeof reader.read>>; try { next = await reader.read(); } catch (error) { if (!options?.replayReadErrors) throw error; // The native relay still owns post-header transport failures. Preserve // the bounded prefix and the errored reader; cancelling it here would // erase the failure before either client relay or inspection sees it. const replay = replayBufferedResponse(response, reader, buffered); const stage = observedResponsesStage({ outputCommitted, terminalStatus, responseCreated }); return { kind: "read-error", response: replay, error, stage }; } if (next.done) { inspector.finish(); } else { if (bufferedBytes + next.value.byteLength > COMBO_STREAM_PREFLIGHT_MAX_BYTES) { // Keep the cap about memory the preflight allocates. The upstream chunk already exists; // copying it before committing would transiently exceed the boundary for no // replay benefit. Preserve it unsliced behind the already-bounded prefix. return { kind: "accepted", response: replayBufferedResponse(response, reader, [...buffered, next.value]), }; } const retained = next.value.slice(); buffered.push(retained); bufferedBytes += retained.byteLength; inspector.feed(retained); } // A bare error event is not a protocol terminal (terminalStatus stays undefined), // so its retryable classification doubles as the terminal evidence. if ((terminalStatus === "failed" || terminalStatus === "incomplete" || retryableTerminalPayload?.type === "error") && !outputCommitted && retryableTerminalPayload) { await reader.cancel("retrying zero-output combo stream terminal").catch(() => undefined); return { kind: "failed", response: failedTerminalResponse(response, retryableTerminalPayload, logCtx) }; } if (next.done || terminalStatus !== undefined || outputCommitted || bufferedBytes >= COMBO_STREAM_PREFLIGHT_MAX_BYTES || buffered.length >= COMBO_STREAM_PREFLIGHT_MAX_CHUNKS) { return { kind: "accepted", response: replayBufferedResponse(response, reader, buffered) }; } } } finally { inspector.dispose(); } } /** Produce a replacement body for a mid-stream failure at `stage`, or null to keep the error. */ export type ProtocolSafeResetRecovery = ( error: unknown, stage: RequestFailureStage, ) => Promise<Response | null>; /** * Defer protocol inspection until the downstream actually pulls the body. * * Direct passthrough must return response headers before the first SSE event arrives, so the * inspection cannot be awaited at the dispatch site the way combo routing awaits it. Wrapping * the body moves it to the first pull, which is the earliest moment the client is willing to * wait anyway. */ export function deferProtocolSafeResetRecovery( response: Response, logCtx: RequestLogContext, recover: ProtocolSafeResetRecovery, options?: { allowMissingContentType?: boolean }, ): Response { if (!response.body) return response; let reader: ReadableStreamDefaultReader<Uint8Array> | undefined; let initialization: Promise<void> | undefined; let closed = false; const cancelBody = (body: ReadableStream<Uint8Array> | null, reason?: unknown): void => { try { void body?.cancel(reason).catch(() => {}); } catch { /* already locked or closed */ } }; const initialize = async (): Promise<void> => { const preflight = await preflightComboStreamResponse( response, logCtx, () => false, { allowMissingContentType: options?.allowMissingContentType === true, replayReadErrors: true }, ); let selected = preflight.response; if (preflight.kind === "read-error") { const replacement = await recover(preflight.error, preflight.stage); if (replacement) { cancelBody(selected.body, "using protocol-safe replacement stream"); selected = replacement; } } if (closed) { cancelBody(selected.body, "downstream cancelled before protocol preflight completed"); return; } reader = selected.body?.getReader(); }; const body = new ReadableStream<Uint8Array>({ async pull(controller) { try { initialization ??= initialize(); await initialization; if (closed) return; if (!reader) { closed = true; controller.close(); return; } const next = await reader.read(); if (closed) return; if (next.done) { closed = true; try { reader.releaseLock(); } catch { /* already released */ } reader = undefined; controller.close(); return; } controller.enqueue(next.value); } catch (error) { if (closed) return; closed = true; try { reader?.releaseLock(); } catch { /* errored reader */ } reader = undefined; controller.error(error); } }, cancel(reason) { if (closed) return; closed = true; if (reader) { try { void reader.cancel(reason).catch(() => {}); } catch { /* already closed */ } try { reader.releaseLock(); } catch { /* already released */ } reader = undefined; } else if (initialization === undefined) { // Nothing has read the upstream yet, so this body is still ours to cancel. cancelBody(response.body, reason); } // A cancel while the preflight is mid-flight falls through deliberately. That body is // locked by the preflight's own reader, so cancelling it here would reject and be // swallowed; `initialize` sees `closed` when it settles and releases whichever body it // ended up selecting, which is the one that actually has to be let go. }, }, { highWaterMark: 0 }); return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers, }); }