UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

260 lines (245 loc) • 10.9 kB
import type { RequestLogContext } from "../request-log"; import type { HandleResponsesOptions } from "./core-options"; import { hasResponseMediaType } from "../../lib/http-response-semantics"; import { readDisplaySafeErrorText } from "./core-errors"; import { isNativePassthroughSseResponse } from "../relay"; /** * How long a streaming combo may spend choosing a child before the client gets a response head * (#8326). Below it nothing changes: the selected child's own head is returned, zero-output * failover stays invisible and an all-targets failure keeps its real HTTP status. Once it passes, * a 200 SSE head is committed so an idle-limited intermediary sees bytes, and the eventual * outcome is delivered inside that stream. It matches the native reset-selection heartbeat. */ export const COMBO_SELECTION_KEEPALIVE_THRESHOLD_MS = 15_000; /** Gap between keepalive comments while selection continues behind a committed head. */ export const COMBO_SELECTION_KEEPALIVE_INTERVAL_MS = 15_000; const KEEPALIVE = new TextEncoder().encode(": keepalive\n\n"); /** Proxy-owned copy for a ladder that threw after the head was committed; never the error text. */ const THROWN_SELECTION_MESSAGE = "opencodex could not complete this combo request"; /** Timer seam: production uses the global timers, tests advance a manual clock. */ export type ComboSelectionKeepaliveClock = NonNullable<NonNullable<HandleResponsesOptions["comboSelectionKeepalive"]>["clock"]>; export interface ComboSelectionKeepaliveHooks { /** The request's abort signal, also aborted when the client cancels the committed stream. */ signal: AbortSignal; /** True once the keepalive head has been handed to the client. */ headCommitted(): boolean; } const systemClock: ComboSelectionKeepaliveClock = { setTimeout: (callback, ms) => setTimeout(callback, ms), clearTimeout: handle => clearTimeout(handle as ReturnType<typeof setTimeout>), }; /** * Only a streaming Responses turn from the HTTP ingress opts in. WebSocket frames carry their own * status, Chat and Messages ingresses consume a different failure vocabulary, and internal * decision calls read the HTTP status themselves. */ export function comboSelectionKeepaliveApplies( rawBody: unknown, options: Pick<HandleResponsesOptions, "comboSelectionKeepalive" | "inboundTransport" | "inboundWire" | "internalDecisionCall">, ): boolean { return options.comboSelectionKeepalive !== undefined && (rawBody as { stream?: unknown } | null)?.stream === true && options.inboundTransport === undefined && (options.inboundWire === undefined || options.inboundWire === "responses") && options.internalDecisionCall !== true; } /** * The ladder's options behind a possible keepalive head. A committed head is an unmarked SSE * response, so the ingress request-log tap records its terminal; the native passthrough log * callbacks are dropped from then on so one request is never logged twice. */ export function comboSelectionKeepaliveLadderOptions<T extends HandleResponsesOptions>( options: T, hooks: ComboSelectionKeepaliveHooks, ): T { const { comboSelectionKeepalive: _optIn, ...rest } = options; const onTerminal = options.onNativePassthroughTerminal; const onCancel = options.onNativePassthroughCancel; return { ...rest, abortSignal: hooks.signal, ...(onTerminal ? { onNativePassthroughTerminal: status => { if (!hooks.headCommitted()) onTerminal(status); } } : {}), ...(onCancel ? { onNativePassthroughCancel: () => { if (!hooks.headCommitted()) onCancel(); } } : {}), } as T; } function isRecord(value: unknown): value is Record<string, unknown> { return value !== null && typeof value === "object" && !Array.isArray(value); } function failedFrame(model: string, error: { type?: string; code: string; message: string }): Uint8Array { const failure = { ...(error.type ? { type: error.type } : {}), code: error.code, message: error.message }; const payload = { type: "response.failed", sequence_number: 0, response: { id: `resp_${crypto.randomUUID().replaceAll("-", "")}`, object: "response", created_at: Math.floor(Date.now() / 1000), status: "failed", model, output: [], error: failure, last_error: failure, }, }; return new TextEncoder().encode(`event: response.failed\ndata: ${JSON.stringify(payload)}\n\n`); } /** Restate a non-streamable selection (an error envelope) as the in-stream failure terminal. */ async function failureFromResponse(response: Response, signal: AbortSignal): Promise<{ type?: string; code: string; message: string }> { const fallback = `opencodex combo request failed with HTTP ${response.status}`; const text = await readDisplaySafeErrorText(response, signal, ""); let record: Record<string, unknown> | undefined; try { const parsed: unknown = JSON.parse(text); if (isRecord(parsed)) record = isRecord(parsed.error) ? parsed.error : parsed; } catch { /* not JSON: the status-only fallback below applies */ } const type = typeof record?.type === "string" && record.type ? record.type : undefined; const code = typeof record?.code === "string" && record.code ? record.code : typeof record?.code === "number" && Number.isFinite(record.code) ? String(record.code) : type ?? (response.ok ? "invalid_response" : "upstream_error"); const message = typeof record?.message === "string" && record.message.trim() ? record.message : fallback; return { ...(type ? { type } : {}), code, message }; } /** * Run combo selection and return its response, or a keepalive SSE response once selection has * waited past the threshold. Behind the keepalive head, an accepted SSE child is streamed as is and * any other outcome becomes one `response.failed` with the same error code. */ export function selectComboWithKeepalive( select: (hooks: ComboSelectionKeepaliveHooks) => Promise<Response>, context: { model: string; logCtx: RequestLogContext; signal?: AbortSignal; clock?: ComboSelectionKeepaliveClock }, ): Promise<Response> { const clock = context.clock ?? systemClock; const cancel = new AbortController(); const signal = context.signal ? AbortSignal.any([context.signal, cancel.signal]) : cancel.signal; let committed = false; let settled = false; let closed = false; let keepaliveOwed = true; let source: ReadableStreamDefaultReader<Uint8Array> | undefined; let wake: (() => void) | undefined; let timer: unknown; const stopTimer = (): void => { if (timer !== undefined) clock.clearTimeout(timer); timer = undefined; }; const notify = (): void => { const next = wake; wake = undefined; next?.(); }; const setSource = (stream: ReadableStream<Uint8Array> | null): void => { if (closed) { void stream?.cancel("client cancelled the committed combo stream").catch(() => {}); return; } source = (stream ?? new ReadableStream<Uint8Array>({ start: controller => controller.close() })).getReader(); notify(); }; const deliverFailure = (status: number, failure: { type?: string; code: string; message: string }): void => { context.logCtx.terminalHttpStatus = status; const frame = failedFrame(context.model, failure); setSource(new ReadableStream<Uint8Array>({ start(controller) { controller.enqueue(frame); controller.close(); } })); }; const deliver = async (response: Response): Promise<void> => { if (closed) { void response.body?.cancel("client cancelled the committed combo stream").catch(() => {}); return; } if (response.ok && hasResponseMediaType(response.headers, "text/event-stream")) { // The outer tap already exists when selection finishes. Decide from the selected // response's process-owned marker before its first byte reaches that tap. The // native child's original inspector still publishes real usage through its callback. context.logCtx.clientSseUsageAlreadyInspected = isNativePassthroughSseResponse(response); setSource(response.body); return; } if (response.status === 499) { void response.body?.cancel().catch(() => {}); setSource(null); return; } const failure = await failureFromResponse(response, signal); deliverFailure(response.ok ? 502 : response.status, failure); }; const body = new ReadableStream<Uint8Array>({ async pull(controller) { try { for (;;) { if (closed) return; if (source) { const next = await source.read(); if (closed) return; if (next.done) { closed = true; controller.close(); } else { controller.enqueue(next.value); } return; } if (keepaliveOwed) { keepaliveOwed = false; controller.enqueue(KEEPALIVE); return; } await new Promise<void>(resolve => { wake = resolve; }); } } catch (error) { if (closed) return; closed = true; controller.error(error); } }, cancel(reason) { closed = true; stopTimer(); cancel.abort(reason); notify(); if (source) void source.cancel(reason).catch(() => {}); }, }, { highWaterMark: 0 }); const scheduleKeepalive = (): void => { timer = clock.setTimeout(() => { timer = undefined; if (closed || settled) return; keepaliveOwed = true; notify(); scheduleKeepalive(); }, COMBO_SELECTION_KEEPALIVE_INTERVAL_MS); }; return new Promise<Response>((resolve, reject) => { const selection = select({ signal, headCommitted: () => committed }); timer = clock.setTimeout(() => { timer = undefined; // A cancelled request is about to settle as a client cancellation; it needs no head. if (settled || signal.aborted) return; committed = true; scheduleKeepalive(); resolve(new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", Connection: "keep-alive", "X-Accel-Buffering": "no", }, })); }, COMBO_SELECTION_KEEPALIVE_THRESHOLD_MS); selection.then(response => { settled = true; stopTimer(); if (!committed) { resolve(response); return; } deliver(response).catch(() => { deliverFailure(502, { type: "server_error", code: "server_error", message: THROWN_SELECTION_MESSAGE }); }); }, error => { settled = true; stopTimer(); if (!committed) { reject(error); return; } console.warn(`[combo] selection failed after its keepalive head: ${error instanceof Error ? error.name : typeof error}`); deliverFailure(502, { type: "server_error", code: "server_error", message: THROWN_SELECTION_MESSAGE }); }); }); }