UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

450 lines (428 loc) • 22.1 kB
import type { NativeResponseControl } from "./native-response-control"; import type { Server } from "bun"; import { codexWsUpstreamFetch, currentBunRuntimeIdentity, shouldUseCodexWsUpstream, type BunRuntimeGateInput, } from "./ws-upstream"; import type { OcxProviderConfig } from "../../types"; import type { WsData } from "../ws-bridge"; import { requestPacingMaxConcurrentRequests, sendTrackingRequestSlot, waitForProviderRequestSlot, type ProviderRequestSlot, } from "../../providers/request-pacing"; import { withUpstreamHttpVersion } from "../../lib/upstream-http-version"; import type { CodexWsQuotaObserver } from "./codex-ws-metadata"; import { configuredOutboundFetch } from "../../lib/proxy-env"; import { isLoopbackUrl, rewriteUpstream } from "../../plugins/upstream-hooks"; import { describeProviderEgressForLog, markEgressTransparentExecutor, providerEgressSendInit, providerEgressIsExplicit, resolveProviderEgress, type ProviderEgressBinding, } from "../../lib/provider-egress"; import { redactSecretString } from "../../lib/redact"; export { withUpstreamHttpVersion }; const egressWebsocketDowngradeWarned = new Set<string>(); /** A provider name is configuration-controlled, so the notice set is bounded like any cache. */ const EGRESS_DOWNGRADE_NOTICE_LIMIT = 64; const pacingWebsocketDowngradeWarned = new Set<string>(); /** * Marks an init whose provider egress route an outer physical-send boundary already decided. * * Own symbol keys survive object spread, so the mark travels through the rebuild a * `dispatchOverride` performs, and an unknown symbol on a `RequestInit` is inert at the wire. */ const EGRESS_DECIDED = Symbol.for("opencodex.provider-egress.decided"); const UPSTREAM_REWRITTEN = Symbol.for("opencodex.plugins.upstream-rewritten"); /** * Announce once, per provider, that an explicit egress route moved this provider off the * WebSocket fast lane. * * The WebSocket upstream selects its proxy from the process environment when it dials, so it * cannot carry a per-provider route. Serving the turn over HTTP/SSE honours the operator's * egress choice, which is the one that has to win — but a transport change the operator did * not ask for is exactly the kind of substitution this batch refuses to make silently, so it * is stated rather than merely done. */ function warnEgressWebsocketDowngradeOnce(providerName: string, egress: string): void { if (egressWebsocketDowngradeWarned.has(providerName)) return; if (egressWebsocketDowngradeWarned.size >= EGRESS_DOWNGRADE_NOTICE_LIMIT) return; egressWebsocketDowngradeWarned.add(providerName); console.warn( // The name is caller-controlled and can be token-shaped, so it is redacted and JSON-escaped // before it reaches a log, exactly as at the management error boundary. `[opencodex] provider ${JSON.stringify(redactSecretString(providerName))} declares egress ${egress}; the WebSocket upstream ` + "selects its proxy from the process environment and cannot carry a per-provider route, " + "so these turns are served over HTTP/SSE.", ); } /** Test seam: the downgrade notice is once per provider per process, not once per request. */ export function __resetEgressWebsocketDowngradeNotices(): void { egressWebsocketDowngradeWarned.clear(); pacingWebsocketDowngradeWarned.clear(); } export function disableResponsesRequestTimeout(req: Request, server: Pick<Server<WsData>, "timeout"> | undefined): boolean { if (!server) return false; try { server.timeout(req, 0); return true; } catch { return false; } } export function safeHostLabel(url: string): string { try { return new URL(url).host; } catch { return "upstream"; } } /** Canonical origin (scheme + host) for failure-attribution keys: http and * https for the same host must not share one ledger entry (#914 review). */ export function safeOriginLabel(url: string): string { try { return new URL(url).origin.toLowerCase(); } catch { return "upstream"; } } /** * Check whether a target host should bypass Bun's keep-alive pool reuse. * Configured via the `OCX_FRESH_CONNECTION_HOSTS` environment variable (comma-separated). */ export function wantsFreshConnection( input: Parameters<typeof globalThis.fetch>[0], hostsEnv = process.env.OCX_FRESH_CONNECTION_HOSTS, ): boolean { if (!hostsEnv) return false; try { const rawUrl = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; const host = new URL(rawUrl).hostname.toLowerCase(); const targets = hostsEnv .split(",") .map(h => h.trim().toLowerCase().replace(/^\.+/, "")) .filter(Boolean); for (const target of targets) { if (host === target || host.endsWith(`.${target}`)) return true; } } catch { /* unparseable target URL keeps default connection behavior */ } return false; } export interface PaceAwareFetch { waitForPacing?: (signal?: AbortSignal) => Promise<ProviderRequestSlot | undefined>; unpacedFetch?: typeof globalThis.fetch; } export type ProviderFetch = typeof globalThis.fetch & PaceAwareFetch; /** * Apply the physical-send connection policy to whichever fetch actually performs the send. * * The executor `providerFetch` builds is not the only physical boundary. A `dispatchOverride` * that revalidates credentials re-reads `route.provider.fetch` at send time -- reselection can * install a different provider transport after this wrapper was constructed -- and then calls * that fetch directly instead of the supplied executor. Keeping the policy inside the executor * alone therefore left every provider-scoped transport reusing a pooled socket for a host the * operator had named in `OCX_FRESH_CONNECTION_HOSTS` (#4992). The policy belongs around the * selected fetch so it follows the selection rather than the construction. * * Idempotent on purpose: an override that hands the send back to the supplied executor passes * through here twice, and both passes derive the same headers from the same wire URL. */ export function sendWithConnectionPolicy( physicalFetch: typeof globalThis.fetch, rawInput: Parameters<typeof globalThis.fetch>[0], init?: RequestInit, egress?: ProviderEgressBinding, ): Promise<Response> { let input = rawInput; let headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); // Plugin rewrites (src/plugins/upstream-hooks.ts) run here, after the caller chose between // the Codex WebSocket and HTTP, and before the connection and egress decisions below so // those follow the rewritten destination. Nested passes rewrite once, like the egress mark. // A rewrite onto this machine's loopback dials directly: a proxy chosen for the provider // (per-provider or HTTP_PROXY) cannot reach a local sidecar. The WebSocket dial does the same. const rewriteDone = (init as Record<symbol, unknown> | undefined)?.[UPSTREAM_REWRITTEN] === true; let redirectedToLoopback = false; if (!rewriteDone) { const original = input instanceof Request ? input.url : String(input); const target = rewriteUpstream(original, headers, "http"); headers = target.headers as Headers; if (target.url !== original) { redirectedToLoopback = isLoopbackUrl(target.url); input = input instanceof Request ? new Request(target.url, input) : target.url; } } const fresh = wantsFreshConnection(input); if (fresh) { headers.set("Connection", "close"); } // Decided here, against the destination this send is actually going to, and around whichever // executor was just selected. A `dispatchOverride` that rebuilds a queued request can change // both the upstream host and the provider transport after the wrapper was constructed, so a // route resolved at construction could be applied to a different host than it was decided for. // These calls nest: an override decides with its own binding and then hands the send to the // executor `providerFetch` supplied, which is another one of these. The outermost caller holds // the reselected provider and the rebuilt destination, so it decides and marks the init; the // inner pass honours that mark rather than recomputing from a stale closure. const alreadyDecided = (init as Record<symbol, unknown> | undefined)?.[EGRESS_DECIDED] === true; const decide = egress !== undefined && !alreadyDecided && !redirectedToLoopback; const egressInit = redirectedToLoopback ? { proxy: false as const } : decide ? providerEgressSendInit(egress, physicalFetch, input) : {}; return physicalFetch(input, { ...init, headers, redirect: "manual", ...(fresh ? { keepalive: false } : {}), ...egressInit, ...(decide || redirectedToLoopback ? { [EGRESS_DECIDED]: true } : {}), ...{ [UPSTREAM_REWRITTEN]: true }, }); } export interface ProviderFetchOptions { /** * Keep this send on HTTP even where the WebSocket upstream would normally be selected. * * Set by a caller replacing an HTTP stream that already failed: a WS create frame is a * different send on a different transport, and the replacement has to be the same kind of * exchange the client is already reading. */ httpOnly?: boolean; nativeControl?: NativeResponseControl; providerName?: string; modelId?: string; /** One pacing slot was acquired immediately before this fetch wrapper was created. */ pacingSlotAcquired?: boolean; pacingSlot?: ProviderRequestSlot; /** A runTurn transport holds one concurrency lease across overlapping physical sends. */ turnScopedPacing?: boolean; /** Captured selected-account observer, attached before the native WS send. */ onCodexWsQuota?: CodexWsQuotaObserver; /** Synchronous admission at actual credential dispatch, after pacing/backoff. */ beforeDispatch?: (headers: Headers) => void; /** Revalidate/rebuild a queued request at its physical send boundary, after pacing. */ dispatchOverride?: (input: Parameters<typeof globalThis.fetch>[0], init: RequestInit, execute: typeof globalThis.fetch) => Promise<Response>; } export function providerFetch( provider: OcxProviderConfig, runtime: BunRuntimeGateInput = currentBunRuntimeIdentity(), options: ProviderFetchOptions = {}, ): ProviderFetch { const providerName = options.providerName ?? "<unnamed provider>"; const customExecutor = (provider as OcxProviderConfig & { fetch?: typeof globalThis.fetch }).fetch; // The route is applied at the physical send (see `sendWithConnectionPolicy`). This binding is // only what that boundary needs to decide it. const egressBinding: ProviderEgressBinding = { providerName, provider }; // Resolved per request, not once per wrapper: `providers.<name>.noProxy` is evaluated against // the destination, so two requests through the same executor can legitimately take different // routes. A malformed value throws and rejects the request rather than degrading to the // global proxy or to direct, either of which would read as success at the call site. const egressFor = (input: Parameters<typeof globalThis.fetch>[0]) => resolveProviderEgress({ providerName, provider, url: typeof input === "string" ? input : input instanceof URL ? input : input.url, }); // The built-in executor forwards its init to a transport that honours the proxy option. const configuredFetch = markEgressTransparentExecutor(Object.assign( (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => configuredOutboundFetch(input, init), { preconnect: globalThis.fetch.preconnect?.bind(globalThis.fetch) }, ) as typeof globalThis.fetch); const base = customExecutor ?? configuredFetch; const preconnect = (...args: Parameters<typeof globalThis.fetch.preconnect>): void => { base.preconnect?.(...args); }; // Rebuilt dispatches must use the same physical-send boundary as ordinary HTTP sends. // Return the original 3xx so the response owner retains its retry/health/relay contract. // // Marked transparent because it forwards its init to a transport that honours the proxy // option. Leaving it unmarked would make an ordinary configured provider refuse its own route // on every overridden path, after the attempt had already been recorded — an override selects // `provider.fetch ?? execute`, and `execute` is this wrapper. It still carries the binding, so // an override that simply calls it gets the route decided rather than dropped; an override // that decided for itself has already marked the init and this pass defers to that decision. const dispatch = markEgressTransparentExecutor(Object.assign( (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => sendWithConnectionPolicy(base, input, init, egressBinding), { preconnect }, ) as typeof globalThis.fetch); const httpFetch = Object.assign( async (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => { // Refuse before any dispatch side effect where that is sound. `beforeDispatch` commits // attempt accounting and consumes admission state, so a refusal firing after it would // charge an attempt for a send that never happens, and a throwing hook would mask the // egress error with an unrelated one. // // With no override, this input and `base` ARE the final destination and executor, so the // full decision can be made now. With an override, only the configured value is checked: // the override may rebuild against a different host and select a different transport, and // refusing on this destination would reject a request whose real route is fine. if (options.dispatchOverride) egressFor(input); else providerEgressSendInit(egressBinding, base, input); // The hook inspects the outgoing headers and refuses the send by throwing; it is not a // mutator, and the copy it receives is deliberately not threaded onward. `Connection` // is decided inside `dispatch`, which runs after this, so the fresh-connection policy // wins regardless of what any caller or hook put in the header. options.beforeDispatch?.(new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined))); // No proxy option is attached here: a `dispatchOverride` may rebuild this request against // a different destination, so the route is decided at the physical send instead. const dispatchInit = { ...withUpstreamHttpVersion(input, init, provider), timeout: 0 }; return options.dispatchOverride ? options.dispatchOverride(input, dispatchInit, dispatch) : dispatch(input, dispatchInit); }, { preconnect }, ) as typeof globalThis.fetch; // ChatGPT Codex backend: streaming turns ride the responses_websockets // transport (measured ~3s faster TTFT than the SSE POST queue); everything // else keeps the provider's HTTP fetch. See ws-upstream.ts for the details. const unpaced = async (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => { const upstreamWebsocket = provider.upstreamWebsocket; if (!options.httpOnly && typeof input === "string" && init && shouldUseCodexWsUpstream(input, init, runtime, upstreamWebsocket)) { if (options.providerName && requestPacingMaxConcurrentRequests(provider, options.modelId) > 0) { if (!pacingWebsocketDowngradeWarned.has(providerName) && pacingWebsocketDowngradeWarned.size < EGRESS_DOWNGRADE_NOTICE_LIMIT) { pacingWebsocketDowngradeWarned.add(providerName); console.warn(`[opencodex] provider ${JSON.stringify(redactSecretString(providerName))} has a request concurrency cap; serving over HTTP/SSE.`); } return httpFetch(input, init); } const egress = egressFor(input); if (providerEgressIsExplicit(egress)) { warnEgressWebsocketDowngradeOnce(providerName, describeProviderEgressForLog(egress)); return httpFetch(input, init); } // The fallback has to be the same HTTP fetch the non-WS branch would have // used, protocol pin included: a WS turn that falls back is serving the // request over HTTP, and dropping the provider's `upstreamHttpVersion` // there would silently negotiate a transport the operator ruled out. return codexWsUpstreamFetch(input, init, httpFetch, runtime, options.onCodexWsQuota, options.beforeDispatch, options.nativeControl, async () => { (await waitForPacing(init.signal ?? undefined))?.release(); }); } return httpFetch(input, init); }; let pacingSlotAcquired = options.pacingSlotAcquired === true; const waitForPacing = async (signal?: AbortSignal): Promise<ProviderRequestSlot | undefined> => { if (pacingSlotAcquired) { pacingSlotAcquired = false; if (!options.pacingSlot && requestPacingMaxConcurrentRequests(provider, options.modelId) > 0) { throw new Error("A pre-acquired concurrency slot must be passed to providerFetch"); } // Cursor RunSSE and BidiAppend can overlap. The runTurn owner releases this slot when // the whole turn finishes; transferring it to RunSSE's body would admit another turn // while a BidiAppend from this one is still active. return options.turnScopedPacing ? undefined : options.pacingSlot; } if (options.providerName) { return waitForProviderRequestSlot(options.providerName, provider, options.modelId, signal, options.turnScopedPacing && options.pacingSlot?.leased && !options.pacingSlot.released ? { concurrency: false } : undefined); } if (requestPacingMaxConcurrentRequests(provider, options.modelId) > 0) { throw new Error("A provider name is required for request concurrency pacing"); } }; const wrapped = async (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => { return sendTrackingRequestSlot(await waitForPacing(init?.signal ?? undefined), () => unpaced(input, init)); }; // The returned wrapper forwards its init down to `dispatch`, which applies the route at the // physical send. Adapters that hand this executor back as `provider.fetch` (Cursor does) // therefore still carry a per-provider route instead of being refused as opaque. const paceAware = Object.assign(wrapped, { preconnect, waitForPacing, unpacedFetch: Object.assign(unpaced, { preconnect }), }); markEgressTransparentExecutor(paceAware as unknown as typeof globalThis.fetch); return paceAware; } /** * Wrap a provider fetch so `onDispatch` fires immediately before the send, not before pacing. * * `fetchWithHeaderTimeout` awaits `waitForPacing` and only then calls the executor, so a caller * that signals at the call site records a dispatch even when a rejected pacing wait means nothing * reached the network. That matters when the signal bounds later recovery: the request would lose * its fallback on the strength of a send that never happened. * * The pacing surface is preserved deliberately. `waitForPacing` and `unpacedFetch` are read off * the executor by `fetchWithHeaderTimeout`, so a plain function wrapper would silently drop * provider pacing and double-send the slot. */ export function storedPoolReplayDispatchNotifier( executor: ProviderFetch, onDispatch: (() => void) | undefined, ): ProviderFetch { if (!onDispatch) return executor; let notified = false; const notifyOnce = (): void => { if (notified) return; notified = true; onDispatch(); }; const unpacedSource = executor.unpacedFetch ?? executor; const unpaced = Object.assign( (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => { notifyOnce(); return unpacedSource(input, init); }, { preconnect: unpacedSource.preconnect }, ) as ProviderFetch["unpacedFetch"]; const wrapped = async (input: Parameters<typeof globalThis.fetch>[0], init?: RequestInit) => { return sendTrackingRequestSlot(await executor.waitForPacing?.(init?.signal ?? undefined), () => unpaced!(input, init)); }; return Object.assign(wrapped, { preconnect: executor.preconnect, waitForPacing: executor.waitForPacing, unpacedFetch: unpaced, }) as ProviderFetch; } /** * Fetch through the header deadline with redirects always manual. * @param _manualRedirect Ignored; retained for call compatibility. Even false uses manual. */ export async function fetchWithHeaderTimeout( url: string, init: Omit<RequestInit, "signal">, abortSignal: AbortSignal, timeoutMs: number, preferIdentityEncoding = false, executor: typeof globalThis.fetch = globalThis.fetch, // Retained for existing callers; credential-bearing transport no longer opts out. _manualRedirect = false, ): Promise<Response> { const pacing = executor as ProviderFetch; const slot = await pacing.waitForPacing?.(abortSignal); const fetchExecutor = pacing.unpacedFetch ?? executor; let timer: ReturnType<typeof setTimeout> | undefined; try { const timeout = new AbortController(); timer = setTimeout(() => { if (!timeout.signal.aborted) timeout.abort(new DOMException("Timeout elapsed", "TimeoutError")); }, timeoutMs); const headers = new Headers(init.headers); // Compressed SSE can be held until the decompressor has a complete block. Streaming calls // default to identity for low-latency frame delivery, while an explicit caller choice wins. if (preferIdentityEncoding && !headers.has("accept-encoding")) { headers.set("accept-encoding", "identity"); } return await sendTrackingRequestSlot(slot, () => fetchExecutor(url, { ...init, headers, // Never replay provider credentials or request bodies to a redirect destination. // Preserve the 3xx for the owner's existing response/health policy (#914, #1471). redirect: "manual", signal: AbortSignal.any([abortSignal, timeout.signal]), timeout: 0, })); } finally { if (timer) clearTimeout(timer); if (!slot?.bodyTracked) slot?.release(); } }