UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

254 lines (242 loc) • 13.4 kB
import { createHash } from "node:crypto"; import type { OcxProviderConfig } from "../types"; import { isEgressTransparentExecutor, markEgressTransparentExecutor } from "../lib/provider-egress"; import { isPassiveKeyTransparent, markPassiveKeyTransparent, markPassiveKeyDeferred, PASSIVE_KEY_SEND } from "../lib/passive-key-fetch-proof"; import { configuredOutboundFetch } from "../lib/proxy-env"; import { waitOutsideProviderHeaderDeadline } from "../lib/provider-admission-wait"; import { markSameTargetRetryHold, sleepWithAbort } from "../lib/upstream-retry"; import { RequestPacingQueueOverloadError } from "./request-pacing"; import type { OcxProviderTransport } from "./xai-transport"; const MAX_IDENTITIES = 128; /** Queued waiters per credential+model identity; one hot identity never refuses another. */ const MAX_WAITERS_PER_IDENTITY = 128; /** * Process-wide waiter bound. Model strings come from clients, so identities are not bounded by * configuration; this keeps memory and pending requests finite. It is a multiple of the * per-identity cap, so no single identity can fill it alone. */ const MAX_WAITERS_TOTAL = 4 * MAX_WAITERS_PER_IDENTITY; const DEFAULT_MAX_WAIT_MS = 60_000; // Existing retry wait budget, not an inferred rate window. const MINUTE_MS = 60 * 1000; // Unit named by the observed headers, never an account quota. type RecoveryState = { generation: number; probe?: { done: Promise<void>; finish(): void } }; /** Match the supported key-auth Chat destination, including one optional trailing slash. */ export function isCanonicalMistralChat( provider: Pick<OcxProviderConfig, "adapter" | "authMode" | "baseUrl">, ): boolean { if (provider.adapter !== "openai-chat" || (provider.authMode !== undefined && provider.authMode !== "key")) return false; try { const url = new URL(provider.baseUrl); return url.origin === "https://api.mistral.ai" && /^\/v1\/?$/.test(url.pathname) && !url.username && !url.password && !url.search && !url.hash; } catch { return false; } } /** Read a nonnegative integer counter; reject missing or malformed values. */ function count(headers: Headers, name: string): number | undefined { const raw = headers.get(name)?.trim(); if (!raw || !/^\d+$/.test(raw)) return undefined; const value = Number(raw); return Number.isSafeInteger(value) ? value : undefined; } /** Read the server delay as seconds or an HTTP date; ignore invalid hints. */ function retryDelay(headers: Headers, now: number): number | undefined { const raw = headers.get("retry-after")?.trim(); if (!raw) return undefined; const value = /^\d+(?:\.\d+)?$/.test(raw) ? Number(raw) * 1000 : Date.parse(raw) - now; return Number.isFinite(value) ? Math.max(0, value) : undefined; } /** Classify measured refusal evidence only; never predict a future cache hit or token charge. */ export function mistralMinuteRefusal(headers: Headers): "temporary" | "charge-above-allowance" | "unknown" { const month = count(headers, "x-ratelimit-limit-tokens-month"); if (month !== undefined && count(headers, "x-ratelimit-remaining-tokens-month") === 0) return "unknown"; const limit = count(headers, "x-ratelimit-limit-tokens-minute"); const remaining = count(headers, "x-ratelimit-remaining-tokens-minute"); const cost = count(headers, "x-ratelimit-tokens-query-cost"); if (limit !== undefined && limit > 0 && cost !== undefined && cost > limit) return "charge-above-allowance"; if (limit !== undefined && limit > 0 && remaining !== undefined && remaining <= limit && (remaining === 0 || (cost !== undefined && cost > remaining))) return "temporary"; const requests = count(headers, "x-ratelimit-limit-req-minute"); if (requests !== undefined && requests > 0 && count(headers, "x-ratelimit-remaining-req-minute") === 0) return "temporary"; return "unknown"; } /** Process-local observations, never a fixed account/plan quota or a persisted credential. */ export class MistralBudgetPacer { private readonly deadlines = new Map<string, number>(); private readonly recoveries = new Map<string, RecoveryState>(); /** Latest live deadline evicted at capacity; unknown identities wait it out instead of sending early. */ private evictedUntil = 0; /** * Parked waiters keyed by identity. A request counts only once it is about to wait, so a held * credential or model fills only its own queue and a request that can send now is never answered * queue_full (HTTP 429). The map holds identities with live waiters only; the total is capped by * MAX_WAITERS_TOTAL. */ private readonly waiting = new Map<string, number>(); private waitingTotal = 0; constructor( private readonly now: () => number = Date.now, private readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void> = sleepWithAbort, ) {} /** Record server delays and confirmed minute pressure for this credential and model. */ observe(identity: string, headers: Headers, status = 200): void { const tokens = count(headers, "x-ratelimit-remaining-tokens-minute"); const requests = count(headers, "x-ratelimit-remaining-req-minute"); // Healthy remaining counters never predict the next request's cache/charge. if (status !== 429 && tokens !== 0 && requests !== 0) return; // With no explicit time, activate a conservative minute hold only AFTER a // refusal establishes pressure. Healthy responses never predict the next cost. const delay = retryDelay(headers, this.now()) ?? (status === 429 && mistralMinuteRefusal(headers) === "temporary" ? MINUTE_MS : undefined); if (delay === undefined || delay === 0) return; for (const [key, deadline] of this.deadlines) { if (deadline <= this.now()) this.deadlines.delete(key); } if (!this.deadlines.has(identity) && this.deadlines.size >= MAX_IDENTITIES) { const [evicted, until] = this.deadlines.entries().next().value!; this.evictedUntil = Math.max(this.evictedUntil, until); this.deadlines.delete(evicted); } this.deadlines.set(identity, Math.max(this.deadlines.get(identity) ?? 0, this.now() + delay)); if (status === 429) { if (!this.recoveries.has(identity) && this.recoveries.size >= MAX_IDENTITIES) { const idle = [...this.recoveries].find(([, state]) => !state.probe); if (!idle) return; this.recoveries.delete(idle[0]); } const state = this.recoveries.get(identity) ?? { generation: 0 }; state.generation++; this.recoveries.set(identity, state); } } /** Wait within the admission budget and return a release callback for one recovery probe. */ async acquire(identity: string, signal?: AbortSignal, maxWaitMs = DEFAULT_MAX_WAIT_MS): Promise<(success?: boolean) => void> { signal?.throwIfAborted(); const deadline = this.now() + maxWaitMs; let parked = false; const park = (): void => { if (parked) return; const queued = this.waiting.get(identity) ?? 0; if (queued >= MAX_WAITERS_PER_IDENTITY || this.waitingTotal >= MAX_WAITERS_TOTAL) { throw new RequestPacingQueueOverloadError("mistral", "queue_full", 60); } this.waiting.set(identity, queued + 1); this.waitingTotal++; parked = true; }; try { for (;;) { signal?.throwIfAborted(); // An evicted identity may have re-entered with a shorter delay; the evicted one still binds. const live = this.evictedUntil > this.now() ? this.evictedUntil : undefined; const own = this.deadlines.get(identity); const readyAt = own === undefined ? live : live === undefined ? own : Math.max(own, live); if (readyAt !== undefined && readyAt > deadline) { const delay = readyAt - this.now(); throw new RequestPacingQueueOverloadError("mistral", "queue_expired", Math.ceil(delay / 1000)); } if (readyAt !== undefined && readyAt > this.now()) { park(); await this.sleep(readyAt - this.now(), signal); continue; } const state = this.recoveries.get(identity); if (!state) { this.deadlines.delete(identity); return () => {}; } if (state.probe) { const remaining = deadline - this.now(); if (remaining <= 0) throw new RequestPacingQueueOverloadError("mistral", "queue_expired", 1); park(); const local = new AbortController(); try { await Promise.race([state.probe.done, this.sleep(remaining, AbortSignal.any([local.signal, ...(signal ? [signal] : [])]))]); } finally { local.abort(); } continue; } let finish!: () => void; const probe = { done: new Promise<void>(resolve => { finish = resolve; }), finish: () => finish() }; state.probe = probe; const generation = state.generation; let released = false; return success => { if (released) return; released = true; if (state.probe === probe) delete state.probe; // Only this probe can settle recovery; an older concurrent success must // not erase a more recent refusal or the server's minimum wait. if (success && state.generation === generation) { if (this.recoveries.get(identity) === state) this.recoveries.delete(identity); if ((this.deadlines.get(identity) ?? 0) <= this.now()) this.deadlines.delete(identity); } probe.finish(); }; } } finally { if (parked) { this.waitingTotal--; const left = (this.waiting.get(identity) ?? 1) - 1; if (left > 0) this.waiting.set(identity, left); else this.waiting.delete(identity); } } } } const pacer = new MistralBudgetPacer(); const wrapped = new WeakSet<typeof globalThis.fetch>(); /** Wrap canonical Mistral Chat sends with bounded admission and server-delay protection. */ export function resolveMistralTransport( provider: OcxProviderTransport, budgetPacer = pacer, ): OcxProviderTransport { if (!isCanonicalMistralChat(provider) || (provider.fetch && wrapped.has(provider.fetch))) return provider; const baseFetch = provider.fetch ?? markEgressTransparentExecutor(((input, init) => configuredOutboundFetch(input, init)) as typeof globalThis.fetch); if (!provider.fetch) markPassiveKeyTransparent(baseFetch); const attemptFetch = (async (input, init) => { const send = () => { const observe = (init as (RequestInit & { [PASSIVE_KEY_SEND]?: (input: RequestInfo | URL, init: RequestInit) => void }) | undefined)?.[PASSIVE_KEY_SEND]; if (!observe) return baseFetch(input, init); const { [PASSIVE_KEY_SEND]: _hook, ...physicalInit } = init! as RequestInit & { [PASSIVE_KEY_SEND]?: typeof observe }; observe?.(input, physicalInit); return baseFetch(input, physicalInit); }; const url = new URL(input instanceof Request ? input.url : String(input)); if (url.origin !== "https://api.mistral.ai" || url.pathname !== "/v1/chat/completions" || url.search || url.hash || init?.method?.toUpperCase() !== "POST" || typeof init.body !== "string") { return send(); } // The Chat adapter serializes model first. Read only that bounded prefix rather // than parsing/copying a potentially million-token conversation a second time. const encodedModel = /^\s*\{\s*"model"\s*:\s*("(?:[^"\\]|\\.)*")\s*[,}]/.exec(init.body.slice(0, 4096))?.[1]; let model: unknown; try { model = encodedModel === undefined ? undefined : JSON.parse(encodedModel); } catch { return send(); } const authorization = new Headers(init.headers).get("authorization"); if (!authorization || typeof model !== "string" || !model || model.length > 256) return send(); // Bind to the actual send credential, including after pool rotation; keep no raw key. const identity = createHash("sha256").update(JSON.stringify([authorization, model])).digest("hex"); const pacing = provider.requestPacing?.enabled !== false && provider.retryOn429?.respectRetryAfter !== false; const release = pacing ? await waitOutsideProviderHeaderDeadline(init, signal => budgetPacer.acquire(identity, signal, provider.retryOn429?.maxIntervalMs)) : undefined; let response: Response; try { init.signal?.throwIfAborted(); response = await send(); if (pacing) budgetPacer.observe(identity, response.headers, response.status); } catch (error) { release?.(); throw error; } release?.(response.ok); const retry = provider.retryOn429; if (response.status === 429 && retry?.respectRetryAfter !== false) { const delay = retryDelay(response.headers, Date.now()); if (delay !== undefined && delay > (retry?.maxIntervalMs ?? DEFAULT_MAX_WAIT_MS)) { // The shared retry helper caps waits. Stop same-key replays sooner than Mistral permits; // another credential may still serve, and the original status/body/Retry-After reach the caller. markSameTargetRetryHold(response); } } return response; }) as typeof globalThis.fetch; if (isEgressTransparentExecutor(baseFetch)) markEgressTransparentExecutor(attemptFetch); if (isPassiveKeyTransparent(baseFetch)) { markPassiveKeyTransparent(attemptFetch); markPassiveKeyDeferred(attemptFetch); } wrapped.add(attemptFetch); return { ...provider, fetch: attemptFetch }; }