UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

351 lines (342 loc) • 22.5 kB
import { clientFacingModelText } from "../../lib/error-destination"; import type { AdapterEvent, OcxConfig } from "../../types"; import { carryResponseToolOwnership } from "./response-tool-ownership"; import { routeConcreteModel, type RouteResult } from "../../router"; import { copyPlainData } from "../../lib/plain-data"; import { jsonUtf8Bytes } from "../../lib/json-byte-size"; import type { TranslatorBudget } from "../../lib/translator-budget"; import { readBoundedResponseBytes } from "../../lib/bounded-body"; import { hasResponseMediaType } from "../../lib/http-response-semantics"; import { isRequestExecutionBudget, type RequestExecutionBudget, type SingleUseDispatchPermit } from "../../lib/request-execution-budget"; import { isNonReplayableResponse, isNonReplayableUpstreamCode, markResponseNonReplayable, TRANSIENT_RETRY_MAX_ATTEMPTS } from "../../lib/upstream-retry"; import { isCyberPolicyCode, isTerminalRefusalCode } from "../../lib/errors"; import { isCanonicalOpenAiForwardProvider, supportsNativeResponsesCompactEndpoint, supportsNativeResponsesCompactionTrigger } from "../../providers/openai-tiers"; import { bridgeToResponsesSSE, formatErrorResponse } from "../../bridge"; import { buildCompactV1Output, decodeCompactionSummary, encodeCompactionSummary, extractCompactUserMessages } from "../../responses/compaction"; import { decodeRetainedCompaction, encodeRetainedCompaction } from "../../responses/retained-compaction"; import { collectReasoningTexts } from "../../responses/reasoning-retention"; import { finishRequestAttempt, usageFromResponsesPayload, type RequestLogContext } from "../request-log"; import { restoreLocalFailureState, takeLocalFailureState } from "../request-log-failure-detail"; import { linkRequestSessionLane } from "../request-log-conversation"; import { isNativePassthroughSseResponse, markNativePassthroughSseResponse, isEagerRelaySseResponse, markEagerRelaySseResponse } from "../relay"; import type { HandleResponsesOptions } from "./core-options"; import { consumeComboFailure, createChildPassthroughCallbackGate } from "./core-combo-failure"; import { preflightComboStreamResponse } from "./combo-stream-preflight"; import { conversationCarriesUploadedFiles } from "./account-change-state"; import { selfContainedResponsesBody } from "./reset-replay"; import { decideCompactionRecovery, readCompactionRecoveryConfig } from "./compaction-recovery-policy"; type Options = HandleResponsesOptions & { translatorBudget: TranslatorBudget }; type Dispatch = (req: Request, config: OcxConfig, log: RequestLogContext, options: Options) => Promise<Response>; const MAX_BYTES = 32 * 1024 * 1024; const RETAINED_USER_CHARS = 80_000; const record = (value: unknown): value is Record<string, unknown> => !!value && typeof value === "object" && !Array.isArray(value); const token = (value: unknown): string | undefined => typeof value === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(value) ? value : undefined; function identity(route: RouteResult): string { return JSON.stringify([route.providerName, route.modelId, route.codexAccountMode ?? "", route.codexAccountNamespace ?? ""]); } function physicalSends(log: RequestLogContext): number { // activeAttempt normally also belongs to attempts: count each receipt exactly once. const attempts = new Set([...(log.attempts ?? []), ...(log.activeAttempt ? [log.activeAttempt] : [])]); return [...attempts].reduce((sum, attempt) => sum + Math.max(0, attempt.sendCount), 0); } /** Reconcile only this leg's physical receipts; never charge already-booked adapter sends again. */ function settlePhysicalSends(log: RequestLogContext, budget: RequestExecutionBudget, beforeSends: number, beforeUsed: number, reported: number, permit?: SingleUseDispatchPermit): number { const sent = Math.max(0, physicalSends(log) - beforeSends); // A legacy fetch leg does not claim the hop through adapterDispatchBudget. Settle its // prepaid booking explicitly; an adapter-owned leg already claimed it, making this a no-op. if (sent > 0) permit?.assumeCharge(); else permit?.release(); // An external report may settle a prepaid booking without changing used. Its explicit // receipt outranks the numeric delta, or the same source send would be charged twice. const unreported = Math.max(0, sent - Math.max(reported, Math.max(0, budget.used - beforeUsed))); if (unreported > 0 && !budget.spendEnforced) budget.used += unreported; return sent; } function portableBody(body: Record<string, unknown>): boolean { if (!Array.isArray(body.input) || body.store === true || conversationCarriesUploadedFiles(body)) return false; // Native ciphertext cannot be summarized by another provider. Never silently replace it with a note. if (body.input.some(item => record(item) && ["compaction", "compaction_summary", "context_compaction"].includes(String(item.type)) && typeof item.encrypted_content === "string" && !item.encrypted_content.startsWith("ocx1:") && !decodeRetainedCompaction(item.encrypted_content))) return false; const input = body.input.filter(item => !record(item) || item.type !== "compaction_trigger"); return selfContainedResponsesBody({ ...body, store: false, input }); } function routed(route: RouteResult, kind: "compaction-v1" | "compaction-v2"): boolean { return !route.combo && route.routeKind !== "policy" && route.routeReason !== "default-provider" && !isCanonicalOpenAiForwardProvider(route.provider) && !(kind === "compaction-v1" ? supportsNativeResponsesCompactEndpoint(route.providerName, route.provider) : supportsNativeResponsesCompactionTrigger(route.provider)); } /** One reader, bounded bytes, and an exact replacement body; never clone a live stream. */ async function bufferedJson(response: Response, signal: AbortSignal): Promise<{ response: Response; json?: Record<string, unknown> }> { const bytes = await readBoundedResponseBytes(response, { signal, maxBytes: MAX_BYTES, inactivityTimeoutMs: 300_000 }); if (bytes.oversized) return { response: formatErrorResponse(502, "translation_buffer_limit", "Compaction recovery response exceeded its byte limit") }; const headers = new Headers(response.headers); headers.delete("content-length"); headers.delete("content-encoding"); const replacement = new Response(bytes.bytes, { status: response.status, statusText: response.statusText, headers }); carryResponseToolOwnership(response, replacement); if (isNonReplayableResponse(response)) markResponseNonReplayable(replacement); try { const json: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes.bytes)); return { response: replacement, ...(record(json) ? { json } : {}) }; } catch { return { response: replacement }; } } /** * Opt-in recovery for routed compaction only. Native compact/ciphertext, stored continuations, * policy/combo routes and hosted tools retain their original failure. This owns no credentials. */ export async function runWithCompactionRecovery( req: Request, config: OcxConfig, logCtx: RequestLogContext, options: Options, dispatch: Dispatch, ): Promise<Response> { const recovery = readCompactionRecoveryConfig(config.compactionRecovery); const kind = options.compactionRecoveryKind ?? "compaction-v2"; if (!recovery || options.compactionRecoveryAttempted || options.comboAttempt || (options.inboundWire && options.inboundWire !== "responses")) { return dispatch(req, config, logCtx, options); } let snapshot: Record<string, unknown> | undefined; let snapshotBytes = 0; let sourceRoute: RouteResult | undefined; let partialOutput = false; let replayUnsafe = false; let adapterError: Extract<AdapterEvent, { type: "error" }> | undefined; let sourceFailure: Response | undefined; let recoveryPermit: SingleUseDispatchPermit | undefined; let restoreSourceLog: ((attemptStatus?: number) => void) | undefined; let sourceReportedSends = 0; const gate = createChildPassthroughCallbackGate(options); const signal = options.abortSignal ?? req.signal; const spentBefore = options.sendBudget?.used ?? 0; const sendsBefore = physicalSends(logCtx); const firstOptions: Options = { ...options, onCompactionRecoverySendsReported(count) { sourceReportedSends += count; options.onCompactionRecoverySendsReported?.(count); }, onRequestBodyParsed(body) { options.onRequestBodyParsed?.(body); if (!record(body) || !Array.isArray(body.input) || typeof body.model !== "string" || !body.input.some(item => record(item) && item.type === "compaction_trigger") || !portableBody(body)) return; const users = extractCompactUserMessages(body.input); if ((users.at(-1)?.length ?? 0) > RETAINED_USER_CHARS) return; try { const bytes = jsonUtf8Bytes(body, MAX_BYTES); const reservation = options.translatorBudget.reserveTransient(bytes, { kind: "request_copies" }); try { const copy = copyPlainData(body); if (!copy.ok) return; snapshot = copy.value; snapshotBytes = bytes; reservation.commitRetained(); } finally { reservation.release(); } } catch { /* Optional recovery cannot reject an otherwise valid original request. */ } }, onCompactionRecoveryRoute(route) { options.onCompactionRecoveryRoute?.(route); if (snapshot && routed(route, kind)) sourceRoute = { ...route }; }, onCompactionRecoveryAdapterEvent(event) { options.onCompactionRecoveryAdapterEvent?.(event); if (!snapshot) return; if (event.type === "heartbeat") replayUnsafe ||= event.replayUnsafe === true; else if (event.type === "error") adapterError = event; else if (event.type !== "done") partialOutput = true; }, onResponseComplete: model => snapshot ? gate.onResponseComplete(model) : options.onResponseComplete?.(model), onNativePassthroughTerminal: status => snapshot ? gate.onTerminal(status) : options.onNativePassthroughTerminal?.(status), onNativePassthroughCancel: () => snapshot ? gate.onCancel() : options.onNativePassthroughCancel?.(), }; try { let response = await dispatch(req, config, logCtx, firstOptions); const keep = (value: Response) => { gate.commit(); return value; }; if (!snapshot || !sourceRoute || signal.aborted || req.signal.aborted || isNonReplayableResponse(response)) return keep(response); let target: RouteResult; try { target = routeConcreteModel(config, recovery.model); } catch { return keep(response); } if (!routed(target, kind) || identity(sourceRoute) === identity(target)) return keep(response); const originalModel = firstOptions.clientModelLabel ? clientFacingModelText(firstOptions.clientModelLabel) : firstOptions.compactionRoutingOverride?.sourceModel ?? String(snapshot.model); // Use the established protocol commit boundary. For runTurn streams the direct event // observer additionally preserves side-effect heartbeats that the bridge does not publish. if (response.ok && hasResponseMediaType(response.headers, "text/event-stream")) { const native = isNativePassthroughSseResponse(response); const eager = isEagerRelaySseResponse(response); const preflight = await preflightComboStreamResponse(response, logCtx); response = preflight.response; if (preflight.kind !== "failed") { if (native) markNativePassthroughSseResponse(response); if (eager) markEagerRelaySseResponse(response); return keep(response); } } else if (response.ok) { const buffered = await bufferedJson(response, signal); response = buffered.response; const json = buffered.json; if (!json || json.status !== "failed" || (Array.isArray(json.output) && json.output.length > 0)) return keep(response); // HTTP 200 can carry a failed terminal. The original structured error remains intact. response = Response.json({ error: json.error, response: json }, { status: adapterError?.status ?? 502 }); } if (response.ok || replayUnsafe || partialOutput || signal.aborted || req.signal.aborted) { if (replayUnsafe) markResponseNonReplayable(response); return keep(response); } const failure = await consumeComboFailure(response, signal); response = failure.response; const code = token(adapterError?.code) ?? token(failure.upstreamCode); const errorType = token(adapterError?.errorType) ?? token(failure.upstreamType); const budget = options.sendBudget; // Reset-only fetch legs report their physical receipt but historically leave used alone. // Reconcile only a failed, eligible compaction; successful/disabled requests stay unchanged. const sourceSends = budget && isRequestExecutionBudget(budget) ? settlePhysicalSends(logCtx, budget, sendsBefore, spentBefore, sourceReportedSends) : 0; const decision = decideCompactionRecovery(recovery, { requestKind: options.compactionRecoveryKind ?? "compaction-v2", recoveryAttempts: 0, cancelled: signal.aborted || req.signal.aborted, nonReplayable: !!failure.nonReplayable || isNonReplayableUpstreamCode(code), partialOutput, toolEffects: replayUnsafe, remainingSends: budget && isRequestExecutionBudget(budget) ? budget.remainingBaseSends(TRANSIENT_RETRY_MAX_ATTEMPTS) : 0, originalModel: identity(sourceRoute), fallbackModel: identity(target), provider: sourceRoute.provider.adapter, httpStatus: adapterError?.status ?? response.status, responseStatus: "failed", errorCode: code, errorType, authenticationDenied: response.status === 401 || response.status === 403, policyDenied: isCyberPolicyCode(code), budgetDenied: code === "translation_buffer_limit", refusal: isTerminalRefusalCode(code), upstreamFailure: sourceSends > 0, }); if (!decision.recover || !budget || !isRequestExecutionBudget(budget)) return keep(response); const fallbackBeforeSends = physicalSends(logCtx); const fallbackBeforeUsed = budget.used; const reservation = budget.reserveDispatch({ sendClass: "combo-failover", targetKey: `compaction:${identity(target)}`, countedExternally: true, replaySafe: true }); if (!reservation.allowed) return keep(response); recoveryPermit = reservation.permit; sourceFailure = response; gate.discard(); // Finish the first physical attempt while retaining its receipt in attempts[]. if (logCtx.activeAttempt) finishRequestAttempt(logCtx.activeAttempt, response.status, Math.max(0, Date.now() - (logCtx.activeAttemptStartedAt ?? Date.now())), logCtx.activeAttempt.usage ?? logCtx.usage); // Snapshot the original log fields before the fallback rewrites them: a failed fallback // returns the original failure, so the log must keep describing that failure, not the // fallback's model, provider, route decision or terminal error. const SOURCE_LOG_FIELDS = [ "model", "provider", "providerAdapter", "requestedAlias", "servedModel", "wireModel", "resolvedModel", "routeDecision", "tierOutcome", "activeTierMetadata", "usage", "usageFromBridge", "upstreamError", "terminalHttpStatus", "terminalErrorCode", "terminalIncompleteReason", "errorCode", ] as const; const sourceLog: Partial<Record<keyof RequestLogContext, unknown>> = {}; const logFields = logCtx as unknown as Record<string, unknown>; for (const field of SOURCE_LOG_FIELDS) sourceLog[field] = logCtx[field]; // The source's local verdict leaves with it; a non-local detail stays as the source's own record. const sourceLocal = takeLocalFailureState(logCtx); const sourceDetail = logCtx.failureDetail; restoreSourceLog = (attemptStatus?: number) => { if (logCtx.activeAttempt) finishRequestAttempt(logCtx.activeAttempt, attemptStatus ?? sourceFailure!.status, Math.max(0, Date.now() - (logCtx.activeAttemptStartedAt ?? Date.now())), logCtx.activeAttempt.usage ?? logCtx.usage); delete logCtx.activeAttempt; delete logCtx.activeAttemptStartedAt; // The fallback's own local verdict (mark and detail) must not describe the restored source. if (takeLocalFailureState(logCtx).local && sourceDetail) logCtx.failureDetail = sourceDetail; for (const field of SOURCE_LOG_FIELDS) { if (sourceLog[field] === undefined) delete logFields[field]; else logFields[field] = sourceLog[field]; } restoreLocalFailureState(logCtx, sourceLocal); }; delete logCtx.activeAttempt; delete logCtx.activeAttemptStartedAt; delete logCtx.usage; delete logCtx.usageFromBridge; delete logCtx.upstreamError; delete logCtx.terminalHttpStatus; delete logCtx.terminalErrorCode; delete logCtx.terminalIncompleteReason; const headers = new Headers(req.headers); headers.delete("authorization"); headers.delete("chatgpt-account-id"); headers.delete("content-length"); headers.delete("content-encoding"); headers.set("content-type", "application/json"); const nextBody = { ...snapshot, model: decision.model, stream: false, store: false }; const bytes = jsonUtf8Bytes(nextBody, MAX_BYTES); const serialization = options.translatorBudget.reserveTransient(bytes, { kind: "request_copies" }); let fallback: Response; let fallbackReportedSends = 0; try { const child = new Request(req.url, { method: "POST", headers, body: JSON.stringify(nextBody), signal: req.signal }); linkRequestSessionLane(req, child); // The source send is finished. The child reuses this account-load holder and may // select the same Kiro account, so return its lease before child admission can // acquire and replace the holder's reference. if (options.accountLoad?.lease) { options.accountLoad.lease.release(); options.accountLoad.lease = null; } fallback = await dispatch(child, config, logCtx, { ...options, clientModelLabel: firstOptions.clientModelLabel, compactionRecoveryAttempted: true, compactionRecoveryPermit: recoveryPermit, compactionRoutingOverride: { sourceModel: originalModel }, onRequestBodyRead: undefined, onRequestBodyParsed: undefined, onCompactionRecoveryRoute: undefined, onCompactionRecoveryAdapterEvent: undefined, onCompactionRecoverySendsReported: count => { fallbackReportedSends += count; options.onCompactionRecoverySendsReported?.(count); }, onResponseComplete: undefined, onNativePassthroughTerminal: undefined, onNativePassthroughCancel: undefined, }); } finally { settlePhysicalSends(logCtx, budget, fallbackBeforeSends, fallbackBeforeUsed, fallbackReportedSends, recoveryPermit); serialization.release(); } if (signal.aborted || req.signal.aborted) { void fallback.body?.cancel().catch(() => undefined); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } if (!fallback.ok) { void fallback.body?.cancel().catch(() => undefined); restoreSourceLog?.(fallback.status); return response; } const completed = await bufferedJson(fallback, signal); const json = completed.json; const items = json && Array.isArray(json.output) ? json.output : []; const compactions = items.filter(value => record(value) && value.type === "compaction"); const permitted = items.every(value => record(value) && (value.type === "compaction" || value.type === "reasoning")); const item = permitted && compactions.length === 1 ? compactions[0] as Record<string, unknown> : undefined; const structured = item && typeof item.encrypted_content === "string" ? decodeRetainedCompaction(item.encrypted_content) : null; const summary = structured?.summary ?? (item && typeof item.encrypted_content === "string" ? decodeCompactionSummary(item.encrypted_content) : null); if (json?.status !== "completed" || !summary?.trim()) { void completed.response.body?.cancel().catch(() => undefined); restoreSourceLog?.(completed.response.status); return response; } // v1 unpacks this text through buildCompactV1Output, which already re-adds retained user // messages as items; embedding them here too would duplicate the same text in the output. const preserved = options.compactionRecoveryKind === "compaction-v1" ? summary : (() => { const usersOnly = Array.isArray(snapshot.input) ? snapshot.input.filter(item => collectReasoningTexts([item]).length === 0) : snapshot.input; const retained = buildCompactV1Output(extractCompactUserMessages(usersOnly), summary).slice(0, -1); const userText = extractCompactUserMessages(retained).map((text, index) => `User message ${index + 1}:\n${text}`).join("\n\n"); return `${summary}\n\nRetained original user messages (verbatim; preserve their goals and constraints):\n${userText}`; })(); const encryptedContent = structured ? encodeRetainedCompaction(preserved, structured.reasoning) : encodeCompactionSummary(preserved); item!.encrypted_content = encryptedContent; json.model = originalModel; void completed.response.body?.cancel().catch(() => undefined); void response.body?.cancel().catch(() => undefined); if (snapshot.stream === true) { const usage = usageFromResponsesPayload(json.usage); async function* events(): AsyncGenerator<AdapterEvent> { yield { type: "text_delta", text: preserved }; yield { type: "done", compactionEncryptedContent: encryptedContent, ...(usage ? { usage } : {}) }; } return new Response(bridgeToResponsesSSE(events(), originalModel, undefined, undefined, undefined, undefined, 2_000, { compaction: true, translatorBudget: options.translatorBudget, onCompletedResponse: () => options.onResponseComplete?.(originalModel) }), { headers: { "content-type": "text/event-stream" } }); } options.onResponseComplete?.(originalModel); return Response.json(json); } catch (error) { gate.discard(); if (signal.aborted || req.signal.aborted) return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); if (sourceFailure) { restoreSourceLog?.(); return sourceFailure; } throw error; } finally { if (snapshotBytes) options.translatorBudget.releaseRetained(snapshotBytes, { kind: "request_copies" }); recoveryPermit?.release(); snapshot = undefined; } }