UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

272 lines (251 loc) • 14.4 kB
/** * Gemini-compatible inbound API: GET /v1beta/models[/{model}] and POST * /v1beta|v1/models/{model}:generateContent|streamGenerateContent (countTokens and the embed * methods answer 501). * * Like the Ollama inbound (./ollama-inbound), the catalog starts with configured * provider routes. Unless the native leg applies, generation is translated into an internal Chat Completions request handled by * the ordinary Chat path, and this handler owns the final request-log row. Errors use the Gemini * `{ error: { code, message, status } }` body with a fixed message chosen by HTTP status. */ import { GeminiCodecError, chatToGeminiResponse, consumeGeminiChatChunk, createGeminiStreamState, geminiStreamTerminal, geminiToChat } from "../protocols/codecs/gemini"; import { createTranslatorBudget, isTranslatorBudgetExceededError } from "../lib/translator-budget"; import type { OcxConfig, OcxParsedRequest } from "../types"; import type { RouteResult } from "../router"; import { configuredContextWindow, configuredMaxInputTokens } from "../codex/catalog/model-hints"; import { modelRecordValue } from "../reasoning-effort"; import { getModelMetadata, resolveMetadataProvider } from "../generated/model-metadata"; import { chatConversationToIr } from "../protocols/codecs/chat-ir"; import { ChatCompletionsRequestError } from "../chat/inbound"; import { estimateInputTokens } from "./responses/input-admission"; import type { DataPlaneAdmission } from "./auth-cors"; import { resolveAdmissionModelScope, routeAllowedByScope } from "./admission-model-scope"; import type { GeminiMethod } from "./gemini-native-paths"; import { geminiNativeCatalogAliases, geminiNativeCatalogLookup, geminiNativeDiscoveredRoutes } from "./gemini-native-catalog"; import { applyLocalRefusalCode, commitCancelled, drainedInnerErrorMessage, handleNativeChat, nativeRefusalParts, nativeRetryAfter, streamNativeFromChat, type NativeLogIds, } from "./native-chat-bridge"; import { createNativeFinalLog, type NativeFinalLog } from "./native-final-log"; import { findOllamaRoute, listOllamaRoutes, preflightOllamaRoute } from "./ollama-route-policy"; import { DecompressedBodyTooLargeError, readJsonRequestBody, resolveInboundBodyLimitBytes } from "./request-decompress"; import { addFinalRequestLog, type RequestLogContext } from "./request-log"; const STATUS_NAMES: Readonly<Record<number, string>> = { 400: "INVALID_ARGUMENT", 401: "UNAUTHENTICATED", 403: "PERMISSION_DENIED", 404: "NOT_FOUND", 413: "INVALID_ARGUMENT", 429: "RESOURCE_EXHAUSTED", 499: "CANCELLED", 501: "UNIMPLEMENTED", 503: "UNAVAILABLE", 504: "DEADLINE_EXCEEDED", // google.rpc.Code by HTTP mapping; 402 has no canonical code, so the closest precondition. 402: "FAILED_PRECONDITION", 405: "UNIMPLEMENTED", 408: "DEADLINE_EXCEEDED", 409: "ABORTED", 422: "INVALID_ARGUMENT", 500: "INTERNAL", 502: "UNAVAILABLE", }; export function geminiErrorBody(status: number, message: string): { error: { code: number; message: string; status: string } } { return { error: { code: status, message, status: STATUS_NAMES[status] ?? "INTERNAL" } }; } /** * Every Gemini error response is built from scratch: content type, our own error text, and * optionally a reformatted Retry-After. Callers add CORS and the request-log id. */ export function geminiError(status: number, message: string, retryAfter?: string | null): Response { const headers = new Headers({ "content-type": "application/json" }); const advice = nativeRetryAfter(retryAfter); if (advice !== undefined) headers.set("Retry-After", advice); return new Response(JSON.stringify(geminiErrorBody(status, message)), { status, headers }); } /** Reshape an existing proxy refusal (capacity, workflow, drain) keeping its status and retry advice. */ export function toGeminiRefusal(response: Response): Response { const parts = nativeRefusalParts(response); return applyLocalRefusalCode(geminiError(parts.status, parts.message, parts.retryAfter), parts.localRefusalCode); } function geminiCatalogRoutes(config: OcxConfig, req: Request) { const configured = listOllamaRoutes(config, req); const selectors = new Set(configured.map(row => row.selector)); const discovered = geminiNativeDiscoveredRoutes(config, req).filter(row => !selectors.has(row.selector)); return [...configured, ...discovered]; } /** Token limits from configuration first, then generated model metadata; omitted when unknown. */ function tokenLimits(config: OcxConfig, route: RouteResult): { inputTokenLimit?: number; outputTokenLimit?: number } { const metadataProvider = resolveMetadataProvider(route.providerName) ?? route.providerName; const metadata = getModelMetadata(metadataProvider, route.modelId); const custom = config.customModels?.find(row => row.provider === route.providerName && row.modelId === route.modelId); const customContext = custom?.contextWindow; const input = configuredMaxInputTokens(route.provider, route.modelId) ?? configuredContextWindow(route.provider, route.modelId) ?? (typeof customContext === "number" && customContext > 0 ? customContext : metadata?.contextWindow); const output = modelRecordValue(route.provider.modelMaxOutputTokens, route.modelId) ?? route.provider.defaultMaxOutputTokens ?? metadata?.maxTokens; return { ...(typeof input === "number" && input > 0 ? { inputTokenLimit: input } : {}), ...(typeof output === "number" && output > 0 ? { outputTokenLimit: output } : {}), }; } const modelResource = (config: OcxConfig, row: { selector: string; route: RouteResult }) => ({ name: `models/${row.selector}`, displayName: row.selector, ...tokenLimits(config, row.route), supportedGenerationMethods: ["generateContent", "streamGenerateContent", "countTokens"], }); export function handleGeminiModels(config: OcxConfig, req: Request, admission: DataPlaneAdmission): Response { const scope = resolveAdmissionModelScope(config, admission); const candidates = geminiCatalogRoutes(config, req); const rows = candidates.filter(row => routeAllowedByScope(scope, row.route)); // A key can allow a concrete effort tier while excluding its source family. const aliases = geminiNativeCatalogAliases(config, req, candidates, scope); return Response.json({ models: [...rows, ...aliases].map(row => modelResource(config, row)) }); } function findGeminiCatalogRow(config: OcxConfig, req: Request, admission: DataPlaneAdmission, model: string) { const row = findOllamaRoute(config, model, req) ?? geminiNativeDiscoveredRoutes(config, req, model)[0]; const scope = resolveAdmissionModelScope(config, admission); if (row && routeAllowedByScope(scope, row.route)) return row; return geminiNativeCatalogLookup(config, req, model, () => geminiCatalogRoutes(config, req), scope); } export function handleGeminiModel(config: OcxConfig, req: Request, admission: DataPlaneAdmission, model: string): Response { const row = findGeminiCatalogRow(config, req, admission, model); return row ? Response.json(modelResource(config, row)) : geminiError(404, "model not found"); } /** * `:countTokens`: a local estimate over the same Chat projection generation uses, for the routes the * key may reach. No upstream call. Body: `{contents, systemInstruction, tools}` or `{generateContentRequest}`. */ export async function handleGeminiCountTokens(config: OcxConfig, req: Request, admission: DataPlaneAdmission, model: string): Promise<Response> { const row = findGeminiCatalogRow(config, req, admission, model); if (!row) return geminiError(404, "model not found"); const limit = resolveInboundBodyLimitBytes(config.maxInboundBodyBytes); const budget = createTranslatorBudget(); try { const body = await readJsonRequestBody(req, budget, limit); const source = body && typeof body === "object" && !Array.isArray(body) && (body as Record<string, unknown>).generateContentRequest !== undefined ? (body as Record<string, unknown>).generateContentRequest : body; const chat = geminiToChat(row.route.modelId, source, false); const conversation = chatConversationToIr(chat, row.route.modelId); const totalTokens = estimateInputTokens({ ...conversation, modelId: row.route.modelId } as unknown as OcxParsedRequest, row.route.modelId, row.route.provider); return Response.json({ totalTokens }); } catch (error) { if (req.signal.aborted) return geminiError(499, "request cancelled"); if (error instanceof DecompressedBodyTooLargeError) return geminiError(413, "request body too large"); if (isTranslatorBudgetExceededError(error)) return geminiError(502, "request translation buffer exceeded the safe limit"); if (error instanceof GeminiCodecError || error instanceof ChatCompletionsRequestError) return geminiError(400, error.message); return geminiError(400, "invalid JSON body"); } finally { budget.dispose(); } } function failBeforeInner(ids: NativeLogIds, logCtx: RequestLogContext, status: number, message: string): Response { addFinalRequestLog(ids.requestId, ids.start, logCtx, status, { closeReason: "non_stream" }); return geminiError(status, message); } function cancelledNonStream(finalLog: NativeFinalLog): Response { commitCancelled(finalLog); return geminiError(499, "request cancelled"); } /** generateContent and streamGenerateContent. `req` is the admitted, session-promoted request. */ export async function handleGeminiGeneration( method: Extract<GeminiMethod, "generateContent" | "streamGenerateContent">, model: string, sse: boolean, req: Request, config: OcxConfig, logCtx: RequestLogContext, ids: NativeLogIds, ): Promise<Response> { const limit = resolveInboundBodyLimitBytes(config.maxInboundBodyBytes); const stream = method === "streamGenerateContent"; const budget = createTranslatorBudget(); let originalBody: unknown; let chat: Record<string, unknown>; try { originalBody = await readJsonRequestBody(req, budget, limit); chat = geminiToChat(model, originalBody, stream); } catch (error) { if (req.signal.aborted) return failBeforeInner(ids, logCtx, 499, "request cancelled"); if (error instanceof DecompressedBodyTooLargeError) return failBeforeInner(ids, logCtx, 413, "request body too large"); if (isTranslatorBudgetExceededError(error)) return failBeforeInner(ids, logCtx, 502, "request translation buffer exceeded the safe limit"); return failBeforeInner(ids, logCtx, 400, error instanceof GeminiCodecError ? error.message : "invalid JSON body"); } finally { budget.dispose(); } logCtx.requestedModel = model; const preflight = preflightOllamaRoute(config, model, req); if (!preflight.ok) { logCtx.errorCode = preflight.error.code; return failBeforeInner(ids, logCtx, preflight.error.status, preflight.error.message); } const chatJson = JSON.stringify(chat); if (Buffer.byteLength(chatJson) > limit) return failBeforeInner(ids, logCtx, 413, "request body too large"); const finalLog = createNativeFinalLog(ids.requestId, ids.start, logCtx); let inner: Response; try { inner = await handleNativeChat(req, originalBody, chatJson, config, logCtx, ids); } catch { await finalLog.waitForInner(); if (req.signal.aborted) return cancelledNonStream(finalLog); finalLog.commit({ status: 502, terminalStatus: "failed", closeReason: "non_stream" }); return geminiError(502, "request failed"); } if (!inner.ok || !inner.body) { const message = await drainedInnerErrorMessage(inner); await finalLog.waitForInner(); if (req.signal.aborted) return cancelledNonStream(finalLog); finalLog.commit({ status: inner.ok ? 502 : inner.status, terminalStatus: "failed", closeReason: "non_stream" }); return geminiError(inner.ok ? 502 : inner.status, message, inner.headers.get("retry-after")); } return stream ? streamGemini(model, sse, inner.body, finalLog, req.signal) : bufferedGemini(model, inner, finalLog, req.signal); } async function bufferedGemini(model: string, inner: Response, finalLog: NativeFinalLog, signal: AbortSignal): Promise<Response> { let native: Record<string, unknown>; try { native = chatToGeminiResponse(model, JSON.parse(await inner.text())); } catch { await finalLog.waitForInner(); if (signal.aborted) return cancelledNonStream(finalLog); finalLog.commit({ status: 502, terminalStatus: "failed", closeReason: "non_stream" }); return geminiError(502, "invalid upstream response"); } const settled = await finalLog.waitForInner(); if (signal.aborted) return cancelledNonStream(finalLog); if (!settled) { finalLog.commit({ status: 502, terminalStatus: "failed", closeReason: "non_stream" }); return geminiError(502, "request did not complete"); } finalLog.commit({ status: 200, terminalStatus: "completed", closeReason: "non_stream" }); return Response.json(native); } const encoder = new TextEncoder(); /** How records become bytes: SSE data frames, or the elements of one JSON array. */ interface GeminiFraming { contentType: string; frames(records: unknown[]): Uint8Array[]; end(records: unknown[]): Uint8Array[]; } const sseFraming = (): GeminiFraming => { const frames = (records: unknown[]) => records.map(record => encoder.encode(`data: ${JSON.stringify(record)}\r\n\r\n`)); return { contentType: "text/event-stream", frames, end: frames }; }; const arrayFraming = (): GeminiFraming => { let opened = false; const frames = (records: unknown[]) => records.map(record => { const prefix = opened ? ",\n" : "[\n"; opened = true; return encoder.encode(prefix + JSON.stringify(record)); }); return { contentType: "application/json", frames, end: records => [...frames(records), encoder.encode("\n]")] }; }; function streamGemini(model: string, sse: boolean, body: ReadableStream<Uint8Array>, finalLog: NativeFinalLog, signal: AbortSignal): Response { const state = createGeminiStreamState(model); const framing = sse ? sseFraming() : arrayFraming(); return streamNativeFromChat({ contentType: framing.contentType, chunk: parsed => framing.frames(consumeGeminiChatChunk(state, parsed)), terminal() { const record = geminiStreamTerminal(state); return record ? { ok: true, frames: () => framing.end([record]) } : { ok: false, error: "request did not complete" }; }, failure: message => framing.end([geminiErrorBody(502, message)]), }, body, finalLog, signal); }