UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

207 lines (194 loc) • 9.32 kB
/** * Describe ONE image via a ROUTED model through the proxy's own * /v1/chat/completions on loopback (#2188 roadmap 180 revised). * * One executor for every provider the router can reach: the chat inbound * translates image_url parts and each adapter compiles its own wire * (Anthropic blocks, Antigravity inlineData, xai Responses input_image, plain * openai-chat), so provider coverage is the router's job, not this file's. * * Recursion fence: the request carries `x-opencodex-vision-describe: 1`. * The Chat surface detects the raw header before its bridge rebuilds headers * and carries it into handleResponses as `visionDescribeTerminal`; a marked * request STRIPS images instead of planning another describe (depth cap 1, * holds under predicate drift and combo re-resolution — audit rounds 2-4). * * Admission ladder (audit round 3): configuredApiAuthToken() (env token) || * service token file || first config.apiKeys entry, sent as * `x-opencodex-api-key` — never Authorization (gateway-cache.ts rule: an * admission secret in a forwardable header is a forwarding hazard). Loopback * binds require no token at all (resolveApiAuth admits loopback). * * Destination (#4236): the unauthenticated loopback listener when one is * enabled, otherwise the BIND address — the former roadmap-170 limitation * ("a bindHost where 127.0.0.1 does not answer cannot reach its own * loopback") is closed by resolving through `localInferenceDestination` * rather than composing 127.0.0.1 by hand. */ import type { OcxConfig } from "../types"; import { localAdmissionToken, localInferenceDestination } from "../lib/local-destinations"; import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort"; import { redactSecretString } from "../lib/redact"; import { sidecarEnter } from "../lib/sidecar-tracker"; import { configuredPort } from "../server/auth-cors"; import type { DescribeOutcome, VisionSettings } from "./describe"; export const VISION_DESCRIBE_TERMINAL_HEADER = "x-opencodex-vision-describe"; const ALLOWED_IMAGE_MIME = new Set(["image/png", "image/jpeg", "image/jpg", "image/webp", "image/gif"]); const MAX_IMAGE_BYTES = 20 * 1024 * 1024; /** Bound the loopback JSON response; descriptions are clamped to ~2k chars by the caller anyway. */ const MAX_ROUTED_RESPONSE_BYTES = 4 * 1024 * 1024; const DESCRIBE_INSTRUCTION = "You are a vision describer for a text-only model that cannot see the image. Describe the image " + "thoroughly and factually so that model can fully reason about it: transcribe any visible text " + "verbatim, and note UI/layout, colors, branding/logos, charts, and notable details. Focus on " + "what's relevant to the user's request. Output only the description."; function validateImageUrl(url: string): string | null { if (url.startsWith("data:")) { const match = /^data:([^;,]+?)(;base64)?,(.*)$/s.exec(url); if (!match) return "malformed data URL"; const mime = match[1].toLowerCase(); if (!ALLOWED_IMAGE_MIME.has(mime)) return `unsupported image type "${mime}"`; if (match[2]) { const bytes = Math.floor((match[3].length * 3) / 4); if (bytes > MAX_IMAGE_BYTES) return `image too large (~${Math.round(bytes / 1024 / 1024)}MB)`; } return null; } if (url.startsWith("https://")) return null; return "unsupported image URL scheme (expected data: or https:)"; } /** * The admission ladder: env token, hardened service token file, first configured API key. * * Shared with every other local client through `localAdmissionToken` so the credential this * self-fetch presents cannot drift from the one the Codex provider table and the Claude launch * env carry. Never the admin token. */ export function routedDescribeAdmissionToken(config: Pick<OcxConfig, "apiKeys">): string | undefined { return localAdmissionToken(config); } /** Base URL seam for tests; production always self-fetches the resolved local destination. */ export function routedDescribeBaseUrl( config: Pick<OcxConfig, "port" | "hostname" | "unauthenticatedLoopbackListener">, ): string { return routedDescribeDestination(config).origin; } /** * The local destination this self-fetch dials, and whether it needs a credential. * * This is a local client like any other: the unauthenticated loopback listener when one is * enabled, otherwise the bind address — on a tailnet-bound hub there is no loopback socket at * all (#4236). The helper sends the OpenAI chat wire, which that listener now admits. */ function routedDescribeDestination( config: Pick<OcxConfig, "port" | "hostname" | "unauthenticatedLoopbackListener">, ) { // config.port can be 0 (ephemeral bind, tests) or stale after a live port override; the // server records its ACTUAL bound port via setCorsOrigin at startup, so prefer that when // config carries no positive port. `configuredPort()` is itself `0` when `_corsOrigin` has no // explicit port (a default-port origin), so the literal default has to backstop it or the // composed URL names port 0 and the self-fetch cannot connect. const port = config.port && config.port > 0 ? config.port : Number(configuredPort()) || 10_100; return localInferenceDestination(config, port); } export async function describeImageRouted( imageUrl: string, _detail: string | undefined, contextText: string, routedModel: string, config: Pick<OcxConfig, "port" | "hostname" | "apiKeys" | "unauthenticatedLoopbackListener">, settings: VisionSettings, abortSignal?: AbortSignal, baseUrlOverride?: string, ): Promise<DescribeOutcome> { const invalid = validateImageUrl(imageUrl); if (invalid) return { text: "", error: invalid }; const headers: Record<string, string> = { "Content-Type": "application/json", [VISION_DESCRIBE_TERMINAL_HEADER]: "1", }; const admission = routedDescribeAdmissionToken(config); if (admission) headers["x-opencodex-api-key"] = admission; // A bind that demands admission with no resolvable credential would return 401 with a body // the caller reports as a describe failure; naming the cause once is the difference between // "vision is broken" and a fixable configuration note. if (!admission && !baseUrlOverride && routedDescribeDestination(config).requiresAdmissionToken) { console.warn( "[vision] routed describe has no opencodex data-plane credential for " + `${routedDescribeBaseUrl(config)} — the self-fetch will be refused.`, ); } const requestBody = { model: routedModel, stream: false, messages: [ { role: "system", content: DESCRIBE_INSTRUCTION }, { role: "user", content: [ ...(contextText ? [{ type: "text", text: `User's request context: ${contextText}` }] : []), { type: "image_url", image_url: { url: imageUrl } }, ], }, ], }; const linkedSignal = signalWithTimeout(settings.timeoutMs, abortSignal); const sidecarExit = sidecarEnter("vision"); const t0 = Date.now(); try { const res = await fetch(`${baseUrlOverride ?? routedDescribeBaseUrl(config)}/v1/chat/completions`, { method: "POST", headers, body: JSON.stringify(requestBody), signal: linkedSignal.signal, redirect: "manual", }); const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal); try { const raw = await res.text(); if (raw.length > MAX_ROUTED_RESPONSE_BYTES) { return { text: "", error: "routed describe response exceeded byte bound" }; } if (!res.ok) { return { text: "", error: `routed describe HTTP ${res.status}: ${redactSecretString(raw.slice(0, 200))}` }; } let payload: unknown; try { payload = JSON.parse(raw); } catch { return { text: "", error: "routed describe returned non-JSON" }; } const content = extractChatContent(payload); if (!content) return { text: "", error: "routed describe returned no text" }; return { text: content }; } finally { detachBodyGuard(); } } catch (e) { const kind = e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error"; console.warn(`[vision] routed describe ${kind} (${Date.now() - t0}ms)`); return { text: "", error: redactSecretString(e instanceof Error ? e.message : String(e)) }; } finally { sidecarExit(); linkedSignal.cleanup(); } } function extractChatContent(payload: unknown): string | undefined { if (!payload || typeof payload !== "object") return undefined; const choices = (payload as { choices?: unknown }).choices; if (!Array.isArray(choices) || choices.length === 0) return undefined; const message = (choices[0] as { message?: unknown })?.message; if (!message || typeof message !== "object") return undefined; const content = (message as { content?: unknown }).content; if (typeof content === "string" && content.trim().length > 0) return content; // Some adapters emit content parts; join text parts. if (Array.isArray(content)) { const joined = content .map(part => (part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string" ? (part as { text: string }).text : "")) .join(""); if (joined.trim().length > 0) return joined; } return undefined; }