@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
149 lines (141 loc) • 7.26 kB
text/typescript
import type { OcxProviderConfig } from "../types";
import type { VisionReasoningEffort } from "../reasoning-effort";
import { mergeCodexSelectedHeaders } from "../codex/forward-transport-headers";
import { signalWithTimeout, cancelBodyOnAbort } from "../lib/abort";
import { redactSecretString } from "../lib/redact";
import { sidecarEnter } from "../lib/sidecar-tracker";
import { applyUpstreamRecoveryInit, fetchWithResetRetry } from "../lib/upstream-retry";
import { parseSidecarSSE } from "../web-search/parse";
import type { SidecarOutcomeRecorder } from "../web-search/executor";
import { NATIVE_RESERVE_MODEL } from "../codex/catalog/native-models";
import { openAiSidecarCreditRefusal } from "../providers/openai-sidecar-credit";
export interface VisionSettings {
model: string;
reasoning: VisionReasoningEffort;
timeoutMs: number;
/** Effective Desktop authless compatibility does not grant auxiliary model use. */
reserveCompatibility?: boolean;
}
/** A description, or an `error` string when it couldn't run (caller injects a graceful marker). */
export type DescribeOutcome = { text: string; error?: string };
const ALLOWED_IMAGE_MIME = new Set(["image/png", "image/jpeg", "image/jpg", "image/webp", "image/gif"]);
/** ~20 MB — generous enough for screenshots; rejects pathological payloads before forwarding. */
const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
/**
* Validate an image URL before forwarding. Data URLs are checked for an allowed media type and a sane
* decoded size (a malformed/huge/unsupported one would otherwise 400 at the backend or waste tokens).
* Remote https URLs are passed through — the ChatGPT backend fetches them, not this proxy (so there's
* no SSRF surface here). Returns an error string when the URL must be rejected, else null.
*/
function validateImageUrl(url: string): string | null {
if (url.startsWith("data:")) {
const m = /^data:([^;,]+?)(;base64)?,(.*)$/s.exec(url);
if (!m) return "malformed data URL";
const mime = m[1].toLowerCase();
if (!ALLOWED_IMAGE_MIME.has(mime)) return `unsupported image type "${mime}"`;
if (m[2]) {
const bytes = Math.floor((m[3].length * 3) / 4);
if (bytes > MAX_IMAGE_BYTES) return `image too large (~${Math.round(bytes / 1024 / 1024)}MB)`;
}
return null;
}
if (url.startsWith("https://")) return null;
return "unsupported image URL scheme (expected data: or https:)";
}
/**
* Describe ONE image via a gpt vision model through the ChatGPT forward backend — the path that has
* native image input. Reuses selected forwarded OAuth headers. The user's own request text is
* passed as context so the description is focused. Never throws — returns `{error}` on failure.
*/
export async function describeImage(
imageUrl: string,
detail: string | undefined,
contextText: string,
forwardProvider: OcxProviderConfig,
selectedForwardHeaders: Headers,
settings: VisionSettings,
abortSignal?: AbortSignal,
recordOutcome?: SidecarOutcomeRecorder,
beforeDispatch?: () => void,
): Promise<DescribeOutcome> {
if (settings.reserveCompatibility && settings.model === NATIVE_RESERVE_MODEL) {
return { text: "", error: "Luna Reserve compatibility is only available as a conversation model, not a vision helper. Choose another vision helper model." };
}
const invalid = validateImageUrl(imageUrl);
if (invalid) return { text: "", error: invalid };
const headers = Object.fromEntries(mergeCodexSelectedHeaders(
{ "Content-Type": "application/json", ...forwardProvider.headers }, selectedForwardHeaders,
));
const content: unknown[] = [];
if (contextText) content.push({ type: "input_text", text: `The user's request about this image: ${contextText}` });
content.push({ type: "input_image", image_url: imageUrl, detail: detail ?? "high" });
const body = {
model: settings.model,
instructions:
"You are a vision describer for a text-only model that cannot see the image. Describe the image " +
"thoroughly and factually so that model can fully reason about it: transcribe any visible text " +
"verbatim, and note UI/layout, colors, branding/logos, charts, and notable details. Focus on " +
"what's relevant to the user's request. Output only the description.",
input: [{ type: "message", role: "user", content }],
reasoning: { effort: settings.reasoning },
// The ChatGPT (codex) backend rejects `max_output_tokens` ("Unsupported parameter"); the shared
// SSE parser bounds wire and decoded payload before DESC_MAX_CHARS applies its display clamp.
store: false,
stream: true,
};
const linkedSignal = signalWithTimeout(settings.timeoutMs, abortSignal);
const sidecarExit = sidecarEnter("vision");
const t0 = Date.now();
try {
const res = await fetchWithResetRetry(
// The replay needs `keepalive: false` to abandon the half-closed pooled socket; Bun has
// ignored a bare `Connection: close` (oven-sh/bun#20492).
recovery => {
beforeDispatch?.();
return fetch(`${forwardProvider.baseUrl}/responses`, applyUpstreamRecoveryInit({
method: "POST",
headers,
body: JSON.stringify(body),
signal: linkedSignal.signal,
// Credential-bearing: do not follow a cross-origin 3xx. Bun strips `Authorization`
// across origins but forwards nonstandard headers such as `chatgpt-account-id`,
// `session_id`, and `x-codex-turn-metadata` to the redirect target.
redirect: "manual",
}, recovery));
},
{ replaySafe: true, abortSignal: linkedSignal.signal, label: "vision-sidecar" },
);
const detachBodyGuard = cancelBodyOnAbort(res.body, linkedSignal.signal);
try {
if (!res.ok) {
recordOutcome?.(res.status);
const t = await res.text().catch(() => "");
console.warn(`[vision] sidecar HTTP ${res.status} (${Date.now() - t0}ms)`);
return { text: "", error: `vision sidecar HTTP ${res.status}: ${redactSecretString(t.slice(0, 200))}` };
}
const parsed = await parseSidecarSSE(res);
if (linkedSignal.signal.aborted) throw linkedSignal.signal.reason;
recordOutcome?.(res.status);
// Any parser error invalidates decoded text: it may be a prefix from a bounded or incomplete
// stream and must never be rendered or cached as a complete image description.
if (parsed.error) return { text: "", error: parsed.error };
return { text: parsed.text };
} finally {
detachBodyGuard();
}
} catch (e) {
const policyRefusal = openAiSidecarCreditRefusal(e);
if (policyRefusal) return { text: "", error: policyRefusal.message };
const kind = e instanceof Error && e.name === "TimeoutError" ? "timeout" : "connect_error";
const callerAborted = abortSignal?.aborted === true
&& linkedSignal.signal.aborted
&& linkedSignal.signal.reason === abortSignal.reason
&& e === linkedSignal.signal.reason;
recordOutcome?.(callerAborted ? "connect_neutral" : kind);
console.warn(`[vision] sidecar ${kind} (${Date.now() - t0}ms)`);
return { text: "", error: e instanceof Error ? e.message : String(e) };
} finally {
sidecarExit();
linkedSignal.cleanup();
}
}