@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
210 lines (195 loc) • 9 kB
text/typescript
import { parseDataUrl } from "./image";
import { sniffImageDimensions } from "./anthropic-image-guard";
import { MAX_INPUT_PIXELS } from "./anthropic-image-codec";
import { enterImageDecode } from "./anthropic-image-normalize";
import {
normalizeImageTargets,
type NormalizeOptions,
type NormalizeTarget,
} from "./anthropic-image-normalize";
/**
* Best-effort base64 image budget for translated Chat requests. This leaves room for
* other request fields but is not a guarantee that the complete body fits an upstream
* limit. Remote URLs are never fetched by request construction.
*/
export const OPENAI_CHAT_IMAGE_BASE64_BUDGET = 3_670_016; // 3.5MiB
/** Longest edge an undersized image may be stretched to while reaching the minimum edge. */
export const OPENAI_CHAT_UPSCALE_MAX_EDGE = 4096;
/** Resizes decoded image bytes to exactly `width` x `height` and returns base64 plus media type. */
export type UpscaleFn = (input: Buffer, width: number, height: number) => Promise<{ data: string; mediaType: string }>;
/** Default upscaler: Bun.Image resize, re-encoded as PNG so small text strips stay sharp. */
export const bunImageUpscale: UpscaleFn = async (input, width, height) => {
const image = new Bun.Image(input);
const { width: sourceWidth, height: sourceHeight } = await image.metadata();
if (!Number.isSafeInteger(sourceWidth) || !Number.isSafeInteger(sourceHeight) ||
sourceWidth <= 0 || sourceHeight <= 0 || sourceWidth * sourceHeight > MAX_INPUT_PIXELS) {
throw new Error("image dimensions exceed the safe decode limit");
}
const out = await image.resize(width, height).png().toBuffer();
return { data: Buffer.from(out).toString("base64"), mediaType: "image/png" };
};
export interface NormalizeOpenAIChatImagesOptions
extends Pick<NormalizeOptions, "encode" | "tierBias" | "validate"> {
abortSignal?: AbortSignal;
}
/** Whether `value` is a plain object, so message and part shapes can be walked safely. */
function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
/**
* Walk every well-formed `image_url` part in a Chat Completions message array, ignoring
* malformed shapes rather than throwing on them. Returning false from `visit` stops the walk.
*/
function forEachImagePart(
messages: unknown,
visit: (imageUrl: Record<string, unknown>, url: string) => boolean | void,
): void {
if (!Array.isArray(messages)) return;
for (const message of messages) {
if (!isRecord(message) || !Array.isArray(message.content)) continue;
for (const part of message.content) {
if (!isRecord(part) || part.type !== "image_url" || !isRecord(part.image_url)) continue;
const imageUrl = part.image_url;
if (typeof imageUrl.url !== "string") continue;
if (visit(imageUrl, imageUrl.url) === false) return;
}
}
}
/**
* Whether this turn carries inline image bytes worth normalizing. The adapter uses this
* to stay synchronous for text-only turns, which is every turn on most providers.
*/
export function hasShrinkableOpenAIChatImages(messages: unknown): boolean {
let total = 0;
let found = false;
forEachImagePart(messages, (_imageUrl, url) => {
const source = parseDataUrl(url);
if (!source) return;
total += source.base64.length;
if (total > OPENAI_CHAT_IMAGE_BASE64_BUDGET) {
found = true;
return false;
}
});
return found;
}
/**
* Target size for an image whose short edge is below `minEdge`, or null when it already
* fits or cannot be handled safely. Aspect ratio is kept; the long edge is clamped to
* OPENAI_CHAT_UPSCALE_MAX_EDGE, which only distorts extreme strips.
*/
export function upscaledImageSize(width: number, height: number, minEdge: number): { width: number; height: number } | null {
if (!(width > 0 && height > 0) || Math.min(width, height) >= minEdge) return null;
if (width * height > MAX_INPUT_PIXELS) return null;
const scale = minEdge / Math.min(width, height);
const clamp = (edge: number): number => Math.min(OPENAI_CHAT_UPSCALE_MAX_EDGE, Math.max(minEdge, Math.ceil(edge * scale)));
return { width: clamp(width), height: clamp(height) };
}
/** Whether any inline image has a sniffable short edge below `minEdge`. */
export function hasUndersizedOpenAIChatImages(messages: unknown, minEdge: number): boolean {
let found = false;
forEachImagePart(messages, (_imageUrl, url) => {
const source = parseDataUrl(url);
const dims = source ? sniffImageDimensions(source.base64) : null;
if (dims && upscaledImageSize(dims.width, dims.height, minEdge)) {
found = true;
return false;
}
});
return found;
}
/**
* Upscale inline images below `minEdge` in already-built Chat Completions messages, in place.
* Runs after budget normalization, because that step can itself floor a wide screenshot
* under the minimum. An image that cannot be decoded keeps its original URL.
*/
export async function upscaleUndersizedOpenAIChatImages(
messages: unknown,
minEdge: number,
options: { upscale?: UpscaleFn; abortSignal?: AbortSignal } = {},
): Promise<void> {
const upscale = options.upscale ?? bunImageUpscale;
const jobs: Array<{ imageUrl: Record<string, unknown>; base64: string; width: number; height: number }> = [];
forEachImagePart(messages, (imageUrl, url) => {
const source = parseDataUrl(url);
const dims = source ? sniffImageDimensions(source.base64) : null;
const target = dims ? upscaledImageSize(dims.width, dims.height, minEdge) : null;
if (source && target) jobs.push({ imageUrl, base64: source.base64, ...target });
});
const throwIfAborted = (): void => {
if (options.abortSignal?.aborted) throw options.abortSignal.reason ?? new DOMException("Aborted", "AbortError");
};
for (const job of jobs) {
throwIfAborted();
// Decoded bitmaps share the process-wide decode bound with the other image normalizers.
const release = await enterImageDecode(options.abortSignal);
try {
const out = await upscale(Buffer.from(job.base64, "base64"), job.width, job.height);
job.imageUrl.url = `data:${out.mediaType};base64,${out.data}`;
} catch {
// Keep the original image; the upstream reports its own validation error.
} finally {
release();
}
}
// A cancellation during the last resize must not let a request for a cancelled turn through.
throwIfAborted();
}
/**
* Image work the openai-chat adapter must finish before it builds the request, or null when
* there is none. Budget normalization runs first because it can floor a wide screenshot below
* the upstream minimum. A failure keeps the original images unless the request was cancelled.
*/
export function prepareOpenAIChatImages(
messages: unknown,
minImageEdge: number | undefined,
options: NormalizeOpenAIChatImagesOptions = {},
): Promise<void> | null {
const shrink = hasShrinkableOpenAIChatImages(messages);
if (!shrink && (minImageEdge === undefined || !hasUndersizedOpenAIChatImages(messages, minImageEdge))) return null;
return (async () => {
if (shrink) await normalizeOpenAIChatImages(messages, options);
if (minImageEdge !== undefined) await upscaleUndersizedOpenAIChatImages(messages, minImageEdge, { abortSignal: options.abortSignal });
})().catch(error => {
if (options.abortSignal?.aborted) throw error;
});
}
/**
* Normalize image_url parts in already-built Chat Completions messages, in place.
*
* The drop callback deliberately keeps the original URL. The shared normalizer calls
* drop for corrupt or decode-bomb inputs, and this wire has no downstream guard that
* would re-attach a dropped image, so dropping here would silently lose a user's
* screenshot. Terminal-size overflow uses overflowAction "none" for the same reason:
* an image floored at 320px stays attached rather than being removed.
*/
export async function normalizeOpenAIChatImages(
messages: unknown,
options: NormalizeOpenAIChatImagesOptions = {},
): Promise<void> {
const targets: NormalizeTarget[] = [];
forEachImagePart(messages, (imageUrl, url) => {
const source = parseDataUrl(url);
if (!source) return;
targets.push({
base64: source.base64,
mediaType: source.mediaType,
replace: (data: string, mediaType: string) => {
imageUrl.url = `data:${mediaType};base64,${data}`;
},
drop: () => {
// Preserve the original image URL when it cannot be normalized.
},
// The drop above is a no-op, so these bytes are still on the wire and must keep
// counting against the budget. Without this the core would stop counting them and
// the demotion loop could stop early, shipping a body that is still oversized.
retainsBytesOnDrop: true,
});
});
if (targets.length === 0) return;
await normalizeImageTargets(targets, {
budget: OPENAI_CHAT_IMAGE_BASE64_BUDGET,
overflowAction: "none",
...options,
});
}