@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
211 lines (193 loc) • 9.71 kB
text/typescript
import type { OcxContentPart } from "../types";
import { normalizeImageTargets, type NormalizeOptions, type NormalizeTarget } from "./anthropic-image-normalize";
import { MAX_INPUT_BASE64_LENGTH, MAX_INPUT_PIXELS } from "./anthropic-image-codec";
import { sniffImageDimensions } from "./anthropic-image-guard";
// CodeWhisperer native image part (matches Kiro IDE wire format): the base64 bytes live directly in
// userInputMessage.images, NOT in userInputMessageContext. Verified against kiro-gateway.
export interface KiroImage {
format: string; // "jpeg" | "png" | "webp" | "gif" — derived from the media subtype
source: { bytes: string }; // pure base64, no "data:...;base64," prefix
}
/** Parse inline image bytes; remote URLs are not fetched at request-build time. */
function parseDataUrlImage(imageUrl: string): KiroImage | undefined {
if (!imageUrl.startsWith("data:")) return undefined;
const comma = imageUrl.indexOf(",");
if (comma === -1) return undefined;
const header = imageUrl.slice(5, comma);
const bytes = imageUrl.slice(comma + 1);
if (!bytes) return undefined;
const mediaType = header.split(";")[0] || "image/jpeg";
const subtype = (mediaType.includes("/") ? mediaType.split("/")[1] : mediaType) || "jpeg";
// CodeWhisperer/Bedrock expects "jpeg", not the "jpg" alias.
const format = subtype.toLowerCase() === "jpg" ? "jpeg" : subtype.toLowerCase();
return { format, source: { bytes } };
}
export function extractKiroImages(content: string | OcxContentPart[]): KiroImage[] {
if (typeof content === "string") return [];
const out: KiroImage[] = [];
for (const p of content) {
if (p.type !== "image") continue;
const img = parseDataUrlImage(p.imageUrl);
if (img) out.push(img);
}
return out;
}
/**
* Count images Kiro cannot inline, so the loss is never silent.
*
* Kiro's wire carries base64 bytes only, so a remote reference genuinely cannot be
* sent, and this proxy does not fetch one on a request path. Such a part used to be
* dropped with neither bytes nor any trace that an attachment existed. Counting them
* lets the payload builder attach a bounded marker instead.
*
* The count is all that crosses: a remote image URL can carry a signed token, so the
* URL itself is never echoed into prose.
*/
export function countKiroUninlinableImages(content: string | OcxContentPart[]): number {
if (typeof content === "string") return 0;
let count = 0;
for (const p of content) {
if (p.type !== "image") continue;
// Keyed on the scheme, not on parse success: a malformed data URL also fails
// parseDataUrlImage, and labelling that "remote reference" would misstate the cause.
if (!p.imageUrl.startsWith("data:")) count++;
}
return count;
}
/** Bounded, content-free marker for images Kiro could not inline. */
export function kiroUninlinableImageMarker(count: number): string {
if (count <= 0) return "";
if (count === 1) return "[image omitted: remote image references are not supported by this provider]";
return "[" + String(count) + " images omitted: remote image references are not supported by this provider]";
}
/** Report malformed inline references separately from remote URLs. */
export function kiroImageOmissionMarker(content: string | OcxContentPart[]): string {
if (typeof content === "string") return "";
const remote = kiroUninlinableImageMarker(countKiroUninlinableImages(content));
let malformed = 0;
for (const part of content) {
if (part.type === "image" && part.imageUrl.startsWith("data:") && !parseDataUrlImage(part.imageUrl)) malformed++;
}
const inline = malformed === 1
? "[image omitted: malformed inline image data URL]"
: malformed > 1 ? `[${malformed} images omitted: malformed inline image data URLs]` : "";
return [remote, inline].filter(Boolean).join("\n");
}
/**
* Conservative POLICY caps for the CodeWhisperer GenerateAssistantResponse payload.
* The 100-image request cap comes from Kiro's IMAGE_COUNT_EXCEEDED error
* ("101 exceeds limit 100"); the per-message and byte limits are undocumented
* and derived from adjacent AWS surfaces
* (devlog/260714_image_normalization_pipeline/050): Bedrock `Message` allows 20 images
* per message (Converse), and `InvokeModel` caps requests at 25,000,000 bytes — 18MiB
* bounds the IMAGE share of the body with headroom for text/tools.
*/
export const KIRO_IMAGE_BASE64_BUDGET = 18 * 1024 * 1024;
export const KIRO_MAX_IMAGES_PER_MESSAGE = 20;
export const KIRO_MAX_IMAGES_PER_REQUEST = 100;
const COUNT_CAP_NOTE = "[image omitted: exceeded the 20-image per-message cap; oldest images in this message were dropped]";
const REQUEST_CAP_NOTE = "[images omitted: exceeded the 100-image request cap; oldest images in this message were dropped]";
const REQUEST_CAP_EMPTY_NOTE = "[images omitted: exceeded the 100-image request cap; no images remain in this message]";
/** A kiro wire message that can carry images (history userInputMessage or currentMessage). */
interface KiroImageCarrier {
content?: string;
images?: KiroImage[];
}
function isCarrier(v: unknown): v is KiroImageCarrier {
return typeof v === "object" && v !== null;
}
/** Collect image-bearing userInputMessages in wire order (history oldest-first, then current). */
function collectKiroImageCarriers(payload: unknown): KiroImageCarrier[] {
const state = (payload as { conversationState?: { history?: unknown[]; currentMessage?: { userInputMessage?: unknown } } })?.conversationState;
if (!state) return [];
const carriers: KiroImageCarrier[] = [];
for (const entry of state.history ?? []) {
const uim = (entry as { userInputMessage?: unknown })?.userInputMessage;
if (isCarrier(uim)) carriers.push(uim);
}
const current = state.currentMessage?.userInputMessage;
if (isCarrier(current)) carriers.push(current);
return carriers;
}
function appendNote(carrier: KiroImageCarrier, note: string): void {
carrier.content = carrier.content ? `${carrier.content}\n${note}` : note;
}
/** Cheap structural check (no decode or encode) for the request-wide count. */
function countsTowardRequestCap(image: KiroImage): boolean {
const b64 = typeof image.source?.bytes === "string" ? image.source.bytes : "";
if (b64.length === 0 || b64.length > MAX_INPUT_BASE64_LENGTH) return false;
const dims = sniffImageDimensions(b64);
return dims !== null && dims.width * dims.height <= MAX_INPUT_PIXELS;
}
/**
* Apply the generous image pipeline to a built CodeWhisperer payload (mutates in
* place): per-message 20-image cap, then the 100-image request cap over
* structurally usable images (oldest dropped), then the shared tier machinery
* with the kiro budget and terminal-overflow DROP (kiro has no downstream guard).
* Test seams (encode/validate) forward into the core.
*/
export async function normalizeKiroImages(
payload: unknown,
opts?: Pick<NormalizeOptions, "encode" | "validate">,
): Promise<void> {
const carriers = collectKiroImageCarriers(payload);
if (carriers.length === 0) return;
// Pre-pass: per-message count cap (drop oldest within the message).
for (const carrier of carriers) {
const images = carrier.images;
if (!images || images.length <= KIRO_MAX_IMAGES_PER_MESSAGE) continue;
images.splice(0, images.length - KIRO_MAX_IMAGES_PER_MESSAGE);
appendNote(carrier, COUNT_CAP_NOTE);
}
// Request count cap BEFORE the byte budget: a surplus image must not push survivors to
// lower tiers, which #4532 then pins across turns. Only structurally usable images count
// (bytes present, within the bomb limits, dimensions sniffable), so a corrupt image cannot
// evict valid history; the normalizer below still drops it with its own marker. This stays
// a cheap header check: truncated data that still sniffs as an image does count.
let excess = carriers.reduce((count, carrier) => count + (carrier.images ?? []).filter(countsTowardRequestCap).length, 0)
- KIRO_MAX_IMAGES_PER_REQUEST;
for (const carrier of carriers) {
if (excess <= 0) break;
const images = carrier.images;
if (!images?.length) continue;
const kept: KiroImage[] = [];
for (const image of images) {
if (excess > 0 && countsTowardRequestCap(image)) excess--;
else kept.push(image);
}
if (kept.length === images.length) continue;
if (kept.length === 0) delete carrier.images;
else carrier.images = kept;
appendNote(carrier, kept.length === 0 ? REQUEST_CAP_EMPTY_NOTE : REQUEST_CAP_NOTE);
}
// Targets over the survivors, oldest→newest across carriers. Drops resolve the image
// by OBJECT IDENTITY at execution time (indices go stale after earlier splices) and
// delete an emptied images field per the builder's omission contract.
const targets: NormalizeTarget[] = [];
for (const carrier of carriers) {
for (const img of carrier.images ?? []) {
targets.push({
base64: typeof img.source?.bytes === "string" && img.source.bytes.length > 0 ? img.source.bytes : null,
mediaType: `image/${(img.format || "jpeg").toLowerCase()}`,
replace: (data: string, mediaType: string) => {
img.source.bytes = data;
img.format = (mediaType.split("/")[1] ?? "jpeg").toLowerCase();
},
drop: (note: string) => {
const arr = carrier.images;
if (arr) {
const idx = arr.indexOf(img);
if (idx !== -1) arr.splice(idx, 1);
if (arr.length === 0) delete carrier.images;
}
appendNote(carrier, note);
},
});
}
}
await normalizeImageTargets(targets, {
budget: KIRO_IMAGE_BASE64_BUDGET,
overflowAction: "drop",
...(opts ?? {}),
});
}