@sema-agent/core/dist/core/image-admission.js

Stateless, task-oriented AI agent core

import { MCP_IMAGE_MAX_BASE64 } from "./image-downsample.js";
import { claimedMediaType } from "./media-byte-cap.js";
import { readMember } from "./thrown-value.js";
export const MAX_IMAGE_READ_BYTES = 5 * 1024 * 1024;
export const MAX_IMAGE_DOWNSAMPLE_INPUT_BYTES = 64 * 1024 * 1024;
export const PDF_TARGET_RAW_SIZE = 20 * 1024 * 1024;
export function imageMagicMatches(bytes, mimeType) {
    const b = bytes;
    switch (mimeType) {
        case "image/png":
            return b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47 && b[4] === 0x0d && b[5] === 0x0a && b[6] === 0x1a && b[7] === 0x0a;
        case "image/jpeg":
            return b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff;
        case "image/gif":
            return b.length >= 6 && b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x38 && (b[4] === 0x37 || b[4] === 0x39) && b[5] === 0x61;
        case "image/webp":
            return b.length >= 12 && b[0] === 0x52 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x46 && b[8] === 0x57 && b[9] === 0x45 && b[10] === 0x42 && b[11] === 0x50;
        default:
            return false;
    }
}
export function pdfMagicMatches(bytes) {
    return bytes.length >= 5 && bytes[0] === 0x25 && bytes[1] === 0x50 && bytes[2] === 0x44 && bytes[3] === 0x46 && bytes[4] === 0x2d;
}
export const ADMITTED_IMAGE_MIME_TYPES = Object.freeze(["image/png", "image/jpeg", "image/gif", "image/webp"]);
export function imageReadCap(downsampler) {
    return downsampler ? MAX_IMAGE_DOWNSAMPLE_INPUT_BYTES : MAX_IMAGE_READ_BYTES;
}
export async function admitImageBytes(bytes, claimedMime, downsampler) {
    const rawBytes = bytes.byteLength;
    if (!ADMITTED_IMAGE_MIME_TYPES.includes(claimedMime))
        return { ok: false, reason: "mime_unsupported", rawBytes };
    const cap = imageReadCap(downsampler);
    if (rawBytes > cap)
        return { ok: false, reason: "raw_too_large", rawBytes, cap };
    if (!imageMagicMatches(bytes, claimedMime))
        return { ok: false, reason: "magic_mismatch", rawBytes };
    let base64;
    let mimeType = claimedMime;
    let dimensions;
    if (downsampler) {
        const out = await downsampler(Buffer.from(bytes), claimedMime).catch(() => undefined);
        if (out) {
            base64 = out.base64;
            mimeType = out.mimeType;
            dimensions = out.dimensions;
        }
    }
    base64 ??= Buffer.from(bytes).toString("base64");
    if (base64.length > MCP_IMAGE_MAX_BASE64)
        return { ok: false, reason: "base64_too_large", rawBytes, base64 };
    const resized = dimensions !== undefined &&
        dimensions.displayWidth > 0 &&
        dimensions.displayHeight > 0 &&
        (dimensions.originalWidth !== dimensions.displayWidth || dimensions.originalHeight !== dimensions.displayHeight);
    return { ok: true, block: { type: "image", data: base64, mimeType }, originalSize: rawBytes, ...(dimensions !== undefined ? { dimensions } : {}), resized };
}
export const ADMITTED_DOCUMENT_MIME_TYPES = Object.freeze(["application/pdf"]);
export function admitDocumentBytes(bytes, claimedMime) {
    const rawBytes = bytes.byteLength;
    if (!ADMITTED_DOCUMENT_MIME_TYPES.includes(claimedMime))
        return { ok: false, reason: "mime_unsupported", rawBytes };
    if (rawBytes > PDF_TARGET_RAW_SIZE)
        return { ok: false, reason: "raw_too_large", rawBytes, cap: PDF_TARGET_RAW_SIZE };
    if (!pdfMagicMatches(bytes))
        return { ok: false, reason: "magic_mismatch", rawBytes };
    return { ok: true, block: { type: "document", data: Buffer.from(bytes).toString("base64"), mimeType: claimedMime }, originalSize: rawBytes };
}
const mediaTypeEcho = (mime) => typeof mime === "string" && /^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}\/[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]{0,126}$/.test(mime) ? mime : undefined;
const ADMITTED_FOR = { image: ADMITTED_IMAGE_MIME_TYPES, document: ADMITTED_DOCUMENT_MIME_TYPES };
const REFUSAL_CLAUSE = {
    undecodable: () => "it carried no base64 payload",
    mime_unsupported: (r) => `${r.mimeType ?? "its media type"} is not a ${r.type} type the model accepts (${ADMITTED_FOR[r.type].join(", ")})`,
    raw_too_large: (r) => `its ${r.bytes} bytes exceed the ${r.limit}-byte cap`,
    magic_mismatch: (r) => `its ${r.bytes} bytes do not carry the ${r.mimeType} signature`,
    base64_too_large: (r) => `its base64 encoding exceeds the ${r.limit}-character provider limit (${r.bytes} bytes)`,
};
const placeholderOf = (toolName, r) => ({
    type: "text",
    text: `[${r.type} from the ${JSON.stringify(toolName)} tool not shown: ${REFUSAL_CLAUSE[r.reason](r)} (media admission: ${r.reason})]`,
});
const claimsMedia = (block) => claimedMediaType(block) !== undefined;
async function admitBlock(block) {
    const type = claimedMediaType(block);
    if (type === undefined)
        return { block };
    const data = readMember(block, "data");
    const mime = readMember(block, "mimeType");
    const echo = mediaTypeEcho(mime);
    const refusal = (reason, bytes, limit) => ({
        refusal: { type, reason, bytes, ...(limit !== undefined ? { limit } : {}), ...(echo !== undefined ? { mimeType: echo } : {}) },
    });
    if (typeof data !== "string")
        return refusal("undecodable", 0);
    let a;
    try {
        const bytes = Buffer.from(data, "base64");
        const claimed = typeof mime === "string" ? mime : "";
        a = type === "image" ? await admitImageBytes(bytes, claimed, undefined) : admitDocumentBytes(bytes, claimed);
    }
    catch {
        return refusal("undecodable", 0);
    }
    if (a.ok)
        return { block: a.block };
    if (a.reason === "raw_too_large")
        return refusal(a.reason, a.rawBytes, a.cap);
    if (a.reason === "base64_too_large")
        return refusal(a.reason, a.rawBytes, MCP_IMAGE_MAX_BASE64);
    return refusal(a.reason, a.rawBytes);
}
export function admitToolResultMedia(content, toolName) {
    let blocks;
    try {
        if (!Array.isArray(content))
            return undefined;
        blocks = Array.from(content);
        if (!blocks.some(claimsMedia))
            return undefined;
    }
    catch {
        return undefined;
    }
    return (async () => {
        const refused = [];
        const out = [];
        for (const b of blocks) {
            const r = await admitBlock(b);
            if ("block" in r) {
                out.push(r.block);
            }
            else {
                refused.push(r.refusal);
                out.push(placeholderOf(toolName, r.refusal));
            }
        }
        return { content: out, refused };
    })();
}