@just-every/ensemble
Version:
LLM provider abstraction layer with unified streaming interface
499 lines • 19.3 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.GEMINI_MAX_HEIGHT = exports.GEMINI_MAX_WIDTH = exports.CLAUDE_MAX_HEIGHT = exports.CLAUDE_MAX_WIDTH = exports.OPENAI_MAX_HEIGHT = exports.OPENAI_MAX_WIDTH = exports.DEFAULT_QUALITY = exports.MAX_IMAGE_HEIGHT = void 0;
exports.appendMessageWithImage = appendMessageWithImage;
exports.extractBase64Image = extractBase64Image;
exports.normalizeImageDataUrl = normalizeImageDataUrl;
exports.resizeDataUrl = resizeDataUrl;
exports.resizeAndSplitForOpenAI = resizeAndSplitForOpenAI;
exports.resizeAndTruncateForClaude = resizeAndTruncateForClaude;
exports.resizeAndTruncateForGemini = resizeAndTruncateForGemini;
const buffer_1 = require("buffer");
const uuid_1 = require("uuid");
const image_validation_js_1 = require("./image_validation.cjs");
let sharpModule = null;
function resolveSharpModule(candidate) {
if (typeof candidate === 'function') {
return candidate;
}
if (!candidate || typeof candidate !== 'object') {
return null;
}
const moduleObj = candidate;
if (typeof moduleObj.default === 'function') {
return moduleObj.default;
}
if (moduleObj.default && typeof moduleObj.default === 'object') {
const nested = moduleObj.default;
if (typeof nested.default === 'function') {
return nested.default;
}
}
if (typeof moduleObj.sharp === 'function') {
return moduleObj.sharp;
}
return null;
}
async function getSharp() {
if (!sharpModule) {
try {
const module = await Promise.resolve().then(() => __importStar(require('sharp')));
sharpModule = resolveSharpModule(module);
if (!sharpModule) {
throw new Error('Sharp module loaded but export shape was not callable');
}
}
catch {
throw new Error('Sharp is required for image processing but not installed. Please install it with: npm install sharp');
}
}
return sharpModule;
}
async function getSharpOrNull() {
try {
return await getSharp();
}
catch {
return null;
}
}
exports.MAX_IMAGE_HEIGHT = 2000;
exports.DEFAULT_QUALITY = 80;
exports.OPENAI_MAX_WIDTH = 1024;
exports.OPENAI_MAX_HEIGHT = 768;
exports.CLAUDE_MAX_WIDTH = 1024;
exports.CLAUDE_MAX_HEIGHT = 1120;
exports.GEMINI_MAX_WIDTH = 2048;
exports.GEMINI_MAX_HEIGHT = 2528;
const image_to_text_js_1 = require("./image_to_text.cjs");
async function appendMessageWithImage(model, input, message, param, addImagesToInput, source) {
const content = typeof param === 'string'
? typeof message[param] === 'string'
? message[param]
: JSON.stringify(message[param])
: param.read();
const extracted = extractBase64Image(content);
if (!extracted.found) {
input.push(message);
return input;
}
let imagesConverted = false;
for (const [image_id, imageData] of Object.entries(extracted.images)) {
const imageToText = await (0, image_to_text_js_1.convertImageToTextIfNeeded)(imageData, model);
if (imageToText && typeof imageToText === 'string') {
extracted.replaceContent.replaceAll(`[image #${image_id}]`, `[image #${image_id}: ${imageToText}]`);
imagesConverted = true;
}
}
if (typeof param === 'string') {
const newMessage = { ...message };
newMessage[param] = extracted.replaceContent;
input.push(newMessage);
}
else {
input.push(param.write(extracted.replaceContent));
}
if (!imagesConverted) {
input = await addImagesToInput(input, extracted.images, source || `${message.role} message`);
}
return input;
}
function extractBase64Image(content) {
const result = {
found: false,
originalContent: content,
replaceContent: content,
image_id: null,
images: {},
};
if (typeof content !== 'string')
return result;
if (!content.includes('data:') || !content.includes('base64,'))
return result;
const imgRegex = /data:(?:image\/)?([a-zA-Z0-9.+-]+);base64,[A-Za-z0-9+/\s]*={0,2}/g;
const images = {};
const replaceContent = content.replace(imgRegex, match => {
const id = (0, uuid_1.v4)();
const mimeMatch = match.match(/data:(?:image\/)?([a-zA-Z0-9.+-]+);base64,/);
const mime = mimeMatch ? mimeMatch[1] : '';
const base64Start = match.indexOf('base64,') + 7;
let base64Data = match.substring(base64Start);
base64Data = base64Data.replace(/\s+/g, '');
let startBinary = '';
let endBinary = '';
if (mime === 'png') {
startBinary = '\x89PNG\r\n\x1A\n';
endBinary = '\x00\x00\x00\x00IEND\xAE\x42\x60\x82';
}
else if (mime === 'jpeg' || mime === 'jpg') {
startBinary = '\xFF\xD8\xFF';
endBinary = '\xFF\xD9';
}
else if (mime === 'gif') {
startBinary = 'GIF87a';
if (base64Data.startsWith('R0lGODlh') || base64Data.startsWith('R0lGODdh')) {
startBinary = 'GIF89a';
}
endBinary = '\x3B';
}
let goodBase64 = base64Data;
if (startBinary && endBinary) {
let l = Math.floor(base64Data.length / 4) * 4;
let found = false;
while (l >= ((startBinary.length + endBinary.length) * 4) / 3) {
try {
const bin = atob(base64Data.substr(0, l));
if (bin.startsWith(startBinary) && bin.endsWith(endBinary)) {
goodBase64 = base64Data.substr(0, l);
found = true;
break;
}
}
catch {
}
l -= 4;
}
if (!found) {
const cleanedMatch = base64Data.match(/^([A-Za-z0-9+/]*)(={0,2})$/);
if (cleanedMatch) {
goodBase64 = cleanedMatch[1] + cleanedMatch[2];
}
}
}
else {
const cleanedMatch = base64Data.match(/^([A-Za-z0-9+/]*)(={0,2})$/);
if (cleanedMatch) {
goodBase64 = cleanedMatch[1] + cleanedMatch[2];
}
}
const prefix = mime.includes('/') ? 'data:' : 'data:image/';
images[id] = `${prefix}${mime};base64,${goodBase64}`;
return `[image #${id}]`;
});
if (Object.keys(images).length === 0) {
return result;
}
const firstImageId = Object.keys(images)[0];
return {
found: true,
originalContent: content,
replaceContent: replaceContent,
image_id: firstImageId,
images: images,
};
}
function looksLikeBase64(input) {
const trimmed = input.trim();
if (trimmed.length < 16)
return false;
return /^[A-Za-z0-9+/_-]+={0,2}$/.test(trimmed);
}
function normalizeBase64String(input) {
const cleaned = input.replace(/\s+/g, '');
if (!cleaned)
return '';
const normalized = cleaned.replace(/-/g, '+').replace(/_/g, '/');
const mod = normalized.length % 4;
if (mod === 1)
return null;
const padded = normalized + (mod === 0 ? '' : '='.repeat(4 - mod));
if (!/^[A-Za-z0-9+/]*={0,2}$/.test(padded))
return null;
try {
atob(padded);
}
catch {
return null;
}
return (0, image_validation_js_1.isValidBase64)(padded) ? padded : null;
}
function looksLikeHex(input) {
const trimmed = input.trim();
if (trimmed.length < 32 || trimmed.length % 2 !== 0)
return false;
return /^[0-9a-fA-F]+$/.test(trimmed);
}
function normalizeHexToBase64(input) {
if (!looksLikeHex(input))
return null;
try {
return buffer_1.Buffer.from(input.trim(), 'hex').toString('base64');
}
catch {
return null;
}
}
function getMimeFromMeta(meta) {
const parts = meta
.split(';')
.map(part => part.trim())
.filter(Boolean);
const isBase64 = parts.some(part => part.toLowerCase() === 'base64');
const charset = parts.find(part => part.toLowerCase().startsWith('charset='));
const mime = parts.find(part => !part.toLowerCase().startsWith('charset=') && part.toLowerCase() !== 'base64');
return { mime, charset, isBase64 };
}
function appendCharset(mime, charset) {
return charset ? `${mime};${charset}` : mime;
}
function looksLikeUrl(input) {
const trimmed = input.trim();
if (trimmed.startsWith('http://') || trimmed.startsWith('https://') || trimmed.startsWith('blob:'))
return true;
if (trimmed.startsWith('//'))
return true;
if (trimmed.includes(' ') || trimmed.includes('\n'))
return false;
if (/^[^\s]+\.[^\s]+\//.test(trimmed))
return true;
if (/^[^\s]+\.[^\s]+$/.test(trimmed))
return true;
return false;
}
function normalizeBinaryInput(input) {
const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
return buffer_1.Buffer.from(bytes).toString('base64');
}
function normalizeImageDataUrl(input) {
const raw = input.data ?? input.image_url ?? input.url;
if (!raw)
return {};
if (raw instanceof Uint8Array || raw instanceof ArrayBuffer) {
const base64 = normalizeBinaryInput(raw);
const mimeType = input.mime_type || (0, image_validation_js_1.detectImageType)(base64) || 'image/png';
return { dataUrl: `data:${mimeType};base64,${base64}` };
}
if (typeof raw !== 'string')
return {};
const trimmed = raw.trim();
if (!trimmed)
return {};
if (trimmed.startsWith('data:')) {
const match = trimmed.match(/^data:([^,]*?),(.*)$/s);
if (!match)
return { dataUrl: trimmed };
const meta = match[1] || '';
const payload = match[2] || '';
const { mime, charset, isBase64 } = getMimeFromMeta(meta);
const normalizedBase64 = isBase64 || looksLikeBase64(payload) ? normalizeBase64String(payload) : null;
if (normalizedBase64) {
const detected = (0, image_validation_js_1.detectImageType)(normalizedBase64);
const baseMime = mime || input.mime_type || detected || 'image/png';
const mimeType = appendCharset(baseMime, charset);
return { dataUrl: `data:${mimeType};base64,${normalizedBase64}` };
}
const rawPayload = payload.trim();
const decoded = (() => {
try {
return decodeURIComponent(rawPayload);
}
catch {
return rawPayload;
}
})();
if (decoded) {
const svgLike = /^<\?xml|<svg/i.test(decoded);
const baseMime = mime || input.mime_type || (svgLike ? 'image/svg+xml' : 'image/png');
const mimeType = appendCharset(baseMime, charset);
const base64 = buffer_1.Buffer.from(decoded, 'utf8').toString('base64');
return { dataUrl: `data:${mimeType};base64,${base64}` };
}
return {};
}
if (trimmed.includes(';base64,')) {
const match = trimmed.match(/^([^,]*?);base64,(.*)$/s);
if (match) {
const meta = match[1] || '';
const payload = match[2] || '';
const normalizedBase64 = normalizeBase64String(payload);
if (normalizedBase64) {
const { mime, charset } = getMimeFromMeta(meta);
const detected = (0, image_validation_js_1.detectImageType)(normalizedBase64);
const baseMime = mime || input.mime_type || detected || 'image/png';
const mimeType = appendCharset(baseMime, charset);
return { dataUrl: `data:${mimeType};base64,${normalizedBase64}` };
}
}
}
if (trimmed.startsWith('http://') || trimmed.startsWith('https://') || trimmed.startsWith('blob:')) {
return { url: trimmed };
}
if (trimmed.startsWith('//')) {
return { url: `https:${trimmed}` };
}
const base64Candidate = normalizeBase64String(trimmed);
if (base64Candidate) {
const mimeType = input.mime_type || (0, image_validation_js_1.detectImageType)(base64Candidate) || 'image/png';
return { dataUrl: `data:${mimeType};base64,${base64Candidate}` };
}
const hexCandidate = normalizeHexToBase64(trimmed);
if (hexCandidate) {
const mimeType = input.mime_type || (0, image_validation_js_1.detectImageType)(hexCandidate) || 'image/png';
return { dataUrl: `data:${mimeType};base64,${hexCandidate}` };
}
if (looksLikeUrl(trimmed)) {
return { url: trimmed.startsWith('http') ? trimmed : `https://${trimmed.replace(/^\/\//, '')}` };
}
return {};
}
async function resizeDataUrl(dataUrl, width, height, opts) {
const match = /^data:([^;]+);base64,(.+)$/.exec(dataUrl);
if (!match)
return dataUrl;
const [, mime, b64] = match;
const sharp = await getSharpOrNull();
if (!sharp) {
return dataUrl;
}
const input = buffer_1.Buffer.from(b64, 'base64');
const image = sharp(input);
const fit = opts?.fit || 'cover';
const background = opts?.background || (mime.includes('png') ? { r: 0, g: 0, b: 0, alpha: 0 } : '#000');
let pipeline = image.resize({ width, height, fit, background });
const format = opts?.format || (mime.includes('jpeg') ? 'jpeg' : 'png');
if (format === 'jpeg') {
pipeline = pipeline.jpeg({ quality: 92 });
}
else {
pipeline = pipeline.png();
}
const out = await pipeline.toBuffer();
const outMime = format === 'jpeg' ? 'image/jpeg' : 'image/png';
return `data:${outMime};base64,${out.toString('base64')}`;
}
async function resizeAndSplitForOpenAI(imageData) {
const MAX_WIDTH = 1024;
const MAX_HEIGHT = 768;
const sharp = await getSharpOrNull();
if (!sharp) {
return [imageData];
}
try {
const base64Image = imageData.replace(/^data:image\/\w+;base64,/, '');
const imageFormat = imageData.match(/data:image\/(\w+);/)?.[1] || 'png';
const imageBuffer = buffer_1.Buffer.from(base64Image, 'base64');
const { width: origW = 0, height: origH = 0 } = await sharp(imageBuffer).metadata();
if (origW <= MAX_WIDTH && origH <= MAX_HEIGHT) {
return [imageData];
}
const newWidth = Math.min(origW, MAX_WIDTH);
const resizedBuffer = await sharp(imageBuffer)
.resize({ width: newWidth })
.flatten({ background: '#fff' })
.toFormat(imageFormat)
.toBuffer();
const { height: resizedH = 0 } = await sharp(resizedBuffer).metadata();
const result = [];
if (resizedH > MAX_HEIGHT) {
const segments = Math.ceil(resizedH / MAX_HEIGHT);
for (let i = 0; i < segments; i++) {
const top = i * MAX_HEIGHT;
const height = Math.min(MAX_HEIGHT, resizedH - top);
if (height <= 0)
continue;
const segmentBuf = await sharp(resizedBuffer)
.extract({ left: 0, top, width: newWidth, height })
.toFormat(imageFormat)
.toBuffer();
const segmentDataUrl = `data:image/${imageFormat};base64,${segmentBuf.toString('base64')}`;
result.push(segmentDataUrl);
}
}
else {
const singleUrl = `data:image/${imageFormat};base64,${resizedBuffer.toString('base64')}`;
result.push(singleUrl);
}
return result;
}
catch {
return [imageData];
}
}
function stripDataUrl(dataUrl) {
const match = dataUrl.match(/^data:image\/([^;]+);base64,(.+)$/);
if (!match)
throw new Error('Invalid data-URL');
return { format: match[1], base64: match[2] };
}
async function processAndTruncate(imageBuffer, format, maxW, maxH) {
const sharp = await getSharpOrNull();
if (!sharp) {
return imageBuffer;
}
const resized = await sharp(imageBuffer)
.rotate()
.resize({ width: maxW, withoutEnlargement: true })
.flatten({ background: '#fff' })
.toFormat(format)
.toBuffer();
const { width, height } = await sharp(resized).metadata();
if (height > maxH) {
return await sharp(resized)
.extract({ left: 0, top: 0, width: width, height: maxH })
.toFormat(format)
.toBuffer();
}
return resized;
}
async function resizeAndTruncateForClaude(imageData) {
const { format, base64 } = stripDataUrl(imageData);
const buf = buffer_1.Buffer.from(base64, 'base64');
const sharp = await getSharpOrNull();
if (!sharp) {
return imageData;
}
const meta = await sharp(buf).metadata();
if (meta.width <= exports.CLAUDE_MAX_WIDTH && meta.height <= exports.CLAUDE_MAX_HEIGHT) {
return imageData;
}
const outBuf = await processAndTruncate(buf, format, exports.CLAUDE_MAX_WIDTH, exports.CLAUDE_MAX_HEIGHT);
return `data:image/${format};base64,${outBuf.toString('base64')}`;
}
async function resizeAndTruncateForGemini(imageData) {
const { format, base64 } = stripDataUrl(imageData);
const buf = buffer_1.Buffer.from(base64, 'base64');
const sharp = await getSharpOrNull();
if (!sharp) {
return imageData;
}
const meta = await sharp(buf).metadata();
if (meta.width <= exports.GEMINI_MAX_WIDTH && meta.height <= exports.GEMINI_MAX_HEIGHT) {
return imageData;
}
const outBuf = await processAndTruncate(buf, format, exports.GEMINI_MAX_WIDTH, exports.GEMINI_MAX_HEIGHT);
return `data:image/${format};base64,${outBuf.toString('base64')}`;
}
//# sourceMappingURL=image_utils.js.map