@just-every/ensemble
Version:
LLM provider abstraction layer with unified streaming interface
226 lines • 8.74 kB
JavaScript
import { Buffer } from 'buffer';
import { v4 as uuidv4 } from 'uuid';
let sharpModule = null;
async function getSharp() {
if (!sharpModule) {
try {
const module = await import('sharp');
sharpModule = module.default || module;
}
catch {
throw new Error('Sharp is required for image processing but not installed. Please install it with: npm install sharp');
}
}
return sharpModule;
}
export const MAX_IMAGE_HEIGHT = 2000;
export const DEFAULT_QUALITY = 80;
export const OPENAI_MAX_WIDTH = 1024;
export const OPENAI_MAX_HEIGHT = 768;
export const CLAUDE_MAX_WIDTH = 1024;
export const CLAUDE_MAX_HEIGHT = 1120;
export const GEMINI_MAX_WIDTH = 1024;
export const GEMINI_MAX_HEIGHT = 1536;
import { convertImageToTextIfNeeded } from './image_to_text.js';
export async function appendMessageWithImage(model, input, message, param, addImagesToInput, source) {
const content = typeof param === 'string'
? typeof message[param] === 'string'
? message[param]
: JSON.stringify(message[param])
: param.read();
const extracted = extractBase64Image(content);
if (!extracted.found) {
input.push(message);
return input;
}
let imagesConverted = false;
for (const [image_id, imageData] of Object.entries(extracted.images)) {
const imageToText = await convertImageToTextIfNeeded(imageData, model);
if (imageToText && typeof imageToText === 'string') {
extracted.replaceContent.replaceAll(`[image #${image_id}]`, `[image #${image_id}: ${imageToText}]`);
imagesConverted = true;
}
}
if (typeof param === 'string') {
const newMessage = { ...message };
newMessage[param] = extracted.replaceContent;
input.push(newMessage);
}
else {
input.push(param.write(extracted.replaceContent));
}
if (!imagesConverted) {
input = await addImagesToInput(input, extracted.images, source || `${message.role} message`);
}
return input;
}
export function extractBase64Image(content) {
const result = {
found: false,
originalContent: content,
replaceContent: content,
image_id: null,
images: {},
};
if (typeof content !== 'string')
return result;
if (!content.includes('data:') || !content.includes('base64,'))
return result;
const imgRegex = /data:(?:image\/)?([a-zA-Z0-9.+-]+);base64,[A-Za-z0-9+/\s]*={0,2}/g;
const images = {};
const replaceContent = content.replace(imgRegex, match => {
const id = uuidv4();
const mimeMatch = match.match(/data:(?:image\/)?([a-zA-Z0-9.+-]+);base64,/);
const mime = mimeMatch ? mimeMatch[1] : '';
const base64Start = match.indexOf('base64,') + 7;
let base64Data = match.substring(base64Start);
base64Data = base64Data.replace(/\s+/g, '');
let startBinary = '';
let endBinary = '';
if (mime === 'png') {
startBinary = '\x89PNG\r\n\x1A\n';
endBinary = '\x00\x00\x00\x00IEND\xAE\x42\x60\x82';
}
else if (mime === 'jpeg' || mime === 'jpg') {
startBinary = '\xFF\xD8\xFF';
endBinary = '\xFF\xD9';
}
else if (mime === 'gif') {
startBinary = 'GIF87a';
if (base64Data.startsWith('R0lGODlh') || base64Data.startsWith('R0lGODdh')) {
startBinary = 'GIF89a';
}
endBinary = '\x3B';
}
let goodBase64 = base64Data;
if (startBinary && endBinary) {
let l = Math.floor(base64Data.length / 4) * 4;
let found = false;
while (l >= ((startBinary.length + endBinary.length) * 4) / 3) {
try {
const bin = atob(base64Data.substr(0, l));
if (bin.startsWith(startBinary) && bin.endsWith(endBinary)) {
goodBase64 = base64Data.substr(0, l);
found = true;
break;
}
}
catch {
}
l -= 4;
}
if (!found) {
const cleanedMatch = base64Data.match(/^([A-Za-z0-9+/]*)(={0,2})$/);
if (cleanedMatch) {
goodBase64 = cleanedMatch[1] + cleanedMatch[2];
}
}
}
else {
const cleanedMatch = base64Data.match(/^([A-Za-z0-9+/]*)(={0,2})$/);
if (cleanedMatch) {
goodBase64 = cleanedMatch[1] + cleanedMatch[2];
}
}
const prefix = mime.includes('/') ? 'data:' : 'data:image/';
images[id] = `${prefix}${mime};base64,${goodBase64}`;
return `[image #${id}]`;
});
if (Object.keys(images).length === 0) {
return result;
}
const firstImageId = Object.keys(images)[0];
return {
found: true,
originalContent: content,
replaceContent: replaceContent,
image_id: firstImageId,
images: images,
};
}
export async function resizeAndSplitForOpenAI(imageData) {
const MAX_WIDTH = 1024;
const MAX_HEIGHT = 768;
const base64Image = imageData.replace(/^data:image\/\w+;base64,/, '');
const imageFormat = imageData.match(/data:image\/(\w+);/)?.[1] || 'png';
const imageBuffer = Buffer.from(base64Image, 'base64');
const sharp = await getSharp();
const { width: origW = 0, height: origH = 0 } = await sharp(imageBuffer).metadata();
if (origW <= MAX_WIDTH && origH <= MAX_HEIGHT) {
return [imageData];
}
const newWidth = Math.min(origW, MAX_WIDTH);
const resizedBuffer = await sharp(imageBuffer)
.resize({ width: newWidth })
.flatten({ background: '#fff' })
.toFormat(imageFormat)
.toBuffer();
const { height: resizedH = 0 } = await sharp(resizedBuffer).metadata();
const result = [];
if (resizedH > MAX_HEIGHT) {
const segments = Math.ceil(resizedH / MAX_HEIGHT);
for (let i = 0; i < segments; i++) {
const top = i * MAX_HEIGHT;
const height = Math.min(MAX_HEIGHT, resizedH - top);
if (height <= 0)
continue;
const segmentBuf = await sharp(resizedBuffer)
.extract({ left: 0, top, width: newWidth, height })
.toFormat(imageFormat)
.toBuffer();
const segmentDataUrl = `data:image/${imageFormat};base64,${segmentBuf.toString('base64')}`;
result.push(segmentDataUrl);
}
}
else {
const singleUrl = `data:image/${imageFormat};base64,${resizedBuffer.toString('base64')}`;
result.push(singleUrl);
}
return result;
}
function stripDataUrl(dataUrl) {
const match = dataUrl.match(/^data:image\/([^;]+);base64,(.+)$/);
if (!match)
throw new Error('Invalid data-URL');
return { format: match[1], base64: match[2] };
}
async function processAndTruncate(imageBuffer, format, maxW, maxH) {
const sharp = await getSharp();
const resized = await sharp(imageBuffer)
.rotate()
.resize({ width: maxW, withoutEnlargement: true })
.flatten({ background: '#fff' })
.toFormat(format)
.toBuffer();
const { width, height } = await sharp(resized).metadata();
if (height > maxH) {
return await sharp(resized)
.extract({ left: 0, top: 0, width: width, height: maxH })
.toFormat(format)
.toBuffer();
}
return resized;
}
export async function resizeAndTruncateForClaude(imageData) {
const { format, base64 } = stripDataUrl(imageData);
const buf = Buffer.from(base64, 'base64');
const sharp = await getSharp();
const meta = await sharp(buf).metadata();
if (meta.width <= CLAUDE_MAX_WIDTH && meta.height <= CLAUDE_MAX_HEIGHT) {
return imageData;
}
const outBuf = await processAndTruncate(buf, format, CLAUDE_MAX_WIDTH, CLAUDE_MAX_HEIGHT);
return `data:image/${format};base64,${outBuf.toString('base64')}`;
}
export async function resizeAndTruncateForGemini(imageData) {
const { format, base64 } = stripDataUrl(imageData);
const buf = Buffer.from(base64, 'base64');
const sharp = await getSharp();
const meta = await sharp(buf).metadata();
if (meta.width <= GEMINI_MAX_WIDTH && meta.height <= GEMINI_MAX_HEIGHT) {
return imageData;
}
const outBuf = await processAndTruncate(buf, format, GEMINI_MAX_WIDTH, GEMINI_MAX_HEIGHT);
return `data:image/${format};base64,${outBuf.toString('base64')}`;
}
//# sourceMappingURL=image_utils.js.map