@just-every/ensemble
Version:
LLM provider abstraction layer with unified streaming interface
214 lines • 8.62 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.GEMINI_MAX_HEIGHT = exports.GEMINI_MAX_WIDTH = exports.CLAUDE_MAX_HEIGHT = exports.CLAUDE_MAX_WIDTH = exports.OPENAI_MAX_HEIGHT = exports.OPENAI_MAX_WIDTH = exports.DEFAULT_QUALITY = exports.MAX_IMAGE_HEIGHT = void 0;
exports.appendMessageWithImage = appendMessageWithImage;
exports.extractBase64Image = extractBase64Image;
exports.resizeAndSplitForOpenAI = resizeAndSplitForOpenAI;
exports.resizeAndTruncateForClaude = resizeAndTruncateForClaude;
exports.resizeAndTruncateForGemini = resizeAndTruncateForGemini;
const buffer_1 = require("buffer");
const uuid_1 = require("uuid");
let sharpModule = null;
async function getSharp() {
if (!sharpModule) {
try {
const module = await Promise.resolve().then(() => __importStar(require('sharp')));
sharpModule = module.default || module;
}
catch {
throw new Error('Sharp is required for image processing but not installed. Please install it with: npm install sharp');
}
}
return sharpModule;
}
exports.MAX_IMAGE_HEIGHT = 2000;
exports.DEFAULT_QUALITY = 80;
exports.OPENAI_MAX_WIDTH = 1024;
exports.OPENAI_MAX_HEIGHT = 768;
exports.CLAUDE_MAX_WIDTH = 1024;
exports.CLAUDE_MAX_HEIGHT = 1120;
exports.GEMINI_MAX_WIDTH = 1024;
exports.GEMINI_MAX_HEIGHT = 1536;
const image_to_text_js_1 = require("./image_to_text.cjs");
async function appendMessageWithImage(model, input, message, param, addImagesToInput, source) {
const content = typeof param === 'string'
? typeof message[param] === 'string'
? message[param]
: JSON.stringify(message[param])
: param.read();
const extracted = extractBase64Image(content);
if (!extracted.found) {
input.push(message);
return input;
}
let imagesConverted = false;
for (const [image_id, imageData] of Object.entries(extracted.images)) {
const imageToText = await (0, image_to_text_js_1.convertImageToTextIfNeeded)(imageData, model);
if (imageToText && typeof imageToText === 'string') {
extracted.replaceContent.replaceAll(`[image #${image_id}]`, `[image #${image_id}: ${imageToText}]`);
imagesConverted = true;
}
}
if (typeof param === 'string') {
const newMessage = { ...message };
newMessage[param] = extracted.replaceContent;
input.push(newMessage);
}
else {
input.push(param.write(extracted.replaceContent));
}
if (!imagesConverted) {
input = await addImagesToInput(input, extracted.images, source || `${message.role} message`);
}
return input;
}
function extractBase64Image(content) {
const result = {
found: false,
originalContent: content,
replaceContent: content,
image_id: null,
images: {},
};
if (typeof content !== 'string')
return result;
if (!content.includes('data:image/'))
return result;
const imgRegex = /data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\s]+/g;
const images = {};
const replaceContent = content.replace(imgRegex, match => {
const id = (0, uuid_1.v4)();
images[id] = match.replace(/\s+/g, '');
return `[image #${id}]`;
});
if (Object.keys(images).length === 0) {
return result;
}
const firstImageId = Object.keys(images)[0];
return {
found: true,
originalContent: content,
replaceContent: replaceContent,
image_id: firstImageId,
images: images,
};
}
async function resizeAndSplitForOpenAI(imageData) {
const MAX_WIDTH = 1024;
const MAX_HEIGHT = 768;
const base64Image = imageData.replace(/^data:image\/\w+;base64,/, '');
const imageFormat = imageData.match(/data:image\/(\w+);/)?.[1] || 'png';
const imageBuffer = buffer_1.Buffer.from(base64Image, 'base64');
const sharp = await getSharp();
const { width: origW = 0, height: origH = 0 } = await sharp(imageBuffer).metadata();
if (origW <= MAX_WIDTH && origH <= MAX_HEIGHT) {
return [imageData];
}
const newWidth = Math.min(origW, MAX_WIDTH);
const resizedBuffer = await sharp(imageBuffer)
.resize({ width: newWidth })
.flatten({ background: '#fff' })
.toFormat(imageFormat)
.toBuffer();
const { height: resizedH = 0 } = await sharp(resizedBuffer).metadata();
const result = [];
if (resizedH > MAX_HEIGHT) {
const segments = Math.ceil(resizedH / MAX_HEIGHT);
for (let i = 0; i < segments; i++) {
const top = i * MAX_HEIGHT;
const height = Math.min(MAX_HEIGHT, resizedH - top);
if (height <= 0)
continue;
const segmentBuf = await sharp(resizedBuffer)
.extract({ left: 0, top, width: newWidth, height })
.toFormat(imageFormat)
.toBuffer();
const segmentDataUrl = `data:image/${imageFormat};base64,${segmentBuf.toString('base64')}`;
result.push(segmentDataUrl);
}
}
else {
const singleUrl = `data:image/${imageFormat};base64,${resizedBuffer.toString('base64')}`;
result.push(singleUrl);
}
return result;
}
function stripDataUrl(dataUrl) {
const match = dataUrl.match(/^data:image\/([^;]+);base64,(.+)$/);
if (!match)
throw new Error('Invalid data-URL');
return { format: match[1], base64: match[2] };
}
async function processAndTruncate(imageBuffer, format, maxW, maxH) {
const sharp = await getSharp();
const resized = await sharp(imageBuffer)
.rotate()
.resize({ width: maxW, withoutEnlargement: true })
.flatten({ background: '#fff' })
.toFormat(format)
.toBuffer();
const { width, height } = await sharp(resized).metadata();
if (height > maxH) {
return await sharp(resized)
.extract({ left: 0, top: 0, width: width, height: maxH })
.toFormat(format)
.toBuffer();
}
return resized;
}
async function resizeAndTruncateForClaude(imageData) {
const { format, base64 } = stripDataUrl(imageData);
const buf = buffer_1.Buffer.from(base64, 'base64');
const sharp = await getSharp();
const meta = await sharp(buf).metadata();
if (meta.width <= exports.CLAUDE_MAX_WIDTH && meta.height <= exports.CLAUDE_MAX_HEIGHT) {
return imageData;
}
const outBuf = await processAndTruncate(buf, format, exports.CLAUDE_MAX_WIDTH, exports.CLAUDE_MAX_HEIGHT);
return `data:image/${format};base64,${outBuf.toString('base64')}`;
}
async function resizeAndTruncateForGemini(imageData) {
const { format, base64 } = stripDataUrl(imageData);
const buf = buffer_1.Buffer.from(base64, 'base64');
const sharp = await getSharp();
const meta = await sharp(buf).metadata();
if (meta.width <= exports.GEMINI_MAX_WIDTH && meta.height <= exports.GEMINI_MAX_HEIGHT) {
return imageData;
}
const outBuf = await processAndTruncate(buf, format, exports.GEMINI_MAX_WIDTH, exports.GEMINI_MAX_HEIGHT);
return `data:image/${format};base64,${outBuf.toString('base64')}`;
}
//# sourceMappingURL=image_utils.js.map