@just-every/ensemble
Version:
LLM provider abstraction layer with unified streaming interface
67 lines • 2.66 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.setEnsembleRequestFunction = setEnsembleRequestFunction;
exports.convertImageToText = convertImageToText;
exports.convertImageToTextIfNeeded = convertImageToTextIfNeeded;
const model_data_js_1 = require("../data/model_data.cjs");
let ensembleRequestFunction = null;
function setEnsembleRequestFunction(fn) {
ensembleRequestFunction = fn;
}
const imageDescriptionCache = {};
function generateImageHash(imageData) {
const sample = imageData.substring(0, 100);
const length = imageData.length;
return `${sample}_${length}`;
}
async function convertImageToText(imageData, modelId) {
if (!imageData.startsWith('data:image/')) {
return imageData;
}
console.log(`Converting image to text description for model ${modelId}`);
const imageHash = generateImageHash(imageData);
if (imageDescriptionCache[imageHash]) {
console.log(`Using cached image description for ${modelId}`);
return imageDescriptionCache[imageHash];
}
if (!ensembleRequestFunction) {
console.error('Ensemble request function not set for image-to-text conversion');
return 'Image found, but could not be converted to text (circular dependency issue)';
}
try {
const stream = ensembleRequestFunction([
{
type: 'message',
role: 'system',
content: 'Please describe the following image in a couple of sentences. Focus on the main visual elements and key details that someone would need to understand what is shown in the image.',
},
{
type: 'message',
role: 'user',
content: imageData,
},
], {
modelClass: 'vision_mini',
});
for await (const event of stream) {
if (event.type === 'message_complete' && 'content' in event) {
imageDescriptionCache[imageHash] = event.content.trim();
return imageDescriptionCache[imageHash];
}
}
}
catch (error) {
console.error('Error generating image description:', error);
}
return 'Image found, but could not be converted to text';
}
async function convertImageToTextIfNeeded(imageData, modelId) {
if (!imageData.startsWith('data:image/')) {
return false;
}
if (modelId && (0, model_data_js_1.findModel)(modelId)?.features?.input_modality?.includes('image')) {
return false;
}
return await convertImageToText(imageData, modelId || 'unknown');
}
//# sourceMappingURL=image_to_text.js.map