contaigents/dist/services/ImageContextService.js

Modular AI Content Ecosystem with Audio Generation

import fs from 'fs/promises';
import path from 'path';
export class ImageContextService {
    constructor(baseDir) {
        this.baseDir = baseDir || process.cwd();
    }
    /**
     * Extract image references from user message text
     * Supports patterns like:
     * - "analyze image.jpg"
     * - "look at ./images/photo.png"
     * - "what's in demo/screenshot.jpeg"
     * - "check this image: path/to/file.webp"
     */
    extractImageReferences(message) {
        const imageExtensions = /\.(jpg|jpeg|png|gif|webp|bmp|tiff|svg)$/i;
        const patterns = [
            // Direct file references with common image extensions
            /(?:^|\s)([^\s]+\.(jpg|jpeg|png|gif|webp|bmp|tiff|svg))(?:\s|$)/gi,
            // Quoted file paths
            /"([^"]+\.(jpg|jpeg|png|gif|webp|bmp|tiff|svg))"/gi,
            // After keywords like "image:", "photo:", "picture:"
            /(?:image|photo|picture|screenshot|img):\s*([^\s]+\.(jpg|jpeg|png|gif|webp|bmp|tiff|svg))/gi,
            // File paths with directories
            /(?:^|\s)((?:\.\/|\.\.\/|\/)?[^\s]*\/[^\s]*\.(jpg|jpeg|png|gif|webp|bmp|tiff|svg))(?:\s|$)/gi
        ];
        const foundImages = new Set();
        for (const pattern of patterns) {
            let match;
            while ((match = pattern.exec(message)) !== null) {
                const imagePath = match[1];
                if (imagePath && imageExtensions.test(imagePath)) {
                    foundImages.add(imagePath.trim());
                }
            }
        }
        return Array.from(foundImages);
    }
    /**
     * Load and process images for LLM context
     */
    async processImages(imagePaths) {
        const processedImages = [];
        for (const imagePath of imagePaths) {
            try {
                const imageAttachment = await this.loadImage(imagePath);
                if (imageAttachment) {
                    processedImages.push(imageAttachment);
                }
            }
            catch (error) {
                console.warn(`⚠️  Failed to load image ${imagePath}: ${error.message}`);
                // Continue processing other images even if one fails
            }
        }
        return processedImages;
    }
    /**
     * Load a single image and convert to base64
     */
    async loadImage(imagePath) {
        // Resolve path relative to base directory
        const resolvedPath = path.resolve(this.baseDir, imagePath);
        // Security check: ensure path is within base directory
        if (!resolvedPath.startsWith(path.resolve(this.baseDir))) {
            throw new Error(`Image path "${imagePath}" is outside the allowed directory`);
        }
        // Check if file exists
        try {
            await fs.access(resolvedPath);
        }
        catch {
            throw new Error(`Image file not found: ${imagePath}`);
        }
        // Get file stats
        const stats = await fs.stat(resolvedPath);
        const maxSize = 20 * 1024 * 1024; // 20MB limit for vision models
        if (stats.size > maxSize) {
            throw new Error(`Image "${imagePath}" is too large (${this.formatFileSize(stats.size)}). Maximum size: 20MB`);
        }
        // Detect MIME type
        const mimeType = this.detectImageMimeType(resolvedPath);
        if (!mimeType) {
            throw new Error(`Unsupported image format for "${imagePath}". Supported formats: JPG, JPEG, PNG, WEBP, GIF`);
        }
        // Read and convert to base64
        const imageBuffer = await fs.readFile(resolvedPath);
        const base64Data = imageBuffer.toString('base64');
        return {
            path: path.relative(this.baseDir, resolvedPath),
            base64Data,
            mimeType,
            size: stats.size,
            description: `Image: ${path.basename(imagePath)}`
        };
    }
    /**
     * Detect MIME type from file extension
     */
    detectImageMimeType(filePath) {
        const ext = path.extname(filePath).toLowerCase();
        const mimeTypes = {
            '.jpg': 'image/jpeg',
            '.jpeg': 'image/jpeg',
            '.png': 'image/png',
            '.gif': 'image/gif',
            '.webp': 'image/webp',
            '.bmp': 'image/bmp',
            '.tiff': 'image/tiff',
            '.tif': 'image/tiff',
            '.svg': 'image/svg+xml'
        };
        return mimeTypes[ext] || null;
    }
    /**
     * Format file size for display
     */
    formatFileSize(bytes) {
        const units = ['B', 'KB', 'MB', 'GB'];
        let size = bytes;
        let unitIndex = 0;
        while (size >= 1024 && unitIndex < units.length - 1) {
            size /= 1024;
            unitIndex++;
        }
        return `${size.toFixed(1)}${units[unitIndex]}`;
    }
    /**
     * Check if a message likely contains image references
     */
    hasImageReferences(message) {
        const imageKeywords = [
            'image', 'photo', 'picture', 'screenshot', 'img', 'pic',
            'analyze', 'look at', 'check', 'see', 'view', 'examine'
        ];
        const imageExtensions = /\.(jpg|jpeg|png|gif|webp|bmp|tiff|svg)/i;
        // Check for file extensions
        if (imageExtensions.test(message)) {
            return true;
        }
        // Check for image-related keywords combined with file-like patterns
        const hasKeyword = imageKeywords.some(keyword => message.toLowerCase().includes(keyword));
        const hasFileLikePattern = /[^\s]+\.[a-zA-Z]{2,4}/.test(message);
        return hasKeyword && hasFileLikePattern;
    }
    /**
     * Create a summary of processed images for logging
     */
    createImageSummary(images) {
        if (images.length === 0) {
            return 'No images processed';
        }
        const summary = images.map(img => `${img.description} (${this.formatFileSize(img.size)})`).join(', ');
        return `${images.length} image${images.length > 1 ? 's' : ''}: ${summary}`;
    }
}