contaigents
Version:
Modular AI Content Ecosystem with Audio Generation
300 lines (296 loc) ā¢ 11.1 kB
JavaScript
import { BaseTool } from './BaseTool.js';
import { LLMFactory } from '../llm/LLMFactory.js';
import path from 'path';
import fs from 'fs/promises';
export class ImageAnalysisTool extends BaseTool {
constructor(baseDir) {
super();
this.baseDir = baseDir || process.cwd();
}
getName() {
return 'image_analysis';
}
getDescription() {
return `Analyze images and answer specific questions about their content, style, composition, text, or any visual elements.
š **CAPABILITIES:**
- **Visual Analysis**: Describe scenes, objects, people, activities
- **Style Analysis**: Artistic techniques, color palettes, composition
- **Text Extraction**: Read text, signs, documents, handwriting
- **Technical Analysis**: UI elements, diagrams, charts, code screenshots
- **Creative Insights**: Mood, atmosphere, artistic inspiration
šÆ **USAGE TIPS:**
- Ask specific questions to get detailed, focused answers
- Use for reference image analysis before generation
- Extract information from screenshots, documents, designs
- Analyze visual content for content creation workflows
**Example Questions:**
- "What's the artistic style and color palette of this image?"
- "Extract all text from this screenshot and organize it"
- "What UI elements and layout patterns do you see?"
- "Describe the mood and atmosphere for creative inspiration"`;
}
getParameters() {
return [
{
name: 'image_path',
type: 'string',
description: 'Path to the image file (relative to project root). Supports JPG, JPEG, PNG, WEBP, GIF formats.',
required: true
},
{
name: 'question',
type: 'string',
description: 'Specific question or instruction about what to analyze in the image. Be detailed and specific for best results.',
required: true
},
{
name: 'detail_level',
type: 'string',
description: 'Level of detail in analysis: "low" (brief), "high" (comprehensive), "auto" (adaptive)',
required: false
}
];
}
async execute(parameters) {
// Validate parameters
const validationError = this.validateParameters(parameters);
if (validationError) {
return validationError;
}
const { image_path, question, detail_level = 'auto' } = parameters;
try {
// Validate question
if (!question || typeof question !== 'string' || !question.trim()) {
return {
success: false,
error: 'Question cannot be empty. Please specify what you want to analyze about the image.'
};
}
// Load and validate image
const imageResult = await this.loadImage(image_path);
if (!imageResult.success) {
return {
success: false,
error: imageResult.error
};
}
// Get Gemini provider for vision analysis
const llmProvider = await this.getGeminiProvider();
if (!llmProvider) {
return {
success: false,
error: 'No configured Gemini provider found. Please configure the Google provider using \'contaigents configure\' command.'
};
}
// Analyze image with Gemini Vision
console.log(`š Analyzing image with Gemini Vision: ${path.basename(image_path)}`);
const analysis = await this.analyzeWithGemini(imageResult.imageData, question, detail_level, llmProvider.apiKey);
return {
success: true,
data: {
image_path: path.relative(this.baseDir, imageResult.resolvedPath),
question,
analysis,
detail_level,
image_size: imageResult.size,
image_format: imageResult.mimeType
},
message: `Image analysis completed. ${analysis.substring(0, 150)}${analysis.length > 150 ? '...' : ''}`
};
}
catch (error) {
return {
success: false,
error: `Failed to analyze image: ${error.message}`
};
}
}
/**
* Load and validate image file
*/
async loadImage(imagePath) {
try {
// Validate and resolve path
const pathValidation = await this.validateAndResolveOutputPath(imagePath);
if (!pathValidation.success) {
return {
success: false,
error: `Invalid image path "${imagePath}": ${pathValidation.error}`
};
}
// Check if file exists
try {
await fs.access(pathValidation.resolvedPath);
}
catch {
return {
success: false,
error: `Image not found: ${imagePath}`
};
}
// Validate file format
const mimeType = this.detectImageMimeType(imagePath);
if (!mimeType) {
return {
success: false,
error: `Unsupported image format for "${imagePath}". Supported formats: JPG, JPEG, PNG, WEBP, GIF`
};
}
// Read and validate file size
const imageBuffer = await fs.readFile(pathValidation.resolvedPath);
const maxSize = 20 * 1024 * 1024; // 20MB limit for vision models
if (imageBuffer.length > maxSize) {
return {
success: false,
error: `Image "${imagePath}" is too large (${this.formatFileSize(imageBuffer.length)}). Maximum size: 20MB`
};
}
// Convert to base64
const base64Data = imageBuffer.toString('base64');
return {
success: true,
imageData: base64Data,
resolvedPath: pathValidation.resolvedPath,
size: imageBuffer.length,
mimeType
};
}
catch (error) {
return {
success: false,
error: `Failed to load image: ${error.message}`
};
}
}
/**
* Analyze image using Gemini Vision
*/
async analyzeWithGemini(base64Data, question, detailLevel, apiKey) {
// Enhance question based on detail level
let enhancedQuestion = question;
if (detailLevel === 'high') {
enhancedQuestion = `Please provide a comprehensive and detailed analysis: ${question}`;
}
else if (detailLevel === 'low') {
enhancedQuestion = `Please provide a brief and concise answer: ${question}`;
}
const requestBody = {
contents: [{
parts: [
{
text: enhancedQuestion
},
{
inline_data: {
mime_type: 'image/jpeg', // Gemini accepts various formats
data: base64Data
}
}
]
}],
generationConfig: {
temperature: 0.4,
topK: 32,
topP: 1,
maxOutputTokens: 4096,
}
};
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent?key=${apiKey}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody),
});
if (!response.ok) {
const errorText = await response.text();
let errorMessage = `Gemini Vision API error: ${response.status} ${response.statusText}`;
try {
const errorData = JSON.parse(errorText);
errorMessage = `Gemini Vision API error: ${errorData.error?.message || errorText}`;
}
catch {
errorMessage = `Gemini Vision API error: ${errorText}`;
}
throw new Error(errorMessage);
}
const data = await response.json();
if (!data.candidates || !data.candidates[0] || !data.candidates[0].content) {
throw new Error('No analysis result received from Gemini Vision API');
}
return data.candidates[0].content.parts[0].text;
}
/**
* Get configured Gemini provider
*/
async getGeminiProvider() {
try {
const llm = LLMFactory.getProvider('google');
if (await llm.isConfigured()) {
const config = llm.getConfig();
return { apiKey: config.apiKey };
}
return null;
}
catch {
return null;
}
}
/**
* Validate and resolve output path with security checks
*/
async validateAndResolveOutputPath(outputPath) {
try {
if (!outputPath || typeof outputPath !== 'string') {
return {
success: false,
error: 'Path cannot be empty'
};
}
// Resolve path relative to base directory
const resolvedPath = path.resolve(this.baseDir, outputPath);
// For image analysis, we allow reading files outside the project directory
// This is safe since we're only reading, not writing
return {
success: true,
resolvedPath
};
}
catch (error) {
return {
success: false,
error: `Path validation failed: ${error.message}`
};
}
}
/**
* Detect MIME type from file extension
*/
detectImageMimeType(filePath) {
const ext = path.extname(filePath).toLowerCase();
switch (ext) {
case '.jpg':
case '.jpeg':
return 'image/jpeg';
case '.png':
return 'image/png';
case '.webp':
return 'image/webp';
case '.gif':
return 'image/gif';
default:
return null;
}
}
/**
* Format file size in human-readable format
*/
formatFileSize(bytes) {
if (bytes === 0)
return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i];
}
}