claude-gemini-multimodal-bridge
Version:
Enterprise-grade AI integration bridge connecting Claude Code, Gemini CLI, and Google AI Studio with intelligent routing and advanced multimodal processing capabilities
407 lines (406 loc) • 17 kB
JavaScript
import { MultimodalProcessArgsSchema, } from '../core/types.js';
import { LayerManager } from '../core/LayerManager.js';
import { logger } from '../utils/logger.js';
import { retry, safeExecute } from '../utils/errorHandler.js';
import { AuthVerifier } from '../auth/AuthVerifier.js';
import path from 'path';
import fs from 'fs/promises';
export class MultimodalProcess {
layerManager;
authVerifier;
MAX_FILES = 50;
MAX_FILE_SIZE = 500 * 1024 * 1024;
SUPPORTED_FORMATS = {
images: ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp', '.svg', '.tiff'],
documents: ['.pdf', '.txt', '.md', '.doc', '.docx', '.rtf', '.odt'],
audio: ['.mp3', '.wav', '.m4a', '.flac', '.aac', '.ogg'],
video: ['.mp4', '.mov', '.avi', '.webm', '.mkv', '.wmv'],
presentations: ['.ppt', '.pptx', '.odp'],
spreadsheets: ['.xls', '.xlsx', '.ods', '.csv'],
};
constructor(config) {
const defaultConfig = {
gemini: { api_key: '', model: 'gemini-2.5-pro', timeout: 60000, max_tokens: 16384, temperature: 0.2 },
claude: { code_path: 'claude', timeout: 300000 },
aistudio: { enabled: true, max_files: 10, max_file_size: 100 },
cache: { enabled: true, ttl: 3600 },
logging: { level: 'info' },
};
this.layerManager = new LayerManager(config || defaultConfig);
this.authVerifier = new AuthVerifier();
}
async processMultimodal(args) {
return safeExecute(async () => {
const startTime = Date.now();
logger.info('Starting multimodal processing', {
fileCount: args.files.length,
workflow: args.workflow,
promptLength: args.prompt.length,
});
const validatedArgs = this.validateArgs(args);
const processedFiles = await this.prepareFiles(validatedArgs.files);
await this.verifyRequiredAuthentications(validatedArgs);
const result = await this.executeWorkflow(validatedArgs, processedFiles);
const totalDuration = Date.now() - startTime;
return {
success: true,
content: result.summary || 'Processing completed',
files_processed: processedFiles.map(f => f.path),
processing_time: totalDuration,
workflow_used: validatedArgs.workflow,
layers_involved: this.extractLayersFromWorkflowResult(result),
metadata: {
total_duration: totalDuration,
tokens_used: this.extractTotalTokens(result),
cost: result.metadata?.total_cost,
},
};
}, {
operationName: 'multimodal-process',
layer: 'claude',
timeout: 600000,
});
}
async processSingleFile(filePath, instructions, options) {
const file = await this.createFileReference(filePath);
return this.processMultimodal({
prompt: instructions,
files: [file],
workflow: this.detectWorkflowType([file], instructions),
options: options || {},
});
}
async processBatch(filePaths, instructions, options) {
const files = await Promise.all(filePaths.map(path => this.createFileReference(path)));
return this.processMultimodal({
prompt: instructions,
files,
workflow: 'generation',
options: {
...options,
batchMode: true,
parallelProcessing: options?.parallelProcessing ?? true,
},
});
}
async analyzeContent(filePaths) {
const files = await Promise.all(filePaths.map(path => this.createFileReference(path)));
const instructions = this.generateAnalysisInstructions(files);
return this.processMultimodal({
prompt: instructions,
files,
workflow: 'analysis',
options: {
detailed: true,
extractMetadata: true,
},
});
}
async convertFiles(filePaths, targetFormat, options) {
const files = await Promise.all(filePaths.map(path => this.createFileReference(path)));
const instructions = `Convert the provided files to ${targetFormat} format. Maintain quality and preserve important content structure.`;
return this.processMultimodal({
prompt: instructions,
files,
workflow: 'conversion',
options: {
...options,
outputFormat: targetFormat,
quality_level: 'quality',
},
});
}
async extractInformation(filePaths, extractionType, options) {
const files = await Promise.all(filePaths.map(path => this.createFileReference(path)));
const instructions = this.generateExtractionInstructions(extractionType);
return this.processMultimodal({
prompt: instructions,
files,
workflow: 'extraction',
options: {
...options,
extractionType,
structured: true,
},
});
}
validateArgs(args) {
const validationResult = MultimodalProcessArgsSchema.safeParse(args);
if (!validationResult.success) {
throw new Error(`Invalid arguments: ${validationResult.error.message}`);
}
if (args.files.length === 0) {
const textOnlyWorkflows = ['generation', 'analysis'];
const isTextOnlyAllowed = textOnlyWorkflows.includes(args.workflow);
if (!isTextOnlyAllowed) {
throw new Error('At least one file must be provided for this workflow type');
}
if (!args.prompt || args.prompt.trim().length < 10) {
throw new Error('For text-only workflows, a detailed prompt (minimum 10 characters) is required');
}
logger.debug('Text-only workflow validated', {
workflow: args.workflow,
promptLength: args.prompt.length,
filesProvided: args.files.length
});
}
if (args.files.length > this.MAX_FILES) {
throw new Error(`Too many files. Maximum ${this.MAX_FILES} files allowed`);
}
if (!args.prompt?.trim()) {
throw new Error('Prompt is required for processing');
}
return validationResult.data;
}
async prepareFiles(files) {
return retry(async () => {
const processedFiles = [];
let totalSize = 0;
for (const file of files) {
await this.verifyFileAccess(file.path);
const stats = await fs.stat(file.path);
const size = stats.size;
totalSize += size;
if (totalSize > this.MAX_FILE_SIZE) {
throw new Error(`Total file size exceeds ${this.MAX_FILE_SIZE / 1024 / 1024}MB limit`);
}
const fileType = this.determineFileType(file.path);
const processedFile = {
path: file.path,
size,
type: fileType,
encoding: file.encoding || 'utf-8',
name: path.basename(file.path),
metadata: {
mimeType: this.getMimeType(file.path),
},
};
processedFiles.push(processedFile);
logger.debug('File prepared for processing', {
name: processedFile.name,
path: processedFile.path,
size: processedFile.size,
type: processedFile.type,
});
}
return processedFiles;
}, {
maxAttempts: 2,
delay: 1000,
operationName: 'prepare-files',
});
}
async verifyFileAccess(filePath) {
try {
await fs.access(filePath, fs.constants.R_OK);
}
catch (error) {
throw new Error(`File not accessible: ${filePath}`);
}
}
determineFileType(filePath) {
const ext = path.extname(filePath).toLowerCase();
if (this.SUPPORTED_FORMATS.images.includes(ext)) {
return 'image';
}
if (ext === '.pdf') {
return 'pdf';
}
if (this.SUPPORTED_FORMATS.documents.includes(ext)) {
return 'document';
}
if (this.SUPPORTED_FORMATS.audio.includes(ext)) {
return 'audio';
}
if (this.SUPPORTED_FORMATS.video.includes(ext)) {
return 'video';
}
return 'text';
}
getMimeType(filePath) {
const ext = path.extname(filePath).toLowerCase();
const mimeTypes = {
'.pdf': 'application/pdf',
'.png': 'image/png',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.gif': 'image/gif',
'.mp3': 'audio/mpeg',
'.wav': 'audio/wav',
'.mp4': 'video/mp4',
'.txt': 'text/plain',
'.md': 'text/markdown',
'.doc': 'application/msword',
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
};
return mimeTypes[ext] || 'application/octet-stream';
}
async verifyRequiredAuthentications(args) {
const requiredServices = this.determineRequiredServices(args);
for (const service of requiredServices) {
let authResult;
switch (service) {
case 'gemini':
authResult = await this.authVerifier.verifyGeminiAuth();
break;
case 'aistudio':
authResult = await this.authVerifier.verifyAIStudioAuth();
break;
case 'claude':
authResult = await this.authVerifier.verifyClaudeCodeAuth();
break;
default:
continue;
}
if (!authResult.success) {
throw new Error(`${service} authentication required: ${authResult.error}`);
}
}
}
determineRequiredServices(args) {
const services = new Set();
services.add('claude');
const hasMultimodalFiles = args.files.some(f => this.determineFileType(f.path) !== 'document' && this.determineFileType(f.path) !== 'text');
if (hasMultimodalFiles || args.workflow === 'analysis') {
services.add('aistudio');
}
const needsGrounding = this.needsGrounding(args.prompt);
if (needsGrounding || args.workflow === 'analysis') {
services.add('gemini');
}
return Array.from(services);
}
needsGrounding(instructions) {
const groundingKeywords = [
'latest', 'recent', 'current', 'up-to-date', 'real-time',
'search', 'find information', 'lookup', 'research',
'today', 'yesterday', 'this week', 'this month', 'this year'
];
const lowerInstructions = instructions.toLowerCase();
return groundingKeywords.some(keyword => lowerInstructions.includes(keyword));
}
async executeWorkflow(args, files) {
return retry(async () => {
const workflowTask = {
type: 'multimodal',
action: 'multimodal_processing',
workflowType: args.workflow,
files,
instructions: args.prompt,
options: args.options || {},
};
logger.info('Executing multimodal workflow', {
workflowType: args.workflow,
fileCount: files.length,
optionsSet: Object.keys(args.options || {}).length,
});
return await this.layerManager.processMultimodal(workflowTask.instructions, workflowTask.files, workflowTask.workflowType, workflowTask.options);
}, {
maxAttempts: 3,
delay: 2000,
operationName: 'execute-workflow',
});
}
detectWorkflowType(files, instructions) {
const fileTypes = files.map(f => this.determineFileType(f.path));
const hasImages = fileTypes.includes('image');
const hasDocuments = fileTypes.includes('document');
const hasAudio = fileTypes.includes('audio');
const hasVideo = fileTypes.includes('video');
const lowerInstructions = instructions.toLowerCase();
if (lowerInstructions.includes('convert') || lowerInstructions.includes('transform')) {
return 'conversion';
}
if (lowerInstructions.includes('extract') || lowerInstructions.includes('get data')) {
return 'extraction';
}
if (lowerInstructions.includes('generate') || lowerInstructions.includes('create')) {
return 'generation';
}
return 'analysis';
}
generateAnalysisInstructions(files) {
const fileTypes = [...new Set(files.map(f => f.type))];
const fileCount = files.length;
let instructions = `Please analyze the provided ${fileCount} file${fileCount > 1 ? 's' : ''}. `;
if (fileTypes.includes('image')) {
instructions += 'For images, describe the visual content, identify objects, text, and any notable features. ';
}
if (fileTypes.includes('document')) {
instructions += 'For documents, summarize the content, extract key information, and identify the main topics. ';
}
if (fileTypes.includes('audio')) {
instructions += 'For audio files, transcribe the content and analyze the speech or music. ';
}
if (fileTypes.includes('video')) {
instructions += 'For videos, describe the visual content and extract any audio/speech. ';
}
instructions += 'Provide a comprehensive analysis with structured output.';
return instructions;
}
generateExtractionInstructions(extractionType) {
const instructions = {
text: 'Extract all text content from the provided files. Maintain formatting and structure where possible.',
metadata: 'Extract metadata information from the files including creation date, author, file properties, and technical details.',
data: 'Extract structured data from the files including tables, lists, numbers, and key-value pairs.',
structure: 'Extract the structural information showing the organization, hierarchy, and layout of the content.',
};
return instructions[extractionType] ||
'Extract relevant information from the provided files.';
}
async createFileReference(filePath) {
const stats = await fs.stat(filePath);
return {
path: filePath,
size: stats.size,
type: this.determineFileType(filePath),
encoding: 'utf-8',
};
}
extractLayersFromWorkflowResult(result) {
const validLayers = ['claude', 'gemini', 'aistudio', 'workflow', 'tool', 'orchestrator'];
const layers = new Set();
Object.values(result.results).forEach(layerResult => {
if (layerResult.metadata?.layer && validLayers.includes(layerResult.metadata.layer)) {
layers.add(layerResult.metadata.layer);
}
});
return Array.from(layers);
}
extractTotalTokens(result) {
return Object.values(result.results).reduce((total, layerResult) => {
return total + (layerResult.metadata?.tokens_used || 0);
}, 0);
}
extractLayersUsed(result) {
const validLayers = ['claude', 'gemini', 'aistudio', 'workflow', 'tool', 'orchestrator'];
const layers = new Set();
if (result.metadata?.layer && validLayers.includes(result.metadata.layer)) {
layers.add(result.metadata.layer);
}
if (typeof result.data === 'object' && result.data !== null) {
const dataStr = JSON.stringify(result.data);
validLayers.forEach(layer => {
if (dataStr.includes(layer)) {
layers.add(layer);
}
});
}
return Array.from(layers);
}
getSupportedFormats() {
return this.SUPPORTED_FORMATS;
}
isFileSupported(filePath) {
const ext = path.extname(filePath).toLowerCase();
return Object.values(this.SUPPORTED_FORMATS)
.some(formats => formats.includes(ext));
}
getProcessingLimits() {
return {
maxFiles: this.MAX_FILES,
maxFileSize: this.MAX_FILE_SIZE,
maxFileSizeMB: this.MAX_FILE_SIZE / 1024 / 1024,
};
}
}