claude-gemini-multimodal-bridge
Version:
Enterprise-grade AI integration bridge connecting Claude Code, Gemini CLI, and Google AI Studio with intelligent routing and advanced multimodal processing capabilities
1,172 lines • 88.4 kB
JavaScript
import { execSync, spawn } from 'child_process';
import { createWriteStream, promises as fsPromises } from 'fs';
import { mkdir } from 'fs/promises';
import * as fs from 'fs';
import { dirname, join } from 'path';
import * as path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
import { GoogleGenAI } from '@google/genai';
import { AI_MODELS } from '../core/types.js';
import { logger } from '../utils/logger.js';
import { retry, safeExecute } from '../utils/errorHandler.js';
import { AuthVerifier } from '../auth/AuthVerifier.js';
import { isPlatformWindows, normalizeCrossPlatformPath } from '../utils/platformUtils.js';
import pkg from 'wavefile';
const { WaveFile } = pkg;
const LANGUAGE_PATTERNS = {
ja: /[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FAF]/,
ko: /[\uAC00-\uD7AF]/,
zh: /[\u4E00-\u9FFF]/,
fr: /[àâäéèêëïîôöùûüÿç]/,
de: /[äöüßÄÖÜ]/,
es: /[ñáéíóúü¿¡]/,
ru: /[\u0400-\u04FF]/,
ar: /[\u0600-\u06FF]/,
hi: /[\u0900-\u097F]/,
th: /[\u0E00-\u0E7F]/
};
const SUPPORTED_LANGUAGES = {
ja: 'Japanese',
ko: 'Korean',
zh: 'Chinese',
fr: 'French',
de: 'German',
es: 'Spanish',
ru: 'Russian',
ar: 'Arabic',
hi: 'Hindi',
th: 'Thai'
};
const promptSanitizer = {
'cute': 'friendly-looking',
'adorable': 'appealing',
'sweet': 'pleasant',
'baby': 'young',
'little': 'small-sized',
'tiny': 'miniature',
'sexy': 'elegant',
'hot': 'striking',
'beautiful': 'visually pleasing',
'pretty': 'well-formed'
};
function detectLanguage(text) {
const cleanText = text.trim();
for (const [langCode, pattern] of Object.entries(LANGUAGE_PATTERNS)) {
if (pattern.test(cleanText)) {
return langCode;
}
}
return 'en';
}
function sanitizePrompt(prompt) {
let sanitized = prompt;
for (const [problem, safe] of Object.entries(promptSanitizer)) {
const regex = new RegExp(`\\b${problem}\\b`, 'gi');
sanitized = sanitized.replace(regex, safe);
}
return sanitized;
}
export class AIStudioLayer {
instanceId;
authVerifier;
genAI = null;
geminiLayer;
mcpServerProcess;
persistentMCPProcess;
mcpProcessStartTime = 0;
MCP_PROCESS_TTL = 10 * 60 * 1000;
isInitialized = false;
isLightweightInitialized = false;
lastAuthCheck = 0;
AUTH_CACHE_TTL = 7 * 24 * 60 * 60 * 1000;
DEFAULT_TIMEOUT = 300000;
MAX_RETRIES = 2;
MAX_FILES = 10;
MAX_FILE_SIZE = 50 * 1024 * 1024;
MAX_DOCUMENT_PAGES = 1000;
TOKENS_PER_PAGE = 258;
SUPPORTED_FILE_TYPES = {
images: ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp', '.heic', '.heif'],
documents: ['.pdf', '.txt', '.md', '.doc', '.docx', '.html', '.xml', '.json', '.csv'],
audio: ['.mp3', '.wav', '.m4a', '.flac', '.ogg', '.opus'],
video: ['.mp4', '.mov', '.avi', '.webm', '.mkv', '.flv'],
code: ['.py', '.js', '.ts', '.java', '.cpp', '.c', '.h', '.cs', '.rb', '.go', '.rs']
};
constructor(geminiLayer) {
this.instanceId = `aistudio-${Math.random().toString(36).slice(2, 9)}-${Date.now().toString(36)}`;
logger.info(`🔧 [${this.instanceId}] AIStudioLayer constructor called - with latest translation fixes`, {
instanceId: this.instanceId,
timestamp: Date.now(),
hasGeminiLayer: !!geminiLayer,
version: 'v2025-07-03-instance-tracking'
});
this.setupGhostLogDetection();
this.authVerifier = new AuthVerifier();
this.geminiLayer = geminiLayer;
const apiKey = process.env.AI_STUDIO_API_KEY || process.env.GEMINI_API_KEY || process.env.GOOGLE_AI_STUDIO_API_KEY;
if (apiKey) {
logger.info(`[${this.instanceId}] Creating GoogleGenAI instance with API key`, {
instanceId: this.instanceId,
hasApiKey: !!apiKey,
apiKeySource: process.env.AI_STUDIO_API_KEY ? 'AI_STUDIO_API_KEY' :
process.env.GEMINI_API_KEY ? 'GEMINI_API_KEY' :
process.env.GOOGLE_AI_STUDIO_API_KEY ? 'GOOGLE_AI_STUDIO_API_KEY' : 'unknown'
});
console.trace(`[${this.instanceId}] TRACE: GoogleGenAI instance creation`);
this.genAI = new GoogleGenAI({ apiKey });
logger.info(`[${this.instanceId}] GoogleGenAI instance created successfully`, {
instanceId: this.instanceId,
hasGenAI: !!this.genAI
});
}
}
async initializeLightweight() {
if (this.isLightweightInitialized) {
return;
}
logger.debug(`[${this.instanceId}] Performing lightweight AI Studio initialization...`);
const now = Date.now();
if (now - this.lastAuthCheck > this.AUTH_CACHE_TTL) {
const authResult = await this.authVerifier.verifyAIStudioAuth();
if (!authResult.success) {
throw new Error(`AI Studio authentication failed: ${authResult.error}`);
}
this.lastAuthCheck = now;
}
this.isLightweightInitialized = true;
logger.debug(`[${this.instanceId}] Lightweight AI Studio initialization completed`);
}
async initialize() {
return safeExecute(async () => {
if (this.isInitialized) {
return;
}
logger.info(`[${this.instanceId}] Initializing AI Studio layer...`);
const authResult = await this.authVerifier.verifyAIStudioAuth();
if (!authResult.success) {
throw new Error(`AI Studio authentication failed: ${authResult.error}`);
}
try {
await this.testMCPServerConnection();
}
catch (mcpError) {
logger.warn(`[${this.instanceId}] AI Studio MCP server prerequisites check failed, using MCP-only architecture`, {
instanceId: this.instanceId,
error: mcpError.message,
architecture: 'MCP-only (direct API integration disabled for consistency)'
});
}
this.isInitialized = true;
logger.info(`[${this.instanceId}] AI Studio layer initialized successfully`, {
instanceId: this.instanceId,
authenticated: authResult.success,
architecture: 'MCP-only integration (direct API disabled)',
mcpServerReady: 'Will be started when needed',
});
}, {
operationName: 'initialize-aistudio-layer',
layer: 'aistudio',
timeout: 15000,
});
}
async isAvailable() {
try {
if (!this.isInitialized) {
await this.initialize();
}
return this.isInitialized;
}
catch (error) {
logger.debug('AI Studio layer not available', { error: error.message });
return false;
}
}
canHandle(task) {
if (!task || typeof task !== 'object') {
return false;
}
if (task.type === 'multimodal' || task.action === 'multimodal_processing') {
return true;
}
if (task.action === 'document_analysis' || task.type === 'document') {
return true;
}
if (task.type === 'image' || task.analysisType) {
return true;
}
if (task.action === 'convert' || task.conversion) {
return true;
}
if (task.files && Array.isArray(task.files)) {
return true;
}
if (task.action && (task.action.includes('generate_image') ||
task.action.includes('generate_video') ||
task.action.includes('generate_audio') ||
task.action.includes('generate'))) {
return true;
}
if (task.type === 'generation' || task.workflow === 'generation') {
return true;
}
return false;
}
async execute(task) {
return safeExecute(async () => {
const startTime = Date.now();
if (!this.isInitialized && !this.isLightweightInitialized) {
if (task.action === 'multimodal_process' && (!task.files || task.files.length === 0)) {
await this.initializeLightweight();
}
else {
await this.initialize();
}
}
logger.info(`🔧 [${this.instanceId}] Executing AI Studio task - DEBUG`, {
instanceId: this.instanceId,
taskType: task.type || 'general',
action: task.action || 'execute',
fileCount: task.files ? task.files.length : 0,
taskKeys: Object.keys(task),
taskStructure: JSON.stringify(task, null, 2).substring(0, 500)
});
let result;
switch (task.action || task.type) {
case 'multimodal_processing':
case 'multimodal':
result = await this.processMultimodal(task.files, task.prompt || task.instructions);
break;
case 'document_analysis':
case 'document':
logger.info('🔧 Processing document analysis in execute method', {
hasFiles: !!(task.files || task.documents),
fileCount: (task.files || task.documents || []).length,
hasInstructions: !!task.instructions
});
result = await this.analyzeDocuments(task.files || task.documents, task.instructions);
break;
case 'image':
result = await this.analyzeImage(task.imagePath || task.files?.[0]?.path, task.analysisType || 'detailed');
break;
case 'image_generation':
case 'generate_image':
logger.info('🔧 Calling generateImage from execute method', {
taskType: task.type,
prompt: task.prompt || task.text,
hasOptions: !!task.options
});
result = await this.generateImage(task.prompt || task.text, task.options || {});
break;
case 'video_generation':
case 'generate_video':
throw new Error('Video generation is not yet implemented');
case 'audio_generation':
case 'generate_audio':
result = await this.generateAudio(task.prompt || task.text, task.options || {});
break;
case 'audio_analysis_advanced':
case 'analyze_audio_advanced':
result = await this.analyzeAudioAdvanced(task.audioPath || task.files?.[0]?.path);
break;
case 'convert':
result = await this.convertFiles(task.files, task.outputFormat);
break;
case 'generate_content':
result = await this.processGeneral(task);
break;
default:
result = await this.processGeneral(task);
}
const duration = Date.now() - startTime;
return {
success: true,
data: result,
metadata: {
layer: 'aistudio',
duration,
tokens_used: this.estimateTokensUsed(task, result),
cost: this.calculateCost(task, result),
model: 'gemini-2.5-pro',
},
};
}, {
operationName: 'execute-aistudio-task',
layer: 'aistudio',
timeout: this.getTaskTimeout(task),
});
}
async processMultimodal(files, instructions) {
return retry(async () => {
logger.debug('Processing multimodal files', {
fileCount: files.length,
instructionsLength: instructions.length,
});
this.validateFileTypes(files);
const processedFiles = await this.prepareFilesForProcessing(files);
const startTime = Date.now();
const result = await this.executeMCPCommand('multimodal_process', {
files: processedFiles,
instructions,
options: {
quality: 'high',
includeMetadata: true,
},
});
const processingTime = Date.now() - startTime;
return {
content: result.content || result.response || 'Processing completed',
success: true,
files_processed: files.map(f => f.path),
processing_time: processingTime,
workflow_used: 'analysis',
layers_involved: ['aistudio'],
metadata: {
total_duration: processingTime,
tokens_used: this.estimateTokensUsed({ files, instructions }, result),
cost: this.estimateCost({ files, instructions }, result),
},
};
}, {
maxAttempts: this.MAX_RETRIES,
delay: 3000,
operationName: 'process-multimodal',
});
}
async convertPDFToMarkdown(pdfPath) {
return retry(async () => {
logger.debug('Converting PDF to Markdown', { pdfPath });
const result = await this.executeMCPCommand('convert_pdf', {
input: pdfPath,
output_format: 'markdown',
options: {
preserve_formatting: true,
extract_images: false,
},
});
return result.content || result.markdown || '';
}, {
maxAttempts: this.MAX_RETRIES,
delay: 5000,
operationName: 'convert-pdf-markdown',
});
}
async analyzeImage(imagePath, analysisType) {
return retry(async () => {
const isOCR = analysisType === 'ocr';
const effectiveType = isOCR ? 'extract_text' : analysisType;
logger.debug('Analyzing image', { imagePath, analysisType, isOCR });
const result = await this.executeMCPCommand('analyze_image', {
image: imagePath,
analysis_type: effectiveType,
options: {
detailed: analysisType === 'detailed',
extract_text: effectiveType === 'extract_text',
technical: analysisType === 'technical',
ocr_mode: isOCR,
include_bounding_boxes: isOCR,
},
});
return {
type: analysisType,
description: isOCR
? 'OCR text extraction completed'
: (result.description || 'Image analysis completed'),
extracted_text: result.extracted_text,
technical_details: result.technical_details,
confidence: result.confidence || 0.8,
text_blocks: result.text_blocks,
};
}, {
maxAttempts: this.MAX_RETRIES,
delay: 3000,
operationName: analysisType === 'ocr' ? 'ocr-extraction' : 'analyze-image',
});
}
async transcribeAudio(audioPath) {
return retry(async () => {
logger.debug('Transcribing audio', { audioPath });
const result = await this.executeMCPCommand('transcribe_audio', {
audio: audioPath,
options: {
language: 'auto',
include_timestamps: false,
},
});
return result.transcription || result.text || '';
}, {
maxAttempts: this.MAX_RETRIES,
delay: 5000,
operationName: 'transcribe-audio',
});
}
async generateImage(prompt, options = {}) {
logger.info('🔧 UPDATED generateImage method called - with translation fix', {
timestamp: Date.now(),
version: 'v2025-07-03-translation-fix'
});
logger.info('Generating image using GeminiCLI translation + MCP pattern', {
promptLength: prompt.length,
model: AI_MODELS.IMAGE_GENERATION,
quality: options.quality || 'standard'
});
const startTime = Date.now();
let processedPrompt = prompt;
let translationInfo = {
detectedLanguage: 'en',
languageName: 'English',
wasTranslated: false
};
const safetyPrefixes = [
'digital illustration of',
'artistic rendering of',
'professional diagram showing',
'creative visualization of',
'stylized representation of',
'reference image showing',
'technical visualization of',
'scientific diagram of',
'educational illustration of',
'documentary-style image of'
];
let corePrompt = prompt;
for (const prefix of safetyPrefixes) {
if (prompt.toLowerCase().startsWith(prefix.toLowerCase())) {
corePrompt = prompt.substring(prefix.length).trim();
break;
}
}
logger.info('Language detection analysis', {
originalPrompt: prompt,
corePrompt,
promptLength: prompt.length,
corePromptLength: corePrompt.length
});
const detectedLang = detectLanguage(corePrompt);
if (detectedLang && detectedLang !== 'en') {
const languageName = SUPPORTED_LANGUAGES[detectedLang] || detectedLang;
logger.info(`Non-English prompt detected (${languageName}), using GeminiCLI for translation...`, {
originalPrompt: prompt,
detectedLanguage: detectedLang
});
if (this.geminiLayer) {
logger.info('GeminiCLI layer available, checking translateToEnglish method', {
hasTranslateMethod: typeof this.geminiLayer.translateToEnglish === 'function',
layerType: this.geminiLayer.constructor.name,
availableMethods: Object.getOwnPropertyNames(Object.getPrototypeOf(this.geminiLayer)).slice(0, 10),
hasExecute: typeof this.geminiLayer.execute === 'function'
});
if (typeof this.geminiLayer.translateToEnglish === 'function') {
try {
if (!await this.geminiLayer.isAvailable()) {
logger.warn('GeminiCLI layer not available, initializing...');
await this.geminiLayer.initialize();
}
logger.info('🔧 About to call translateToEnglish method', {
hasMethod: typeof this.geminiLayer.translateToEnglish === 'function',
geminiLayerType: this.geminiLayer.constructor.name,
corePrompt: corePrompt.substring(0, 50)
});
const translatedCore = await this.geminiLayer.translateToEnglish(corePrompt, detectedLang);
const originalPrefix = prompt.substring(0, prompt.length - corePrompt.length);
processedPrompt = originalPrefix + translatedCore;
translationInfo = {
detectedLanguage: detectedLang,
languageName,
originalPrompt: prompt,
translatedPrompt: processedPrompt,
wasTranslated: true
};
logger.info('GeminiCLI translation completed', {
originalPrompt: prompt,
translatedPrompt: processedPrompt,
language: `${languageName} → English`
});
}
catch (translationError) {
logger.warn('Translation unavailable. Continuing with image generation in the input language.', {
error: translationError instanceof Error ? translationError.message : String(translationError),
originalLanguage: detectedLang,
languageName,
originalPrompt: prompt.substring(0, 100)
});
console.log('⚠️ Translation unavailable. Continuing with image generation in the input language.');
}
}
else {
logger.warn('Translation unavailable. Continuing with image generation in the input language.', {
reason: 'translateToEnglish method not found',
originalLanguage: detectedLang,
languageName
});
console.log('⚠️ Translation unavailable. Continuing with image generation in the input language.');
}
}
else {
logger.warn('Translation unavailable. Continuing with image generation in the input language.', {
reason: 'GeminiCLI layer not available',
originalLanguage: detectedLang,
languageName
});
console.log('⚠️ Translation unavailable. Continuing with image generation in the input language.');
}
}
try {
const mcpResult = await this.executeMCPCommand('generate_image', {
prompt: processedPrompt,
numberOfImages: options.numberOfImages || 1,
aspectRatio: options.aspectRatio || '1:1',
personGeneration: options.personGeneration || 'ALLOW',
model: AI_MODELS.IMAGE_GENERATION
});
const duration = Date.now() - startTime;
const outputPath = mcpResult.file?.path || '';
const fileSize = mcpResult.file?.size || 0;
const imageData = mcpResult.imageData || null;
return {
success: true,
generationType: 'image',
outputPath,
originalPrompt: prompt,
metadata: {
duration,
fileSize,
format: 'png',
dimensions: {
width: options.width || 1024,
height: options.height || 1024,
},
model: AI_MODELS.IMAGE_GENERATION,
settings: options,
cost: this.calculateGenerationCost('image', options),
responseText: mcpResult.metadata?.responseText || '',
translation: translationInfo
},
media: {
type: 'image',
data: imageData,
metadata: {
format: 'png',
dimensions: `${options.width || 1024}x${options.height || 1024}`
}
}
};
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
throw new Error(`Image generation failed: ${errorMessage}`);
}
}
async generateAudio(text, options = {}) {
logger.info('Generating audio using MCP command (unified timeout pattern)', {
textLength: text.length,
voice: options.voice || 'Kore',
format: 'wav',
model: AI_MODELS.AUDIO_GENERATION
});
const startTime = Date.now();
try {
const mcpResult = await this.executeMCPCommand('generate_audio', {
text,
voice: options.voice || 'Kore',
model: AI_MODELS.AUDIO_GENERATION
});
const duration = Date.now() - startTime;
const outputPath = mcpResult.file?.path || '';
const fileSize = mcpResult.file?.size || 0;
const audioData = mcpResult.audioData || null;
return {
success: true,
generationType: 'audio',
outputPath,
originalPrompt: text,
metadata: {
duration,
fileSize,
format: 'wav',
model: AI_MODELS.AUDIO_GENERATION,
settings: options,
cost: this.calculateGenerationCost('audio', options),
voice: options.voice || 'Kore'
},
media: {
type: 'audio',
data: audioData,
metadata: {
format: 'wav',
voice: options.voice || 'Kore'
}
}
};
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
throw new Error(`Audio generation failed: ${errorMessage}`);
}
}
async generateAudioWithScript(prompt, options = {}) {
logger.info('Generating audio with script generation', {
prompt: prompt.substring(0, 100),
hasMultipleSpeakers: !!options.speakers
});
try {
const scriptPrompt = options.scriptPrompt ||
`Generate a script for the following request: ${prompt}. ` +
(options.speakers ?
`Include dialogue for speakers: ${options.speakers.map((s) => s.name).join(', ')}.` :
'Write it as a single narrator script.');
const scriptResult = await this.executeMCPCommandOptimized('generate_text', {
prompt: scriptPrompt,
model: 'gemini-2.0-flash',
maxOutputTokens: 1000
});
const script = scriptResult.text || scriptResult.content?.[0]?.text || 'No script generated';
logger.info('Script generated successfully', { scriptLength: script.length });
const audioOptions = {
...options,
script
};
return await this.generateAudio(script, audioOptions);
}
catch (error) {
logger.error('Failed to generate audio with script', error);
throw error;
}
}
async analyzeAudioAdvanced(audioPath) {
return retry(async () => {
logger.debug('Performing advanced audio analysis', { audioPath });
const result = await this.executeMCPCommand('analyze_audio_advanced', {
audio: audioPath,
options: {
include_transcription: true,
include_sentiment: true,
include_emotions: true,
include_speaker_detection: true,
include_metadata: true,
},
});
return {
transcription: result.transcription || '',
language: result.language,
confidence: result.confidence,
sentiment: result.sentiment,
emotions: result.emotions || [],
speakers: result.speakers || [],
metadata: {
duration: result.metadata?.duration || 0,
sampleRate: result.metadata?.sampleRate,
channels: result.metadata?.channels,
format: result.metadata?.format || 'unknown',
},
};
}, {
maxAttempts: this.MAX_RETRIES,
delay: 5000,
operationName: 'analyze-audio-advanced',
});
}
async downloadGeneratedMedia(downloadUrl, mediaType, quality) {
if (!downloadUrl) {
throw new Error('No download URL provided for generated media');
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const extension = this.getFileExtension(mediaType);
const fileName = `generated-${mediaType}-${timestamp}.${extension}`;
const outputDir = join(process.cwd(), 'output', mediaType);
const outputPath = join(outputDir, fileName);
try {
await mkdir(outputDir, { recursive: true });
logger.debug('Downloading generated media', {
url: downloadUrl.substring(0, 100),
outputPath,
mediaType
});
const response = await fetch(downloadUrl);
if (!response.ok) {
throw new Error(`Failed to download media: ${response.statusText}`);
}
const fileStream = createWriteStream(outputPath);
const reader = response.body?.getReader();
if (!reader) {
throw new Error('Failed to get response stream');
}
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
fileStream.write(Buffer.from(value));
}
fileStream.end();
logger.info('Media downloaded successfully', {
outputPath,
mediaType,
fileSize: response.headers.get('content-length')
});
return outputPath;
}
catch (error) {
logger.error('Failed to download generated media', {
error: error.message,
downloadUrl: downloadUrl.substring(0, 100),
mediaType
});
throw error;
}
}
getFileExtension(mediaType) {
switch (mediaType) {
case 'image': return 'png';
case 'video': return 'mp4';
case 'audio': return 'mp3';
default: return 'bin';
}
}
calculateGenerationCost(mediaType, options) {
const baseCosts = {
image: 0.05,
video: 0.25,
audio: 0.02,
};
let cost = baseCosts[mediaType];
if (options.quality === 'high') {
cost *= 2;
}
if (options.quality === 'ultra') {
cost *= 4;
}
if (mediaType === 'video' && options.duration) {
cost *= Math.max(1, options.duration / 5);
}
return Math.round(cost * 1000) / 1000;
}
getCapabilities() {
return [
'multimodal_processing',
'document_analysis',
'image_analysis',
'image_generation',
'video_generation',
'audio_generation',
'audio_transcription',
'audio_analysis_advanced',
'pdf_conversion',
'file_processing',
'batch_processing',
'content_extraction',
'media_download',
];
}
getCost(task) {
const basePrice = 0.001;
if (task.files && task.files.length > 0) {
return basePrice * task.files.length;
}
return basePrice;
}
estimateCost(input, result) {
const basePrice = 0.001;
if (input.files && input.files.length > 0) {
return basePrice * input.files.length;
}
return basePrice;
}
getEstimatedDuration(task) {
if (task.action === 'generate_image' ||
task.action === 'image_generation' ||
task.type === 'image_generation' ||
(task.prompt && this.isImageGenerationRequest(task.prompt))) {
return 120000;
}
if (task.action === 'generate_video' ||
task.action === 'video_generation' ||
task.type === 'video_generation' ||
(task.prompt && this.isVideoGenerationRequest(task.prompt))) {
return 180000;
}
if (task.action === 'generate_audio' ||
task.action === 'audio_generation' ||
task.type === 'audio_generation' ||
(task.prompt && this.isAudioGenerationRequest(task.prompt))) {
return 90000;
}
const baseTime = 15000;
if (task.files && task.files.length > 0) {
return baseTime + (task.files.length * 30000);
}
if (task.type === 'multimodal' || task.action === 'multimodal_processing') {
return baseTime * 4;
}
return baseTime;
}
async analyzeDocuments(files, instructions) {
logger.info('🔧 analyzeDocuments method called - with timeout fix', {
fileCount: files.length,
instructionsLength: instructions.length,
timestamp: Date.now(),
version: 'v2025-07-03-document-fix'
});
const result = await this.executeMCPCommand('analyze_documents', {
documents: files.map(f => f.path),
instructions,
options: {
extract_structure: true,
summarize: true,
},
});
return result.analysis || result.content || 'Document analysis completed';
}
async convertFiles(files, outputFormat) {
logger.debug('Converting files', {
fileCount: files.length,
outputFormat,
});
const results = [];
for (const file of files) {
const result = await this.executeMCPCommand('convert_file', {
input: file.path,
output_format: outputFormat,
options: {
preserve_quality: true,
},
});
results.push({
original: file.path,
converted: result.output_path || result.content,
format: outputFormat,
});
}
return results;
}
canUseDirectAPI(params) {
return false;
}
async executeDirectAPI(command, params) {
logger.error(`[${this.instanceId}] GHOST LOG DETECTED: Direct API call attempted but should be disabled!`, {
instanceId: this.instanceId,
command,
params,
stack: new Error().stack
});
console.trace(`[${this.instanceId}] GHOST LOG TRACE: Direct API call attempted:`, command);
throw new Error(`Direct API integration disabled. All operations must use MCP server for architectural consistency. Command: ${command}`);
}
calculateOptimizedTimeout(command, params) {
let timeout = 60000;
if (command === 'generate_image') {
timeout = 180000;
}
else if (command === 'generate_audio') {
timeout = 120000;
}
else if (command === 'generate_video') {
timeout = 300000;
}
else if (command === 'multimodal_process' || command === 'analyze_documents') {
timeout = 300000;
}
else if (params?.files && params.files.length > 0) {
timeout = 120000 + (params.files.length * 20000);
}
return timeout;
}
async processGeneral(task) {
const prompt = task.prompt || task.instructions || task.request || 'Please process this content.';
if (this.isImageGenerationRequest(prompt)) {
logger.info('Detected image generation request in general processing', {
prompt: prompt.substring(0, 100)
});
return await this.generateImage(prompt, task.options || {});
}
if (this.isVideoGenerationRequest(prompt)) {
logger.info('Detected video generation request in general processing', {
prompt: prompt.substring(0, 100)
});
throw new Error('Video generation is not yet implemented');
}
if (this.isAudioGenerationRequest(prompt)) {
logger.info('Detected audio generation request in general processing', {
prompt: prompt.substring(0, 100)
});
return await this.generateAudio(prompt, task.options || {});
}
if (task.options?.multiplePDFs && task.files && task.files.length > 1) {
const pdfFiles = task.files.filter((file) => file.path.toLowerCase().endsWith('.pdf'));
if (pdfFiles.length > 1) {
logger.info('Processing multiple PDFs with Gemini File API', {
pdfCount: pdfFiles.length,
totalFiles: task.files.length
});
return await this.processMultiplePDFs(pdfFiles, prompt);
}
}
return await this.executeMCPCommandOptimized('multimodal_process', {
files: task.files || [],
instructions: prompt,
model: 'gemini-2.5-flash',
});
}
resolveMCPServerPath() {
const serverFileName = 'ai-studio-mcp-server.js';
const checkedPaths = [];
const fromModulePath = join(__dirname, '..', 'mcp-servers', serverFileName);
checkedPaths.push(fromModulePath);
if (fs.existsSync(fromModulePath)) {
logger.debug('Using ESM module relative path', { path: fromModulePath });
return fromModulePath;
}
const devPath = join(process.cwd(), 'dist', 'mcp-servers', serverFileName);
checkedPaths.push(devPath);
if (fs.existsSync(devPath)) {
logger.debug('Using development MCP server path', { path: devPath });
return devPath;
}
const localNpmPath = join(process.cwd(), 'node_modules', 'claude-gemini-multimodal-bridge', 'dist', 'mcp-servers', serverFileName);
checkedPaths.push(localNpmPath);
if (fs.existsSync(localNpmPath)) {
logger.debug('Using local npm install MCP server path', { path: localNpmPath });
return localNpmPath;
}
const isWindows = process.platform === 'win32';
const globalPaths = isWindows ? [
...(process.env.APPDATA ? [
join(process.env.APPDATA, 'npm', 'node_modules', 'claude-gemini-multimodal-bridge', 'dist', 'mcp-servers', serverFileName)
] : []),
...(process.env.LOCALAPPDATA ? [
join(process.env.LOCALAPPDATA, 'npm', 'node_modules', 'claude-gemini-multimodal-bridge', 'dist', 'mcp-servers', serverFileName)
] : []),
...(process.env.USERPROFILE ? [
join(process.env.USERPROFILE, 'AppData', 'Roaming', 'npm', 'node_modules', 'claude-gemini-multimodal-bridge', 'dist', 'mcp-servers', serverFileName),
join(process.env.USERPROFILE, '.nvm', 'versions', 'node', process.version, 'node_modules', 'claude-gemini-multimodal-bridge', 'dist', 'mcp-servers', serverFileName),
] : []),
join('C:', 'Program Files', 'nodejs', 'node_modules', 'claude-gemini-multimodal-bridge', 'dist', 'mcp-servers', serverFileName),
] : [
...(process.env.HOME ? [
join(process.env.HOME, '.nvm', 'versions', 'node', process.version, 'lib', 'node_modules', 'claude-gemini-multimodal-bridge', 'dist', 'mcp-servers', serverFileName)
] : []),
join('/usr/local/lib/node_modules/claude-gemini-multimodal-bridge/dist/mcp-servers', serverFileName),
join('/opt/homebrew/lib/node_modules/claude-gemini-multimodal-bridge/dist/mcp-servers', serverFileName),
...(process.env.USER ? [
join('/mnt/c/Users', process.env.USER, 'AppData', 'Roaming', 'npm', 'node_modules', 'claude-gemini-multimodal-bridge', 'dist', 'mcp-servers', serverFileName)
] : []),
];
for (const globalPath of globalPaths) {
checkedPaths.push(globalPath);
if (fs.existsSync(globalPath)) {
logger.debug('Using global npm path from search', { path: globalPath });
return globalPath;
}
}
const errorMessage = `MCP server not found. Checked paths:\n${checkedPaths.map(p => ` - ${p}`).join('\n')}\n\nTroubleshooting:\n 1. Run 'npm run build' to compile the MCP server\n 2. Ensure CGMB is properly installed\n 3. Check that dist/mcp-servers/ai-studio-mcp-server.js exists`;
logger.error('MCP server path resolution failed', {
checkedPaths,
cwd: process.cwd(),
__dirname
});
throw new Error(errorMessage);
}
async getPersistentMCPProcess() {
const now = Date.now();
if (!this.persistentMCPProcess ||
(now - this.mcpProcessStartTime > this.MCP_PROCESS_TTL) ||
this.persistentMCPProcess.killed) {
if (this.persistentMCPProcess && !this.persistentMCPProcess.killed) {
try {
if (isPlatformWindows()) {
try {
execSync(`taskkill /pid ${this.persistentMCPProcess.pid} /T /F`, { stdio: 'ignore' });
}
catch {
this.persistentMCPProcess.kill();
}
}
else {
this.persistentMCPProcess.kill('SIGTERM');
}
}
catch (error) {
logger.debug('Error killing old MCP process', { error: error.message });
}
}
logger.debug('Starting persistent AI Studio MCP process...');
const mcpServerPath = this.resolveMCPServerPath();
if (!fs.existsSync(mcpServerPath)) {
logger.error('MCP server not found at resolved path for persistent process', {
mcpServerPath,
cwd: process.cwd(),
__dirname
});
throw new Error(`AI Studio MCP server not found at: ${mcpServerPath}`);
}
const isWindowsSpawn = process.platform === 'win32';
this.persistentMCPProcess = spawn('node', [mcpServerPath], {
stdio: 'pipe',
cwd: process.cwd(),
shell: isWindowsSpawn,
env: {
...process.env,
AI_STUDIO_API_KEY: this.getAIStudioApiKey(),
GEMINI_API_KEY: this.getAIStudioApiKey(),
GOOGLE_AI_STUDIO_API_KEY: this.getAIStudioApiKey(),
},
});
this.mcpProcessStartTime = now;
this.persistentMCPProcess.on('error', (error) => {
logger.warn('Persistent MCP process error', { error: error.message });
this.persistentMCPProcess = undefined;
});
this.persistentMCPProcess.on('exit', (code) => {
logger.debug('Persistent MCP process exited', { code });
this.persistentMCPProcess = undefined;
});
}
return this.persistentMCPProcess;
}
async executeMCPCommandOptimized(command, params) {
if (command === 'multimodal_process' && this.canUseDirectAPI(params)) {
return await this.executeDirectAPI(command, params);
}
const process = await this.getPersistentMCPProcess();
return new Promise((resolve, reject) => {
const timeout = this.calculateOptimizedTimeout(command, params);
logger.debug(`[${this.instanceId}] Executing optimized MCP command`, {
instanceId: this.instanceId,
command,
hasParams: !!params,
timeout,
timeoutMinutes: Math.round(timeout / 60000),
usesPersistentProcess: true,
paramKeys: params ? Object.keys(params) : []
});
let output = '';
let errorOutput = '';
const mcpRequest = {
jsonrpc: '2.0',
id: Date.now(),
method: 'tools/call',
params: {
name: command,
arguments: params
}
};
let isResolved = false;
let timeoutId;
const cleanup = () => {
if (!isResolved) {
isResolved = true;
if (timeoutId) {
clearTimeout(timeoutId);
}
try {
process.stdout.removeListener('data', dataHandler);
process.stderr.removeListener('data', errorHandler);
}
catch (error) {
}
}
};
logger.debug(`[${this.instanceId}] Sending optimized MCP request`, {
instanceId: this.instanceId,
command,
requestId: mcpRequest.id,
requestLength: JSON.stringify(mcpRequest).length
});
try {
process.stdin.write(JSON.stringify(mcpRequest) + '\n');
}
catch (error) {
cleanup();
reject(new Error(`Failed to send MCP request: ${error.message}`));
return;
}
const dataHandler = (data) => {
const chunk = data.toString();
output += chunk;
logger.debug(`[${this.instanceId}] Optimized MCP stdout chunk received`, {
instanceId: this.instanceId,
command,
chunkLength: chunk.length,
totalOutputLength: output.length
});
const lines = output.split(/\r?\n/);
for (const line of lines) {
if (line.trim()) {
try {
const mcpResponse = JSON.parse(line);
if (mcpResponse.result || mcpResponse.error) {
if (!isResolved) {
cleanup();
logger.debug(`[${this.instanceId}] MCP response received - immediate resolution`, {
instanceId: this.instanceId,
command,
hasResult: !!mcpResponse.result,
hasError: !!mcpResponse.error,
responseId: mcpResponse.id
});
if (mcpResponse.error) {
reject(new Error(`MCP Error: ${mcpResponse.error.message || 'Unknown error'}`));
}
else {
resolve(mcpResponse.result);
}
}
return;
}
}
catch {
continue;
}
}
}
};
const errorHandler = (data) => {
const chunk = data.toString();
errorOutput += chunk;
logger.debug(`[${this.instanceId}] Optimized MCP stderr chunk received`, {
instanceId: this.instanceId,
command,
chunkLength: chunk.length,
errorContent: chunk.substring(0, 200)
});
};
timeoutId = setTimeout(() => {
if (!isResolved) {
cleanup();
reject(new Error(`AI Studio MCP command timeout after ${timeout}ms - operation may have completed successfully`));
}
}, timeout);
process.stdout.on('data', dataHandler);
process.stderr.on('data', errorHandler);
});
}
async executeMCPCommand(command, params) {
if (params) {
if (params.files && Array.isArray(params.files)) {
params = {
...params,
files: params.files.map((file) => ({
...file,
path: file.path ? this.normalizeInputPath(file.path) : file.path
}))
};
}
if (params.documents && Array.isArray(params.documents)) {
params = {
...params,
documents: params.documents.map((doc) => this.normalizeInputPath(doc))
};
}
}
let timeout = this.DEFAULT_TIMEOUT;
if (command === 'generate_image' || this.isImageGenerationCommand(command, params)) {
timeout = Math.max(120000, this.DEFAULT_TIMEOUT * 0.8);
logger.debug('Optimized timeout for image generation', {
command,
timeoutMs: timeout,
timeoutMinutes: Math.round(timeout / 60000),
reason: 'Immediate timeout clear on success reduces actual wait time'
});
}
else if (command === 'generate_video' || this.isVideoGenerationCommand(command, param