claude-gemini-multimodal-bridge
Version:
Enterprise-grade AI integration bridge connecting Claude Code, Gemini CLI, and Google AI Studio with intelligent routing and advanced multimodal processing capabilities
721 lines (719 loc) • 31.7 kB
JavaScript
import { LayerManager } from '../core/LayerManager.js';
import { ClaudeCodeLayer } from '../layers/ClaudeCodeLayer.js';
import { GeminiCLILayer } from '../layers/GeminiCLILayer.js';
import { AIStudioLayer } from '../layers/AIStudioLayer.js';
import { logger } from '../utils/logger.js';
import { retry, safeExecute } from '../utils/errorHandler.js';
import { AuthVerifier } from '../auth/AuthVerifier.js';
import path from 'path';
import fs from 'fs/promises';
export class DocumentAnalysis {
layerManager;
claudeLayer;
geminiLayer;
aiStudioLayer;
authVerifier;
MAX_DOCUMENTS = 20;
MAX_DOCUMENT_SIZE = 50 * 1024 * 1024;
SUPPORTED_DOCUMENT_TYPES = [
'.pdf', '.txt', '.md', '.doc', '.docx', '.rtf', '.odt',
'.ppt', '.pptx', '.odp', '.xls', '.xlsx', '.ods', '.csv'
];
constructor() {
const defaultConfig = {
gemini: { api_key: '', model: 'gemini-2.5-pro', timeout: 60000, max_tokens: 16384, temperature: 0.2 },
claude: { code_path: 'claude', timeout: 300000 },
aistudio: { enabled: true, max_files: 10, max_file_size: 100 },
cache: { enabled: true, ttl: 3600 },
logging: { level: 'info' },
};
this.layerManager = new LayerManager(defaultConfig);
this.claudeLayer = new ClaudeCodeLayer();
this.geminiLayer = new GeminiCLILayer();
this.aiStudioLayer = new AIStudioLayer();
this.authVerifier = new AuthVerifier();
}
async analyzeDocuments(args) {
return safeExecute(async () => {
const startTime = Date.now();
logger.info('Starting document analysis', {
documentCount: args.documents.length,
analysisType: args.analysis_type,
requiresComparison: args.analysis_type === 'comparison',
});
await this.validateAnalysisArgs(args);
await this.initializeLayers(args);
const documentRefs = this.convertPathsToFileRefs(args.documents);
const processedDocs = await this.prepareDocuments(documentRefs);
const analysisResult = await this.executeAnalysis(args, processedDocs);
const summary = args.documents.length > 1
? await this.generateSummary(analysisResult, args)
: null;
const totalDuration = Date.now() - startTime;
return {
success: true,
analysis_type: args.analysis_type,
content: typeof analysisResult === 'string' ? analysisResult : JSON.stringify(analysisResult),
documents_processed: args.documents,
processing_time: totalDuration,
insights: await this.generateInsights(analysisResult, args),
metadata: {
total_duration: totalDuration,
tokens_used: this.estimateTokensUsed(analysisResult),
cost: 0,
quality_score: 0.8,
},
};
}, {
operationName: 'document-analysis',
layer: 'tool',
timeout: 900000,
});
}
async analyzeSingleDocument(documentPath, analysisType = 'comprehensive', options) {
return this.analyzeDocuments({
documents: [documentPath],
analysis_type: 'summary',
options: {
depth: options?.depth || 'medium',
extractMetadata: options?.extractImages || false,
},
});
}
async compareDocuments(documentPaths, comparisonType = 'comprehensive') {
if (documentPaths.length < 2) {
throw new Error('At least 2 documents required for comparison');
}
return this.analyzeDocuments({
documents: documentPaths,
analysis_type: 'comparison',
options: {
depth: 'deep',
detailed: true,
},
});
}
async extractStructuredData(documentPaths, dataTypes = ['tables', 'lists', 'key-value-pairs', 'entities']) {
return this.analyzeDocuments({
documents: documentPaths,
analysis_type: 'extraction',
options: {
depth: 'deep',
structured: true,
extractionType: dataTypes.join(','),
},
});
}
async summarizeDocuments(documentPaths, summaryLength = 'detailed') {
return this.analyzeDocuments({
documents: documentPaths,
analysis_type: 'summary',
options: {
depth: summaryLength === 'comprehensive' ? 'deep' : 'medium',
detailed: summaryLength !== 'brief',
},
});
}
async extractPDFText(pdfPath) {
try {
const geminiResult = await this.extractPDFWithGeminiFileAPI(pdfPath);
if (geminiResult && geminiResult.length > 0) {
logger.info('PDF extraction completed using Gemini File API', {
pdfPath,
textLength: geminiResult.length,
textPreview: geminiResult.substring(0, 100) + (geminiResult.length > 100 ? '...' : '')
});
return geminiResult;
}
}
catch (geminiError) {
logger.warn('Gemini File API extraction failed, falling back to pdf-parse', {
pdfPath,
error: geminiError instanceof Error ? geminiError.message : String(geminiError)
});
}
return this.extractPDFWithPdfParse(pdfPath);
}
async extractPDFWithGeminiFileAPI(pdfPath) {
try {
logger.info('Extracting PDF with Gemini File API', { pdfPath });
const stats = await fs.stat(pdfPath);
if (stats.size > 50 * 1024 * 1024) {
throw new Error(`PDF file too large for Gemini File API: ${Math.round(stats.size / 1024 / 1024)}MB (max 50MB)`);
}
await this.aiStudioLayer.initialize();
const result = await this.aiStudioLayer.execute({
action: 'multimodal_processing',
files: [{
path: pdfPath,
type: 'document',
encoding: 'binary'
}],
instructions: `Extract all text content from this PDF document using native vision processing.
Follow these guidelines:
1. Process all pages (up to 1,000 pages supported)
2. Each page equals approximately 258 tokens
3. Preserve document structure and formatting
4. Extract both text and analyze any images within the PDF
5. Provide comprehensive content understanding
6. Focus on accuracy and completeness
Please extract the complete text content while maintaining readability and structure.`
});
if (result.success && result.data) {
const extractedText = typeof result.data === 'string' ? result.data :
(result.data.content && typeof result.data.content === 'string') ? result.data.content :
JSON.stringify(result.data);
const metadata = `[PDF Document: ${path.basename(pdfPath)} - Processed with Gemini File API]\n\n`;
return metadata + extractedText.trim();
}
else {
throw new Error(result.error || 'Failed to extract PDF with Gemini File API');
}
}
catch (error) {
logger.error('Gemini File API PDF extraction failed', {
pdfPath,
error: error instanceof Error ? error.message : String(error)
});
throw error;
}
}
async extractPDFWithPdfParse(pdfPath) {
try {
logger.info('Extracting PDF with pdf-parse (fallback)', { pdfPath });
let pdfParse;
try {
pdfParse = (await import('pdf-parse/lib/pdf-parse.js')).default;
}
catch (importError) {
throw new Error(`PDF processing library not available: ${importError instanceof Error ? importError.message : String(importError)}`);
}
const buffer = await fs.readFile(pdfPath);
const data = await pdfParse(buffer, {
max: 0,
version: 'v1.10.100',
pagerender: (pageData) => {
const renderOptions = {
normalizeWhitespace: false,
disableCombineTextItems: false,
includeMarkedContent: true,
textFilter: undefined
};
return pageData.getTextContent(renderOptions)
.then((textContent) => {
let lastY, text = '';
let lineSpacing = 0;
for (const item of textContent.items) {
if (lastY && Math.abs(lastY - item.transform[5]) > lineSpacing) {
text += '\n' + item.str;
}
else {
const needsSpace = lastY && lastY === item.transform[5] &&
!/[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FAF]/.test(item.str) &&
!/[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FAF]/.test(text.slice(-1));
text += (needsSpace ? ' ' : '') + item.str;
}
lastY = item.transform[5];
if (item.height) {
lineSpacing = Math.max(lineSpacing, item.height * 0.3);
}
}
return text;
});
}
});
const extractedText = data.text.trim();
logger.debug('PDF parsing details', {
pdfPath,
pageCount: data.numpages,
hasMetadata: !!data.metadata,
metadataKeys: data.metadata ? Object.keys(data.metadata) : [],
textLength: data.text.length,
extractedSample: data.text.substring(0, 200),
hasInfo: !!data.info,
infoKeys: data.info ? Object.keys(data.info) : [],
version: data.version || 'unknown'
});
logger.info('PDF text extraction completed', {
pdfPath,
textLength: extractedText.length,
pageCount: data.numpages,
hasText: extractedText.length > 0,
textPreview: extractedText.substring(0, 100) + (extractedText.length > 100 ? '...' : '')
});
const isTextExtractionPoor = extractedText.length < 100 ||
(extractedText.length < data.numpages * 50);
if (extractedText.length === 0 || isTextExtractionPoor) {
logger.warn('Poor text extraction from PDF, attempting AI Studio OCR fallback', {
pdfPath,
textLength: extractedText.length,
expectedMinLength: data.numpages * 50
});
try {
await this.aiStudioLayer.initialize();
const ocrResult = await this.aiStudioLayer.execute({
action: 'multimodal_processing',
files: [{
path: pdfPath,
type: 'document',
encoding: 'binary'
}],
instructions: 'Extract all text content from this PDF document. Preserve the structure and formatting as much as possible. Focus on extracting the actual readable text content.',
});
if (ocrResult.success && ocrResult.data) {
const ocrText = typeof ocrResult.data === 'string' ? ocrResult.data :
(ocrResult.data.content && typeof ocrResult.data.content === 'string') ? ocrResult.data.content :
JSON.stringify(ocrResult.data);
logger.info('AI Studio OCR fallback successful', {
pdfPath,
originalTextLength: extractedText.length,
ocrTextLength: ocrText.length,
improvement: ocrText.length > extractedText.length
});
if (ocrText.length > extractedText.length * 2) {
const metadata = `[PDF Document: ${path.basename(pdfPath)} - ${data.numpages} pages - Processed with AI OCR]\n\n`;
return metadata + ocrText.trim();
}
}
}
catch (ocrError) {
logger.warn('AI Studio OCR fallback failed', {
pdfPath,
error: ocrError instanceof Error ? ocrError.message : String(ocrError)
});
}
if (extractedText.length === 0) {
return `[PDF Document: ${path.basename(pdfPath)}]\nNo text content could be extracted. This may be an image-based PDF or contain special formatting that requires OCR processing.`;
}
}
const metadata = `[PDF Document: ${path.basename(pdfPath)} - ${data.numpages} pages]\n\n`;
return metadata + extractedText;
}
catch (error) {
logger.error('Failed to extract PDF text', {
pdfPath,
error: error instanceof Error ? error.message : String(error)
});
return `[PDF Document: ${path.basename(pdfPath)}]\nText extraction failed: ${error instanceof Error ? error.message : String(error)}\nNote: This PDF may require specialized OCR processing or multimodal AI analysis.`;
}
}
async preprocessDocument(documentPath) {
const ext = path.extname(documentPath).toLowerCase();
if (ext === '.pdf') {
const extractedText = await this.extractPDFText(documentPath);
return {
path: documentPath,
content: extractedText
};
}
return { path: documentPath };
}
async validateAnalysisArgs(args) {
if (!args.documents || args.documents.length === 0) {
throw new Error('At least one document must be provided');
}
if (args.documents.length > this.MAX_DOCUMENTS) {
throw new Error(`Too many documents. Maximum ${this.MAX_DOCUMENTS} allowed`);
}
for (const doc of args.documents) {
try {
await fs.access(doc);
}
catch {
throw new Error(`Document not found: ${doc}`);
}
const stats = await fs.stat(doc);
if (stats.size > this.MAX_DOCUMENT_SIZE) {
throw new Error(`Document too large: ${doc} (max ${this.MAX_DOCUMENT_SIZE / 1024 / 1024}MB)`);
}
const ext = path.extname(doc).toLowerCase();
if (!this.SUPPORTED_DOCUMENT_TYPES.includes(ext)) {
logger.warn('Potentially unsupported document type', { path: doc, extension: ext });
}
}
}
async initializeLayers(args) {
const requiredLayers = this.determineLayersUsed(args);
const initPromises = [];
if (requiredLayers.includes('claude')) {
initPromises.push(this.claudeLayer.initialize());
}
if (requiredLayers.includes('gemini')) {
initPromises.push(this.geminiLayer.initialize());
}
if (requiredLayers.includes('aistudio')) {
initPromises.push(this.aiStudioLayer.initialize());
}
await Promise.all(initPromises);
logger.debug('Initialized layers for document analysis', { layers: requiredLayers });
}
determineLayersUsed(args) {
const layers = new Set();
layers.add('claude');
const hasPDFs = args.documents.some(doc => path.extname(doc).toLowerCase() === '.pdf');
if (hasPDFs || (args.options?.extractMetadata) || (args.options?.structured)) {
layers.add('aistudio');
}
if ((args.options?.requiresGrounding)) {
layers.add('gemini');
}
return Array.from(layers);
}
convertPathsToFileRefs(documentPaths) {
return documentPaths.map(path => ({
path,
type: 'document',
encoding: 'utf-8',
}));
}
async prepareDocuments(documents) {
return retry(async () => {
const processedDocs = [];
for (const doc of documents) {
const stats = await fs.stat(doc.path);
const fileType = this.determineDocumentType(doc.path);
const preprocessed = await this.preprocessDocument(doc.path);
const processedDoc = {
...doc,
size: stats.size,
type: 'document',
encoding: doc.encoding || 'utf-8',
content: preprocessed.content,
};
processedDocs.push(processedDoc);
logger.debug('Document prepared for analysis', {
path: processedDoc.path,
type: processedDoc.type,
size: processedDoc.size,
hasExtractedContent: !!preprocessed.content,
contentLength: preprocessed.content?.length || 0
});
}
return processedDocs;
}, {
maxAttempts: 2,
delay: 1000,
operationName: 'prepare-documents',
});
}
determineDocumentType(filePath) {
const ext = path.extname(filePath).toLowerCase();
const typeMap = {
'.pdf': 'pdf',
'.doc': 'word',
'.docx': 'word',
'.txt': 'text',
'.md': 'markdown',
'.ppt': 'presentation',
'.pptx': 'presentation',
'.xls': 'spreadsheet',
'.xlsx': 'spreadsheet',
'.csv': 'csv',
};
return typeMap[ext] || 'document';
}
async executeAnalysis(args, documents) {
return retry(async () => {
logger.info('Executing document analysis workflow', {
analysisType: args.analysis_type,
documentCount: documents.length,
});
switch (args.analysis_type) {
case 'summary':
return await this.executeSummarizationAnalysis(documents, args);
case 'extraction':
return await this.executeExtractionAnalysis(documents, args);
case 'comparison':
return await this.executeComparativeAnalysis(documents, args);
case 'translation':
return await this.executeTranslationAnalysis(documents, args);
default:
return await this.executeGeneralAnalysis(documents, args);
}
}, {
maxAttempts: 3,
delay: 3000,
operationName: 'execute-analysis',
});
}
async executeComprehensiveAnalysis(documents, args) {
const results = [];
for (const doc of documents) {
const extractionResult = await this.aiStudioLayer.execute({
action: 'document_analysis',
files: [doc],
instructions: 'Extract all content, structure, and metadata from this document. Provide detailed analysis.',
});
results.push({
document: doc.path,
extraction: extractionResult.data,
});
}
const reasoningTask = {
prompt: this.buildComprehensiveAnalysisPrompt(documents, results),
depth: args.options?.depth ?? 'medium',
domain: 'document_analysis',
context: `Analyzing ${documents.length} document(s) for comprehensive understanding`,
};
const reasoningResult = await this.claudeLayer.executeComplexReasoning(reasoningTask);
return {
documentExtractions: results,
analysis: reasoningResult,
processingType: 'comprehensive',
};
}
async executeSummarizationAnalysis(documents, args) {
const summaries = [];
for (const doc of documents) {
const summaryResult = await this.aiStudioLayer.execute({
action: 'document_analysis',
files: [doc],
instructions: `Summarize this document with ${args.options?.detailed ? 'detailed' : 'standard'} level of detail. Focus on key points, main arguments, and important information.`,
});
summaries.push({
document: doc.path,
summary: summaryResult.data,
});
}
if (documents.length > 1) {
const synthesisResult = await this.claudeLayer.synthesizeResponse({
request: 'Create a comprehensive synthesis of all document summaries',
inputs: {
summaries: summaries.map(s => s.summary).join('\n\n'),
documentCount: documents.length,
},
});
return {
individualSummaries: summaries,
overallSynthesis: synthesisResult,
processingType: 'summarization',
};
}
return {
individualSummaries: summaries,
processingType: 'summarization',
};
}
async executeExtractionAnalysis(documents, args) {
const extractions = [];
for (const doc of documents) {
const extractionResult = await this.aiStudioLayer.execute({
action: 'document_analysis',
files: [doc],
instructions: `Extract structured data from this document: ${args.options?.extractionType || 'tables,lists,entities'}. Provide organized, structured output.`,
});
extractions.push({
document: doc.path,
extractedData: extractionResult.data,
});
}
const organizationResult = await this.claudeLayer.execute({
action: 'synthesize_response',
request: 'Organize and structure the extracted data into a comprehensive format',
inputs: {
extractions: extractions,
dataTypes: args.options?.extractionType?.split(',') || [],
},
});
return {
rawExtractions: extractions,
organizedData: organizationResult.data,
processingType: 'extraction',
};
}
async executeComparativeAnalysis(documents, args) {
if (documents.length < 2) {
throw new Error('Comparative analysis requires at least 2 documents');
}
const documentContents = [];
for (const doc of documents) {
const extractionResult = await this.aiStudioLayer.execute({
action: 'document_analysis',
files: [doc],
instructions: 'Extract all textual content and key information for comparison purposes.',
});
documentContents.push({
document: doc.path,
content: extractionResult.data,
});
}
const comparisonResult = await this.claudeLayer.execute({
action: 'complex_reasoning',
prompt: this.buildComparisonPrompt(documentContents, 'comprehensive'),
depth: 'deep',
});
return {
documentContents,
comparison: comparisonResult.data,
processingType: 'comparative',
};
}
async executeContextualAnalysis(documents, args) {
const documentData = [];
for (const doc of documents) {
const extractionResult = await this.aiStudioLayer.execute({
action: 'document_analysis',
files: [doc],
instructions: 'Extract content for contextual analysis. Focus on topics, themes, and key information.',
});
documentData.push({
document: doc.path,
content: extractionResult.data,
});
}
const contextualResult = await this.geminiLayer.execute({
action: 'contextual_analysis',
prompt: `Analyze these documents in current context and provide relevant background information: ${JSON.stringify(documentData)}`,
useSearch: true,
});
const synthesisResult = await this.claudeLayer.synthesizeResponse({
request: 'Combine document analysis with contextual information',
inputs: {
documentAnalysis: documentData,
contextualInformation: contextualResult.data,
},
});
return {
documentData,
contextualInformation: contextualResult.data,
synthesis: synthesisResult,
processingType: 'contextual',
};
}
async executeTranslationAnalysis(documents, args) {
const translations = [];
for (const doc of documents) {
const translationResult = await this.aiStudioLayer.execute({
action: 'document_analysis',
files: [doc],
instructions: `Translate this document. Focus on accurate translation while preserving meaning and context.`,
});
translations.push({
document: doc.path,
translation: translationResult.data,
});
}
return {
translations,
processingType: 'translation',
};
}
async executeGeneralAnalysis(documents, args) {
const results = [];
for (const doc of documents) {
const analysisResult = await this.aiStudioLayer.execute({
action: 'document_analysis',
files: [doc],
instructions: 'Perform general analysis of this document. Identify key information, themes, and structure.',
});
results.push({
document: doc.path,
analysis: analysisResult.data,
});
}
return {
analyses: results,
processingType: 'general',
};
}
async generateSummary(analysisResult, args) {
return retry(async () => {
const summaryPrompt = `Generate a comprehensive summary of the document analysis results. Include key findings, important insights, and main conclusions. Analysis type: ${args.analysis_type}`;
const summaryResult = await this.claudeLayer.synthesizeResponse({
request: summaryPrompt,
inputs: {
analysisResults: analysisResult,
documentCount: args.documents.length,
analysisType: args.analysis_type,
},
});
return summaryResult;
}, {
maxAttempts: 2,
delay: 1500,
operationName: 'generate-summary',
});
}
async generateInsights(analysisResult, args) {
try {
const insightsResult = await this.claudeLayer.execute({
action: 'complex_reasoning',
prompt: `Based on the document analysis results, identify key insights, patterns, and important observations. Provide them as a structured list.`,
context: JSON.stringify(analysisResult),
});
const insightsText = typeof insightsResult.data === 'string'
? insightsResult.data
: JSON.stringify(insightsResult.data);
const insights = insightsText
.split('\n')
.filter(line => {
const trimmed = line.trim();
return trimmed && (trimmed.startsWith('-') || trimmed.startsWith('•') || /^\d+\./.test(trimmed));
})
.map(line => line.trim().replace(/^[-•]\s*/, '').replace(/^\d+\.\s*/, ''))
.filter(insight => insight.length > 10);
return insights.length > 0 ? insights : ['Analysis completed successfully'];
}
catch (error) {
logger.warn('Failed to generate insights', { error: error.message });
return ['Analysis completed successfully'];
}
}
buildComprehensiveAnalysisPrompt(documents, extractions) {
let prompt = `Please perform a comprehensive analysis of ${documents.length} document(s). `;
prompt += 'The documents have been processed and their content extracted. ';
prompt += 'Please analyze the content for:\n';
prompt += '1. Main themes and topics\n';
prompt += '2. Key information and facts\n';
prompt += '3. Document structure and organization\n';
prompt += '4. Important insights and conclusions\n';
prompt += '5. Relationships between different parts of the content\n\n';
prompt += 'Extracted content:\n';
prompt += JSON.stringify(extractions, null, 2);
return prompt;
}
buildComparisonPrompt(documentContents, comparisonType) {
let prompt = `Please perform a ${comparisonType} comparison of the following documents:\n\n`;
documentContents.forEach((doc, index) => {
prompt += `Document ${index + 1} (${path.basename(doc.document)}):\n`;
prompt += `${JSON.stringify(doc.content)}\n\n`;
});
prompt += 'Please analyze and compare these documents focusing on:\n';
switch (comparisonType) {
case 'similarity':
prompt += '- Common themes and topics\n';
prompt += '- Shared information and concepts\n';
prompt += '- Similar conclusions or findings\n';
break;
case 'differences':
prompt += '- Contrasting viewpoints\n';
prompt += '- Different approaches or methodologies\n';
prompt += '- Unique information in each document\n';
break;
default:
prompt += '- Similarities and differences\n';
prompt += '- Complementary information\n';
prompt += '- Contradictions or conflicts\n';
prompt += '- Overall relationship between documents\n';
}
return prompt;
}
getSupportedDocumentTypes() {
return [...this.SUPPORTED_DOCUMENT_TYPES];
}
isDocumentSupported(filePath) {
const ext = path.extname(filePath).toLowerCase();
return this.SUPPORTED_DOCUMENT_TYPES.includes(ext);
}
getProcessingLimits() {
return {
maxDocuments: this.MAX_DOCUMENTS,
maxDocumentSize: this.MAX_DOCUMENT_SIZE,
maxDocumentSizeMB: this.MAX_DOCUMENT_SIZE / 1024 / 1024,
};
}
estimateTokensUsed(analysisResult) {
const resultText = typeof analysisResult === 'string' ? analysisResult : JSON.stringify(analysisResult);
return Math.ceil(resultText.length / 4);
}
}