dbx-mcp-server
Version:
A Model Context Protocol server for Dropbox integration with AI-powered PDF analysis, multi-directory indexing, and parallel processing
742 lines • 34 kB
JavaScript
import { aiParser } from './ai-parser.js';
import { documentDb } from './database.js';
import { schemaManager } from './schema-manager.js';
import { log, config } from './config.js';
import * as dbxApi from './dbx-api.js';
import { minimatch } from 'minimatch';
export async function indexFile(args) {
try {
log.info('Starting file indexing', { path: args.path });
// Get file metadata
const metadata = await dbxApi.getRawFileMetadata(args.path);
// Validate PDF
if (!metadata.name.toLowerCase().endsWith('.pdf')) {
return {
content: [{ type: "text", text: "Error: Only PDF files can be indexed" }],
isError: true,
};
}
// Check if already indexed and up-to-date
const existing = await documentDb.getDocumentByDropboxPath(args.path);
if (existing && existing.modified_time === metadata.client_modified) {
return {
content: [{
type: "text",
text: `Document "${metadata.name}" is already up-to-date in index`
}],
isError: false,
};
}
log.info('Starting AI analysis for indexing', {
fileName: metadata.name,
size: metadata.size
});
// Download and analyze
const startTime = Date.now();
const fileBuffer = await dbxApi.downloadFileAsBuffer(args.path);
const analysisResult = await aiParser.parseWithRetry(fileBuffer, metadata.name, 'application/pdf');
const processingTime = Date.now() - startTime;
log.info('AI analysis completed', {
fileName: metadata.name,
docType: analysisResult.doc_type,
confidence: analysisResult.confidence_score,
processingTimeMs: processingTime
});
// Store in database
const docId = await documentDb.insertDocument({
dropbox_path: args.path,
name: metadata.name,
mime_type: 'application/pdf',
size: metadata.size,
created_time: metadata.server_modified,
modified_time: metadata.client_modified,
ai_model: 'gemini-2.5-flash-preview-05-20',
ai_processing_time_ms: processingTime,
doc_type: analysisResult.doc_type,
synopsis: analysisResult.synopsis,
confidence_score: analysisResult.confidence_score
});
// Store type-specific data
await storeTypeSpecificData(docId, analysisResult);
log.info('Document indexed successfully', {
fileName: metadata.name,
docId,
docType: analysisResult.doc_type
});
return {
content: [{
type: "text",
text: `Successfully indexed "${metadata.name}" as ${analysisResult.doc_type} (confidence: ${(analysisResult.confidence_score * 100).toFixed(1)}%)`
}],
isError: false,
};
}
catch (error) {
log.error('File indexing failed', {
path: args.path,
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined
});
return {
content: [{
type: "text",
text: `Error indexing file: ${error instanceof Error ? error.message : String(error)}`
}],
isError: true,
};
}
}
// Optimized indexing function for high-concurrency parallel processing
async function indexFileOptimized(file) {
try {
const filePath = file.path_display;
// Get file metadata and check if already indexed in parallel
const [metadata, existing] = await Promise.all([
dbxApi.getRawFileMetadata(filePath),
documentDb.getDocumentByDropboxPath(filePath)
]);
// Validate PDF
if (!metadata.name.toLowerCase().endsWith('.pdf')) {
return {
isError: true,
skipped: false,
error: "Only PDF files can be indexed"
};
}
// Check if already indexed and up-to-date
if (existing && existing.modified_time === metadata.client_modified) {
return {
isError: false,
skipped: true
};
}
log.info('Starting parallel AI analysis', {
fileName: metadata.name,
size: metadata.size
});
// Download and analyze with enhanced parallel processing
const startTime = Date.now();
const [fileBuffer, _] = await Promise.all([
dbxApi.downloadFileAsBuffer(filePath),
// Pre-warm any caches or connections in parallel
Promise.resolve()
]);
// AI analysis happens asynchronously
const analysisResult = await aiParser.parseWithRetry(fileBuffer, metadata.name, 'application/pdf');
const processingTime = Date.now() - startTime;
log.info('AI analysis completed', {
fileName: metadata.name,
docType: analysisResult.doc_type,
confidence: analysisResult.confidence_score,
processingTimeMs: processingTime
});
// Prepare document data for batch insertion
const documentData = {
dropbox_path: filePath,
name: metadata.name,
mime_type: 'application/pdf',
size: metadata.size,
created_time: metadata.server_modified,
modified_time: metadata.client_modified,
ai_model: 'gemini-2.5-flash-preview-05-20',
ai_processing_time_ms: processingTime,
doc_type: analysisResult.doc_type,
synopsis: analysisResult.synopsis,
confidence_score: analysisResult.confidence_score
};
return {
isError: false,
skipped: false,
documentData,
typeSpecificData: analysisResult,
filePath // Add file path for better tracking
};
}
catch (error) {
log.error('Optimized file indexing failed', {
path: file.path_display,
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined
});
return {
isError: true,
skipped: false,
error: error instanceof Error ? error.message : String(error)
};
}
}
export async function indexFolder(args) {
try {
const { path, paths, recursive = true, batchSize = 200, excludePaths = [], excludePatterns = [], respectGlobalExclusions = true, scopeToDirectories = false } = args;
// Handle both single path and multiple paths
const targetPaths = paths || (path ? [path] : []);
if (targetPaths.length === 0) {
return {
content: [{
type: "text",
text: "Error: No paths specified. Provide either 'path' or 'paths' parameter."
}],
isError: true,
};
}
log.info('Starting ultra-high-concurrency multi-directory folder indexing', {
paths: targetPaths,
pathCount: targetPaths.length,
recursive,
batchSize,
excludePaths: excludePaths.length,
excludePatterns: excludePatterns.length,
respectGlobalExclusions,
scopeToDirectories,
parallelismLevel: 'ultra-high (200 simultaneous AI calls)'
});
// Get all PDF files from all specified directories
const allFiles = [];
const pathResults = new Map();
for (const targetPath of targetPaths) {
log.info('Scanning directory for PDFs', { path: targetPath });
try {
const pathFiles = await listPDFFiles(targetPath, recursive);
allFiles.push(...pathFiles);
pathResults.set(targetPath, pathFiles.length);
log.info('Directory scan completed', {
path: targetPath,
filesFound: pathFiles.length
});
}
catch (error) {
log.warn('Failed to scan directory', {
path: targetPath,
error: error instanceof Error ? error.message : String(error)
});
pathResults.set(targetPath, 0);
}
}
// Remove duplicates (in case paths overlap)
const uniqueFiles = allFiles.filter((file, index, arr) => arr.findIndex(f => f.path_display === file.path_display) === index);
log.info('Multi-directory scan summary', {
totalPaths: targetPaths.length,
pathResults: Object.fromEntries(pathResults),
totalFiles: allFiles.length,
uniqueFiles: uniqueFiles.length,
duplicatesRemoved: allFiles.length - uniqueFiles.length
});
// Apply exclusion filters with directory scoping
const files = uniqueFiles.filter(file => !isFileExcluded(file.path_display, excludePaths, excludePatterns, respectGlobalExclusions, scopeToDirectories ? targetPaths : undefined));
if (files.length === 0) {
const excludedCount = uniqueFiles.length - files.length;
const pathList = targetPaths.length === 1 ? `"${targetPaths[0]}"` : `${targetPaths.length} directories`;
const message = uniqueFiles.length === 0
? `No PDF files found in ${pathList}`
: `No PDF files found in ${pathList} after exclusions (${excludedCount} files excluded)`;
return {
content: [{
type: "text",
text: message
}],
isError: false,
};
}
log.info('Found PDF files for indexing across all directories', {
count: files.length,
paths: targetPaths,
averagePerPath: Math.round(files.length / targetPaths.length)
});
let indexed = 0;
let skipped = 0;
let errors = 0;
const errorDetails = [];
// Process in high-concurrency parallel batches with optimized database operations
for (let i = 0; i < files.length; i += batchSize) {
const batch = files.slice(i, i + batchSize);
const batchNumber = Math.floor(i / batchSize) + 1;
const totalBatches = Math.ceil(files.length / batchSize);
log.info('Processing ultra-high-concurrency batch', {
batch: batchNumber,
totalBatches,
filesInBatch: batch.length,
concurrency: batchSize,
parallelApiCalls: batch.length
});
// Process batch with maximum parallel concurrency
const batchStartTime = Date.now();
const batchResults = await Promise.allSettled(batch.map(file => indexFileOptimized(file)));
const batchProcessingTime = Date.now() - batchStartTime;
// Collect successful results for optimized batch database operations
const documentsToInsert = [];
const typeSpecificDataWithIds = [];
const successfulResults = [];
// Analyze results with better error tracking
for (let j = 0; j < batchResults.length; j++) {
const result = batchResults[j];
const fileName = batch[j].name;
if (result.status === 'fulfilled') {
if (result.value.isError) {
errors++;
errorDetails.push(`${fileName}: ${result.value.error}`);
}
else if (result.value.skipped) {
skipped++;
}
else {
documentsToInsert.push(result.value.documentData);
successfulResults.push(result.value);
}
}
else {
errors++;
errorDetails.push(`${fileName}: ${result.reason}`);
}
}
// Optimized batch database operations
if (documentsToInsert.length > 0) {
try {
const dbStartTime = Date.now();
// Insert documents in a single optimized transaction
const documentIds = await documentDb.insertDocumentsBatch(documentsToInsert);
indexed += documentIds.length;
// Prepare type-specific data for high-performance batch operations
const typeSpecificEntries = [];
for (let k = 0; k < successfulResults.length; k++) {
if (successfulResults[k].typeSpecificData) {
typeSpecificEntries.push({
docId: documentIds[k],
analysisResult: successfulResults[k].typeSpecificData
});
}
}
// Execute optimized batch type-specific data operations
if (typeSpecificEntries.length > 0) {
try {
// Use individual storage function to respect legacy vs dynamic logic
for (let k = 0; k < typeSpecificEntries.length; k++) {
const entry = typeSpecificEntries[k];
await storeTypeSpecificData(entry.docId, entry.analysisResult);
}
log.debug('Batch type-specific data operations completed', {
entriesProcessed: typeSpecificEntries.length
});
}
catch (typeSpecificError) {
log.warn('Batch type-specific data operations failed', {
error: typeSpecificError,
entriesCount: typeSpecificEntries.length
});
}
}
const dbProcessingTime = Date.now() - dbStartTime;
log.info('Ultra-high-performance batch completed', {
documentsInserted: documentIds.length,
batchProcessingTimeMs: batchProcessingTime,
dbProcessingTimeMs: dbProcessingTime,
avgProcessingPerDoc: Math.round(batchProcessingTime / batch.length),
parallelApiCalls: batch.length,
throughputDocsPerSecond: Math.round((batch.length * 1000) / batchProcessingTime)
});
}
catch (dbError) {
errors += documentsToInsert.length;
errorDetails.push(`Database batch error: ${dbError}`);
log.error('Batch database operation failed', { dbError, batchSize: documentsToInsert.length });
}
}
// Ultra-minimal rate limiting for maximum throughput
if (i + batchSize < files.length) {
log.debug('Ultra-brief rate limiting pause');
await new Promise(resolve => setTimeout(resolve, 50)); // Reduced to 50ms for ultra-high throughput
}
}
const pathSummary = targetPaths.length === 1
? `"${targetPaths[0]}"`
: `${targetPaths.length} directories (${targetPaths.slice(0, 2).join(', ')}${targetPaths.length > 2 ? '...' : ''})`;
let resultText = `Multi-directory indexing complete for ${pathSummary}:\n`;
resultText += `- Indexed: ${indexed} files\n`;
resultText += `- Skipped (up-to-date): ${skipped} files\n`;
resultText += `- Errors: ${errors} files\n`;
if (targetPaths.length > 1) {
resultText += `\nDirectory breakdown:\n`;
for (const [dirPath, foundCount] of pathResults) {
resultText += `- ${dirPath}: ${foundCount} PDFs found\n`;
}
}
if (errorDetails.length > 0 && errorDetails.length <= 5) {
resultText += `\n\nError details:\n${errorDetails.join('\n')}`;
}
else if (errorDetails.length > 5) {
resultText += `\n\nFirst 5 errors:\n${errorDetails.slice(0, 5).join('\n')}\n... and ${errorDetails.length - 5} more errors`;
}
log.info('Folder indexing completed', {
path,
indexed,
skipped,
errors,
totalFiles: files.length
});
return {
content: [{ type: "text", text: resultText }],
isError: errors > 0,
};
}
catch (error) {
log.error('Folder indexing failed', {
path: args.path,
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined
});
return {
content: [{
type: "text",
text: `Error indexing folder: ${error instanceof Error ? error.message : String(error)}`
}],
isError: true,
};
}
}
export async function searchIndexed(args) {
try {
const { query, docType, limit = 20 } = args;
log.info('Searching indexed documents', { query, docType, limit });
const results = await documentDb.searchDocuments(query, docType, limit);
if (results.length === 0) {
return {
content: [{
type: "text",
text: `No indexed documents found matching "${query}"${docType ? ` of type "${docType}"` : ''}`
}],
isError: false,
};
}
let resultText = `Found ${results.length} indexed document(s) matching "${query}"${docType ? ` of type "${docType}"` : ''}:\n\n`;
for (const doc of results) {
resultText += `📄 **${doc.name}** (${doc.doc_type})\n`;
resultText += ` Path: ${doc.dropbox_path}\n`;
resultText += ` Synopsis: ${doc.synopsis}\n`;
resultText += ` Confidence: ${(doc.confidence_score * 100).toFixed(1)}%\n`;
resultText += ` Indexed: ${new Date(doc.indexed_at).toLocaleDateString()}\n\n`;
}
log.info('Search completed', {
query,
docType,
resultsCount: results.length
});
return {
content: [{ type: "text", text: resultText }],
isError: false,
};
}
catch (error) {
log.error('Search failed', {
query: args.query,
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined
});
return {
content: [{
type: "text",
text: `Error searching indexed documents: ${error instanceof Error ? error.message : String(error)}`
}],
isError: true,
};
}
}
export async function getDocumentStats() {
try {
log.info('Getting document statistics');
const stats = await documentDb.getDocumentStats();
const expiringContracts = await documentDb.getExpiringContracts(30);
const overdueInvoices = await documentDb.getOverdueInvoices();
let resultText = `📊 **Document Index Statistics**\n\n`;
resultText += `**Total Documents:** ${stats.total_documents}\n\n`;
if (stats.by_type && stats.by_type.length > 0) {
resultText += `**By Document Type:**\n`;
for (const type of stats.by_type) {
resultText += `- ${type.doc_type}: ${type.count} (avg confidence: ${(type.avg_confidence * 100).toFixed(1)}%)\n`;
}
resultText += '\n';
}
if (expiringContracts.length > 0) {
resultText += `⚠️ **Contracts Expiring in Next 30 Days:** ${expiringContracts.length}\n`;
for (const contract of expiringContracts.slice(0, 5)) {
resultText += `- ${contract.name} (${contract.contract_number || 'No Number'}) - expires ${contract.expiration_date}\n`;
}
if (expiringContracts.length > 5) {
resultText += `... and ${expiringContracts.length - 5} more\n`;
}
resultText += '\n';
}
if (overdueInvoices.length > 0) {
resultText += `🔴 **Overdue Invoices:** ${overdueInvoices.length}\n`;
for (const invoice of overdueInvoices.slice(0, 5)) {
resultText += `- ${invoice.name} (${invoice.invoice_number || 'No Number'}) - due ${invoice.due_date}\n`;
}
if (overdueInvoices.length > 5) {
resultText += `... and ${overdueInvoices.length - 5} more\n`;
}
}
log.info('Document statistics retrieved', {
totalDocs: stats.total_documents,
expiringContracts: expiringContracts.length,
overdueInvoices: overdueInvoices.length
});
return {
content: [{ type: "text", text: resultText }],
isError: false,
};
}
catch (error) {
log.error('Failed to get document statistics', {
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined
});
return {
content: [{
type: "text",
text: `Error getting document statistics: ${error instanceof Error ? error.message : String(error)}`
}],
isError: true,
};
}
}
// Helper functions
async function listPDFFiles(path, recursive) {
// Simple implementation - in a real system you'd want to paginate
const files = [];
// Use the existing search functionality to find PDF files
const searchResult = await dbxApi.searchFiles({
query: '.pdf',
path: path,
maxResults: 1000, // Reasonable limit
fileExtensions: ['pdf'],
fileCategories: ['pdf']
});
// Parse the search results to extract file metadata
const searchContent = searchResult.content[0].text;
try {
const results = JSON.parse(searchContent);
if (results.matches) {
for (const match of results.matches) {
if (match.metadata && match.metadata['.tag'] === 'file') {
files.push(match.metadata);
}
}
}
}
catch (error) {
log.warn('Failed to parse search results for PDF files', { path, error });
}
return files;
}
async function storeTypeSpecificData(docId, result) {
try {
// Use the new dynamic schema-based storage
const typeConfig = schemaManager.getDocumentTypeConfig(result.doc_type);
if (!typeConfig) {
log.warn('Unknown document type for type-specific storage', { docType: result.doc_type });
return;
}
// For backward compatibility with legacy types, use legacy storage methods
if (['contract', 'invoice', 'proposal', 'report'].includes(result.doc_type)) {
log.debug('Attempting legacy type-specific storage', {
docId,
docType: result.doc_type,
hasFields: !!result.fields,
fieldCount: result.fields ? Object.keys(result.fields).length : 0
});
await storeLegacyTypeSpecificData(docId, result);
log.debug('Legacy type-specific storage completed', { docId, docType: result.doc_type });
}
else {
// For new custom types, use the dynamic storage method
log.debug('Using dynamic storage for custom type', { docId, docType: result.doc_type });
await documentDb.batchStoreTypeSpecificData([{ docId, analysisResult: result }]);
}
}
catch (error) {
log.error('Failed to store type-specific data', {
docId,
docType: result.doc_type,
error: error instanceof Error ? error.message : String(error)
});
// Don't throw - the main document is still indexed
}
}
async function storeLegacyTypeSpecificData(docId, result) {
log.debug('Starting legacy storage for document type', {
docId,
docType: result.doc_type,
fieldsSnapshot: result.fields ? Object.keys(result.fields).slice(0, 5) : []
});
switch (result.doc_type) {
case 'invoice':
const invoiceId = await documentDb.insertInvoice(docId, {
invoice_number: result.fields.invoice_number,
invoice_date: result.fields.invoice_date,
due_date: result.fields.due_date,
total_amount: result.fields.total_amount,
currency: result.fields.currency,
supplier_name: result.fields.supplier_name,
customer_name: result.fields.customer_name,
supplier_address: result.fields.supplier_address,
customer_address: result.fields.customer_address,
subtotal: result.fields.subtotal,
tax_amount: result.fields.tax_amount,
payment_status: result.fields.payment_status,
payment_method: result.fields.payment_method,
});
// Insert invoice items if present
if (result.fields.items && result.fields.items.length > 0) {
await documentDb.insertInvoiceItems(invoiceId, result.fields.items);
}
break;
case 'contract':
try {
log.debug('Inserting contract data', {
docId,
contractData: {
contract_number: result.fields.contract_number,
contract_type: result.fields.contract_type,
effective_date: result.fields.effective_date,
expiration_date: result.fields.expiration_date
}
});
const contractId = await documentDb.insertContract(docId, {
contract_number: result.fields.contract_number,
contract_type: result.fields.contract_type,
effective_date: result.fields.effective_date,
expiration_date: result.fields.expiration_date,
auto_renewal: result.fields.auto_renewal,
notice_period_days: result.fields.notice_period_days,
total_value: result.fields.total_value,
currency: result.fields.currency,
payment_terms: result.fields.payment_terms,
governing_law: result.fields.governing_law,
});
log.debug('Contract inserted successfully', { docId, contractId });
// Insert contract parties if present
if (result.fields.parties && result.fields.parties.length > 0) {
log.debug('Inserting contract parties', { docId, contractId, partiesCount: result.fields.parties.length });
await documentDb.insertContractParties(contractId, result.fields.parties.map((party) => ({
party_name: party,
party_role: 'party' // Default role
})));
log.debug('Contract parties inserted successfully', { docId, contractId });
}
}
catch (contractError) {
log.error('Failed to insert contract', {
docId,
error: contractError instanceof Error ? contractError.message : String(contractError),
stack: contractError instanceof Error ? contractError.stack : undefined
});
throw contractError;
}
break;
case 'proposal':
await documentDb.insertProposal(docId, {
proposal_number: result.fields.proposal_number,
proposal_date: result.fields.proposal_date,
client_name: result.fields.client_name,
project_title: result.fields.project_title,
total_value: result.fields.total_value,
currency: result.fields.currency,
validity_period: result.fields.validity_period,
timeline: result.fields.timeline,
});
break;
case 'report':
await documentDb.insertReport(docId, {
report_title: result.fields.report_title,
report_date: result.fields.report_date,
report_type: result.fields.report_type,
author: result.fields.author,
period_covered: result.fields.period_covered,
});
break;
}
}
// Helper function to check if a file should be excluded from indexing
function isFileExcluded(filePath, excludePaths, excludePatterns, respectGlobalExclusions, scopeDirectories) {
// Normalize path for consistent comparison
const normalizedPath = filePath.toLowerCase();
// Check function-specific exclude paths
for (const excludePath of excludePaths) {
const normalizedExcludePath = excludePath.toLowerCase();
if (normalizedPath === normalizedExcludePath ||
normalizedPath.startsWith(normalizedExcludePath + '/')) {
log.debug('File excluded by explicit path', { filePath, excludePath });
return true;
}
}
// Check function-specific exclude patterns
for (const pattern of excludePatterns) {
if (minimatch(normalizedPath, pattern.toLowerCase())) {
log.debug('File excluded by explicit pattern', { filePath, pattern });
return true;
}
}
// Check global exclusions if enabled
if (respectGlobalExclusions) {
// If scoped to directories, only apply global exclusions within those scopes
if (scopeDirectories && scopeDirectories.length > 0) {
// Check if file is within any of the scoped directories
const isWithinScope = scopeDirectories.some(scopeDir => {
const normalizedScopeDir = scopeDir.toLowerCase();
return normalizedPath.startsWith(normalizedScopeDir);
});
if (isWithinScope) {
// Check global exclude paths (but only those within or relative to any scope)
for (const excludePath of config.indexing.globalExcludePaths) {
const normalizedExcludePath = excludePath.toLowerCase();
// Check against each scope directory
for (const scopeDir of scopeDirectories) {
const normalizedScopeDir = scopeDir.toLowerCase();
// Make exclude path relative to scope if it's absolute
const scopedExcludePath = normalizedExcludePath.startsWith('/')
? normalizedScopeDir + normalizedExcludePath
: normalizedScopeDir + '/' + normalizedExcludePath;
if (normalizedPath === scopedExcludePath ||
normalizedPath.startsWith(scopedExcludePath + '/')) {
log.debug('File excluded by scoped global path', { filePath, excludePath, scopeDirectories });
return true;
}
}
}
// Check global exclude patterns (only within scopes)
for (const pattern of config.indexing.globalExcludePatterns) {
for (const scopeDir of scopeDirectories) {
const normalizedScopeDir = scopeDir.toLowerCase();
// Create a scoped pattern
const scopedPattern = pattern.startsWith('*')
? normalizedScopeDir + '/' + pattern
: pattern;
if (minimatch(normalizedPath, scopedPattern.toLowerCase())) {
log.debug('File excluded by scoped global pattern', { filePath, pattern, scopeDirectories });
return true;
}
}
}
}
}
else {
// Original behavior: apply global exclusions everywhere
// Check global exclude paths
for (const excludePath of config.indexing.globalExcludePaths) {
const normalizedExcludePath = excludePath.toLowerCase();
if (normalizedPath === normalizedExcludePath ||
normalizedPath.startsWith(normalizedExcludePath + '/')) {
log.debug('File excluded by global path', { filePath, excludePath });
return true;
}
}
// Check global exclude patterns
for (const pattern of config.indexing.globalExcludePatterns) {
if (minimatch(normalizedPath, pattern.toLowerCase())) {
log.debug('File excluded by global pattern', { filePath, pattern });
return true;
}
}
}
}
return false;
}
//# sourceMappingURL=document-indexer.js.map