dbx-mcp-server
Version:
A Model Context Protocol server for Dropbox integration with AI-powered PDF analysis, multi-directory indexing, and parallel processing
465 lines • 20.6 kB
JavaScript
import Database from 'better-sqlite3';
import { log } from './config.js';
import { schemaManager } from './schema-manager.js';
import * as path from 'path';
import * as fs from 'fs';
export class DocumentDatabase {
constructor(dbPath) {
this.dbPath = dbPath || path.join(process.cwd(), 'dbx-db', 'documents.db');
// Ensure database directory exists
const dbDir = path.dirname(this.dbPath);
if (!fs.existsSync(dbDir)) {
fs.mkdirSync(dbDir, { recursive: true });
}
this.db = new Database(this.dbPath);
this.initializeSchema();
}
initializeSchema() {
log.info('Initializing database schema', { dbPath: this.dbPath });
// Enable foreign keys
this.db.pragma('foreign_keys = ON');
// Create documents table
this.db.exec(`
CREATE TABLE IF NOT EXISTS documents (
id INTEGER PRIMARY KEY AUTOINCREMENT,
dropbox_path TEXT UNIQUE NOT NULL,
name TEXT NOT NULL,
mime_type TEXT NOT NULL,
size INTEGER,
created_time TEXT,
modified_time TEXT,
ai_model TEXT,
ai_processing_time_ms INTEGER,
doc_type TEXT NOT NULL,
synopsis TEXT,
confidence_score REAL,
indexed_at TEXT DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
)
`);
// Create dynamic document type tables with error handling
try {
const dynamicSchemas = schemaManager.generateDatabaseSchema();
for (const schema of dynamicSchemas) {
try {
this.db.exec(schema);
log.debug('Successfully created table schema', { schema: schema.substring(0, 50) + '...' });
}
catch (tableError) {
log.warn('Failed to create table schema, continuing with other tables', {
error: tableError instanceof Error ? tableError.message : String(tableError),
schema: schema.substring(0, 100) + '...'
});
}
}
}
catch (schemaError) {
log.error('Failed to generate database schemas from configuration', {
error: schemaError instanceof Error ? schemaError.message : String(schemaError)
});
// Continue with legacy tables only
}
// Legacy table creation for backward compatibility
this.createLegacyTables();
// Create indexes for better search performance
try {
this.db.exec(`
CREATE INDEX IF NOT EXISTS idx_documents_path ON documents(dropbox_path);
CREATE INDEX IF NOT EXISTS idx_documents_type ON documents(doc_type);
CREATE INDEX IF NOT EXISTS idx_documents_modified ON documents(modified_time);
`);
// Create indexes for legacy tables (only if they exist)
try {
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_contracts_expiration ON contracts(expiration_date);`);
}
catch (e) { /* Table might not exist */ }
try {
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_invoices_due_date ON invoices(due_date);`);
}
catch (e) { /* Table might not exist */ }
try {
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_invoices_supplier ON invoices(supplier_name);`);
}
catch (e) { /* Table might not exist */ }
}
catch (indexError) {
log.warn('Failed to create some indexes', {
error: indexError instanceof Error ? indexError.message : String(indexError)
});
}
log.info('Database schema initialized successfully');
}
createLegacyTables() {
// Create legacy tables for backward compatibility with existing data
this.db.exec(`
CREATE TABLE IF NOT EXISTS contract_parties (
id INTEGER PRIMARY KEY AUTOINCREMENT,
contract_id INTEGER NOT NULL REFERENCES contracts(id) ON DELETE CASCADE,
party_name TEXT NOT NULL,
party_role TEXT NOT NULL,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
)
`);
this.db.exec(`
CREATE TABLE IF NOT EXISTS invoice_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
invoice_id INTEGER NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
description TEXT,
quantity REAL,
unit_price REAL,
total_price REAL,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
)
`);
}
// Document operations
async getDocumentByDropboxPath(path) {
const stmt = this.db.prepare(`
SELECT * FROM documents WHERE dropbox_path = ?
`);
return stmt.get(path);
}
async insertDocument(doc) {
const stmt = this.db.prepare(`
INSERT OR REPLACE INTO documents (
dropbox_path, name, mime_type, size, created_time, modified_time,
ai_model, ai_processing_time_ms, doc_type, synopsis, confidence_score
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const result = stmt.run(doc.dropbox_path, doc.name, doc.mime_type, doc.size, doc.created_time, doc.modified_time, doc.ai_model, doc.ai_processing_time_ms, doc.doc_type, doc.synopsis, doc.confidence_score);
return result.lastInsertRowid;
}
async insertDocumentsBatch(docs) {
// Optimize SQLite for batch operations
this.db.pragma('journal_mode = WAL');
this.db.pragma('synchronous = NORMAL');
this.db.pragma('cache_size = 10000');
this.db.pragma('temp_store = MEMORY');
const insertStmt = this.db.prepare(`
INSERT OR REPLACE INTO documents (
dropbox_path, name, mime_type, size, created_time, modified_time,
ai_model, ai_processing_time_ms, doc_type, synopsis, confidence_score
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const transaction = this.db.transaction((documents) => {
const insertedIds = [];
for (const doc of documents) {
const result = insertStmt.run(doc.dropbox_path, doc.name, doc.mime_type, doc.size, doc.created_time, doc.modified_time, doc.ai_model, doc.ai_processing_time_ms, doc.doc_type, doc.synopsis, doc.confidence_score);
insertedIds.push(result.lastInsertRowid);
}
return insertedIds;
});
return transaction(docs);
}
// High-performance batch insert for type-specific data operations
async batchStoreTypeSpecificData(dataEntries) {
if (dataEntries.length === 0)
return;
// Group by document type for optimized batch inserts
const groupedData = new Map();
for (const entry of dataEntries) {
const docType = entry.analysisResult.doc_type;
if (!groupedData.has(docType)) {
groupedData.set(docType, []);
}
groupedData.get(docType).push(entry);
}
// Execute batch inserts for each document type in parallel
const batchOperations = [];
for (const [docType, entries] of groupedData) {
// Use dynamic batch insertion based on schema configuration
batchOperations.push(this.batchInsertDocumentType(docType, entries));
}
// Execute all batch operations in parallel
await Promise.allSettled(batchOperations);
}
async batchInsertDocumentType(docType, entries) {
const typeConfig = schemaManager.getDocumentTypeConfig(docType);
if (!typeConfig) {
log.warn('Unknown document type for batch insert', { docType });
return;
}
const tableName = this.getTableNameForType(docType);
const fields = Object.keys(typeConfig.fields);
// Build dynamic insert statement
const fieldList = ['document_id', ...fields].join(', ');
const placeholders = ['?', ...fields.map(() => '?')].join(', ');
const insertStmt = this.db.prepare(`
INSERT INTO ${tableName} (${fieldList})
VALUES (${placeholders})
`);
const transaction = this.db.transaction(() => {
for (const entry of entries) {
const values = [entry.docId];
// Extract field values from analysis result
for (const fieldName of fields) {
const fieldValue = entry.analysisResult.fields[fieldName];
// Handle array and object types by storing as JSON
if (fieldValue && (Array.isArray(fieldValue) || typeof fieldValue === 'object')) {
values.push(JSON.stringify(fieldValue));
}
else {
values.push(fieldValue);
}
}
try {
insertStmt.run(...values);
}
catch (error) {
log.warn('Failed to insert type-specific data', {
docType,
fieldCount: fields.length,
error: error instanceof Error ? error.message : String(error)
});
}
}
});
transaction();
}
getTableNameForType(docType) {
// Use schema manager's table naming convention
return docType === 'contract' ? 'contracts' :
docType === 'invoice' ? 'invoices' :
docType === 'proposal' ? 'proposals' :
docType === 'report' ? 'reports' :
`${docType}s`;
}
async batchInsertInvoices(entries) {
const insertInvoiceStmt = this.db.prepare(`
INSERT INTO invoices (
document_id, invoice_number, invoice_date, due_date, total_amount, currency,
supplier_name, customer_name, supplier_address, customer_address,
subtotal, tax_amount, payment_status, payment_method
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const insertItemStmt = this.db.prepare(`
INSERT INTO invoice_items (invoice_id, description, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
`);
const transaction = this.db.transaction(() => {
for (const entry of entries) {
const fields = entry.analysisResult.fields;
const result = insertInvoiceStmt.run(entry.docId, fields.invoice_number, fields.invoice_date, fields.due_date, fields.total_amount, fields.currency, fields.supplier_name, fields.customer_name, fields.supplier_address, fields.customer_address, fields.subtotal, fields.tax_amount, fields.payment_status, fields.payment_method);
const invoiceId = result.lastInsertRowid;
// Batch insert invoice items if present
if (fields.items && fields.items.length > 0) {
for (const item of fields.items) {
insertItemStmt.run(invoiceId, item.description, item.quantity, item.unit_price, item.total_price);
}
}
}
});
transaction();
}
async batchInsertContracts(entries) {
const insertContractStmt = this.db.prepare(`
INSERT INTO contracts (
document_id, contract_number, contract_type, effective_date, expiration_date,
auto_renewal, notice_period_days, total_value, currency, payment_terms, governing_law
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const insertPartyStmt = this.db.prepare(`
INSERT INTO contract_parties (contract_id, party_name, party_role)
VALUES (?, ?, ?)
`);
const transaction = this.db.transaction(() => {
for (const entry of entries) {
const fields = entry.analysisResult.fields;
const result = insertContractStmt.run(entry.docId, fields.contract_number, fields.contract_type, fields.effective_date, fields.expiration_date, fields.auto_renewal, fields.notice_period_days, fields.total_value, fields.currency, fields.payment_terms, fields.governing_law);
const contractId = result.lastInsertRowid;
// Batch insert contract parties if present
if (fields.parties && fields.parties.length > 0) {
for (const party of fields.parties) {
insertPartyStmt.run(contractId, party, 'party');
}
}
}
});
transaction();
}
async batchInsertProposals(entries) {
const insertStmt = this.db.prepare(`
INSERT INTO proposals (
document_id, proposal_number, proposal_date, client_name, project_title,
total_value, currency, validity_period, timeline
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const transaction = this.db.transaction(() => {
for (const entry of entries) {
const fields = entry.analysisResult.fields;
insertStmt.run(entry.docId, fields.proposal_number, fields.proposal_date, fields.client_name, fields.project_title, fields.total_value, fields.currency, fields.validity_period, fields.timeline);
}
});
transaction();
}
async batchInsertReports(entries) {
const insertStmt = this.db.prepare(`
INSERT INTO reports (
document_id, report_title, report_date, report_type, author, period_covered
) VALUES (?, ?, ?, ?, ?, ?)
`);
const transaction = this.db.transaction(() => {
for (const entry of entries) {
const fields = entry.analysisResult.fields;
insertStmt.run(entry.docId, fields.report_title, fields.report_date, fields.report_type, fields.author, fields.period_covered);
}
});
transaction();
}
async searchDocuments(query, docType, limit = 50) {
let sql = `
SELECT * FROM documents
WHERE (name LIKE ? OR synopsis LIKE ?)
`;
const params = [`%${query}%`, `%${query}%`];
if (docType) {
sql += ` AND doc_type = ?`;
params.push(docType);
}
sql += ` ORDER BY indexed_at DESC LIMIT ?`;
params.push(String(limit));
const stmt = this.db.prepare(sql);
return stmt.all(...params);
}
async deleteDocument(id) {
const stmt = this.db.prepare('DELETE FROM documents WHERE id = ?');
stmt.run(id);
}
// Contract operations
async insertContract(documentId, contract) {
const stmt = this.db.prepare(`
INSERT INTO contracts (
document_id, contract_number, contract_type, effective_date, expiration_date,
auto_renewal, notice_period_days, total_value, currency, payment_terms, governing_law
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const result = stmt.run(documentId, contract.contract_number, contract.contract_type, contract.effective_date, contract.expiration_date, contract.auto_renewal, contract.notice_period_days, contract.total_value, contract.currency, contract.payment_terms, contract.governing_law);
return result.lastInsertRowid;
}
async insertContractParties(contractId, parties) {
const stmt = this.db.prepare(`
INSERT INTO contract_parties (contract_id, party_name, party_role)
VALUES (?, ?, ?)
`);
for (const party of parties) {
stmt.run(contractId, party.party_name, party.party_role);
}
}
async getExpiringContracts(days = 30) {
const stmt = this.db.prepare(`
SELECT d.name, d.dropbox_path, c.contract_number, c.expiration_date, c.total_value, c.currency
FROM documents d
JOIN contracts c ON d.id = c.document_id
WHERE c.expiration_date IS NOT NULL
AND date(c.expiration_date) <= date('now', '+' || ? || ' days')
ORDER BY c.expiration_date ASC
`);
return stmt.all(days);
}
// Invoice operations
async insertInvoice(documentId, invoice) {
const stmt = this.db.prepare(`
INSERT INTO invoices (
document_id, invoice_number, invoice_date, due_date, supplier_name,
supplier_address, customer_name, customer_address, subtotal, tax_amount,
total_amount, currency, payment_status, payment_method
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const result = stmt.run(documentId, invoice.invoice_number, invoice.invoice_date, invoice.due_date, invoice.supplier_name, invoice.supplier_address, invoice.customer_name, invoice.customer_address, invoice.subtotal, invoice.tax_amount, invoice.total_amount, invoice.currency, invoice.payment_status, invoice.payment_method);
return result.lastInsertRowid;
}
async insertInvoiceItems(invoiceId, items) {
const stmt = this.db.prepare(`
INSERT INTO invoice_items (invoice_id, description, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
`);
const transaction = this.db.transaction(() => {
for (const item of items) {
stmt.run(invoiceId, item.description, item.quantity, item.unit_price, item.total_price);
}
});
transaction();
}
async getOverdueInvoices() {
const stmt = this.db.prepare(`
SELECT d.name, d.dropbox_path, i.invoice_number, i.due_date, i.total_amount, i.currency
FROM documents d
JOIN invoices i ON d.id = i.document_id
WHERE i.due_date IS NOT NULL
AND date(i.due_date) < date('now')
AND (i.payment_status IS NULL OR i.payment_status != 'paid')
ORDER BY i.due_date ASC
`);
return stmt.all();
}
// Proposal operations
async insertProposal(documentId, proposal) {
const stmt = this.db.prepare(`
INSERT INTO proposals (
document_id, proposal_number, proposal_date, client_name, project_title,
total_value, currency, validity_period, timeline
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const result = stmt.run(documentId, proposal.proposal_number, proposal.proposal_date, proposal.client_name, proposal.project_title, proposal.total_value, proposal.currency, proposal.validity_period, proposal.timeline);
return result.lastInsertRowid;
}
// Report operations
async insertReport(documentId, report) {
const stmt = this.db.prepare(`
INSERT INTO reports (
document_id, report_title, report_date, report_type, author, period_covered
) VALUES (?, ?, ?, ?, ?, ?)
`);
const result = stmt.run(documentId, report.report_title, report.report_date, report.report_type, report.author, report.period_covered);
return result.lastInsertRowid;
}
// Analytics and business intelligence
async getDocumentStats() {
const stats = this.db.prepare(`
SELECT
doc_type,
COUNT(*) as count,
AVG(confidence_score) as avg_confidence
FROM documents
GROUP BY doc_type
`).all();
const totalDocs = this.db.prepare('SELECT COUNT(*) as total FROM documents').get();
return {
total_documents: totalDocs.total,
by_type: stats
};
}
close() {
this.db.close();
}
}
// Export singleton instance with error handling
let documentDb;
try {
documentDb = new DocumentDatabase();
}
catch (error) {
log.error('Failed to initialize document database, using fallback instance', {
error: error instanceof Error ? error.message : String(error)
});
// Create a fallback instance that won't fail the entire server
documentDb = {
async getDocumentByDropboxPath() { return null; },
async insertDocument() { return 0; },
async insertDocumentsBatch() { return []; },
async batchStoreTypeSpecificData() { return; },
async searchDocuments() { return []; },
async getDocumentStats() { return { total_documents: 0, by_type: [] }; },
async getExpiringContracts() { return []; },
async getOverdueInvoices() { return []; },
async insertInvoice() { return 0; },
async insertInvoiceItems() { return; },
async insertContract() { return 0; },
async insertContractParties() { return; },
async insertProposal() { return 0; },
async insertReport() { return 0; },
close() { return; }
};
}
export { documentDb };
//# sourceMappingURL=database.js.map