mcp-gdrive-enhanced-markov
Version:
MCP server for Google Drive and Sheets with write capabilities, AI-powered PDF analysis, SQLite document indexing, and advanced directory exploration tools
285 lines (284 loc) • 12.2 kB
JavaScript
import { google } from 'googleapis';
import { getValidCredentials } from '../auth.js';
import { dbV2 } from './database_v2.js';
import { ParallelIndexer } from './parallel_indexer.js';
export class SmartUpdater {
indexer;
constructor() {
this.indexer = new ParallelIndexer();
}
async checkForUpdates() {
const startTime = Date.now();
const syncState = await dbV2.getSyncState();
const authClient = await getValidCredentials();
const drive = google.drive({ version: 'v3', auth: authClient });
let changeToken = syncState.drive_change_token;
const changes = [];
let newStartPageToken;
try {
// If no change token, get the current one to start tracking from now
if (!changeToken) {
const tokenResponse = await drive.changes.getStartPageToken({
supportsAllDrives: true
});
changeToken = tokenResponse.data.startPageToken;
await dbV2.updateSyncState({
drive_change_token: changeToken,
last_sync_time: new Date().toISOString()
});
console.log('No previous sync state found. Starting fresh tracking from now.');
return {
new_files: 0,
updated_files: 0,
deleted_files: 0,
total_changes: 0,
errors: 0,
duration_seconds: 0
};
}
// Fetch all changes since last sync
console.log(`Checking for changes since token: ${changeToken}`);
let nextPageToken;
let newStartPageToken;
do {
const response = await drive.changes.list({
pageToken: changeToken,
pageSize: 100,
fields: 'nextPageToken,newStartPageToken,changes(fileId,file(id,name,mimeType,modifiedTime,trashed,parents),removed,time)',
supportsAllDrives: true,
includeItemsFromAllDrives: true
});
const changesResponse = response.data;
if (changesResponse.changes) {
for (const change of changesResponse.changes) {
if (change.file && change.file.mimeType === 'application/pdf') {
changes.push({
fileId: change.fileId,
fileName: change.file.name,
changeType: change.removed ? 'deleted' :
change.file.trashed ? 'trashed' :
await this.isNewFile(change.fileId) ? 'created' : 'updated',
time: change.time
});
}
}
}
nextPageToken = changesResponse.nextPageToken;
newStartPageToken = changesResponse.newStartPageToken;
changeToken = nextPageToken || undefined;
} while (changeToken && !newStartPageToken);
// Save the new change token
if (newStartPageToken) {
await dbV2.updateSyncState({
drive_change_token: newStartPageToken,
last_sync_time: new Date().toISOString()
});
}
console.log(`Found ${changes.length} PDF changes`);
// Process changes
const toIndex = [];
const toReindex = [];
const toDelete = [];
for (const change of changes) {
switch (change.changeType) {
case 'created':
toIndex.push(change.fileId);
break;
case 'updated':
toReindex.push(change.fileId);
break;
case 'deleted':
case 'trashed':
toDelete.push(change.fileId);
break;
}
}
// Queue operations
if (toIndex.length > 0) {
console.log(`Queueing ${toIndex.length} new files for indexing`);
await dbV2.addToProcessingQueue(toIndex.map(id => ({
drive_id: id,
operation: 'index',
priority: 5
})));
}
if (toReindex.length > 0) {
console.log(`Queueing ${toReindex.length} files for reindexing`);
await dbV2.addToProcessingQueue(toReindex.map(id => ({
drive_id: id,
operation: 'reindex',
priority: 3 // Higher priority for updates
})));
}
if (toDelete.length > 0) {
console.log(`Deleting ${toDelete.length} files from index`);
// Delete from database
for (const fileId of toDelete) {
const doc = await dbV2.getDocumentByDriveId(fileId);
if (doc) {
await dbV2.transaction(async (trx) => {
// The foreign key constraints with CASCADE will handle related records
trx.prepare('DELETE FROM documents WHERE drive_id = ?').run(fileId);
});
}
}
}
// Update sync state stats
const currentStats = await dbV2.getSyncState();
await dbV2.updateSyncState({
total_files_indexed: currentStats.total_files_indexed + toIndex.length + toReindex.length
});
const duration = (Date.now() - startTime) / 1000;
return {
new_files: toIndex.length,
updated_files: toReindex.length,
deleted_files: toDelete.length,
total_changes: changes.length,
errors: 0,
duration_seconds: duration
};
}
catch (error) {
console.error('Error checking for updates:', error);
// Update error count
const currentStats = await dbV2.getSyncState();
await dbV2.updateSyncState({
total_errors: currentStats.total_errors + 1
});
throw error;
}
}
async isNewFile(fileId) {
const existing = await dbV2.getDocumentByDriveId(fileId);
return !existing;
}
async performFullSync(folderId) {
const startTime = Date.now();
console.log('Starting full synchronization...');
try {
const authClient = await getValidCredentials();
const drive = google.drive({ version: 'v3', auth: authClient });
// If no folder specified, sync entire drive
let query = 'mimeType = "application/pdf" and trashed = false';
if (folderId) {
query = `'${folderId}' in parents and ${query}`;
}
const files = [];
let pageToken;
// Collect all PDF files
do {
const response = await drive.files.list({
q: query,
fields: 'nextPageToken, files(id, name, modifiedTime)',
pageSize: 1000,
pageToken,
supportsAllDrives: true,
includeItemsFromAllDrives: true
});
if (response.data.files) {
for (const file of response.data.files) {
files.push(file.id);
}
}
pageToken = response.data.nextPageToken || undefined;
} while (pageToken);
console.log(`Found ${files.length} PDF files for full sync`);
// Check which files need indexing or updating
const toIndex = [];
const toReindex = [];
for (const fileId of files) {
const existing = await dbV2.getDocumentByDriveId(fileId);
if (!existing) {
toIndex.push(fileId);
}
else {
// Get current file metadata to check if it's been modified
const fileData = await drive.files.get({
fileId,
fields: 'modifiedTime'
});
if (fileData.data.modifiedTime &&
new Date(fileData.data.modifiedTime) > new Date(existing.modified_time)) {
toReindex.push(fileId);
}
}
}
// Find deleted files (in DB but not in Drive)
const allDbDocs = await dbV2.searchDocuments('', {}); // Get all
const driveFileSet = new Set(files);
const toDelete = [];
for (const doc of allDbDocs) {
if (!driveFileSet.has(doc.drive_id)) {
toDelete.push(doc.drive_id);
}
}
console.log(`Full sync summary: ${toIndex.length} new, ${toReindex.length} updated, ${toDelete.length} deleted`);
// Queue operations
if (toIndex.length > 0) {
await dbV2.addToProcessingQueue(toIndex.map(id => ({
drive_id: id,
operation: 'index',
priority: 5
})));
}
if (toReindex.length > 0) {
await dbV2.addToProcessingQueue(toReindex.map(id => ({
drive_id: id,
operation: 'reindex',
priority: 3
})));
}
if (toDelete.length > 0) {
for (const fileId of toDelete) {
await dbV2.transaction(async (trx) => {
trx.prepare('DELETE FROM documents WHERE drive_id = ?').run(fileId);
});
}
}
// Get and save current change token for future incremental updates
const tokenResponse = await drive.changes.getStartPageToken({
supportsAllDrives: true
});
await dbV2.updateSyncState({
drive_change_token: tokenResponse.data.startPageToken,
last_sync_time: new Date().toISOString()
});
const duration = (Date.now() - startTime) / 1000;
return {
new_files: toIndex.length,
updated_files: toReindex.length,
deleted_files: toDelete.length,
total_changes: toIndex.length + toReindex.length + toDelete.length,
errors: 0,
duration_seconds: duration
};
}
catch (error) {
console.error('Error in full sync:', error);
throw error;
}
}
async processQueueWithProgress(progressCallback) {
const indexer = new ParallelIndexer(progressCallback);
let processed = 0;
let failed = 0;
await indexer.processQueue();
// Get stats from queue
const stats = await dbV2.transaction((trx) => {
const completed = trx.prepare('SELECT COUNT(*) as count FROM processing_queue WHERE status = "completed"').get();
const failedCount = trx.prepare('SELECT COUNT(*) as count FROM processing_queue WHERE status = "failed"').get();
return { processed: completed.count, failed: failedCount.count };
});
// Clean up completed items older than 24 hours
await dbV2.transaction((trx) => {
trx.prepare(`
DELETE FROM processing_queue
WHERE status IN ('completed', 'failed')
AND processed_at < datetime('now', '-24 hours')
`).run();
});
return stats;
}
}
// Export singleton
export const smartUpdater = new SmartUpdater();