UNPKG

@boundless-oss/atlas

Version:

Atlas - MCP Server for comprehensive startup project management

918 lines (823 loc) 28.8 kB
import { JSONSchema7 } from 'json-schema'; import { randomUUID } from 'crypto'; import { createTool, createSuccessResult, createErrorResult } from '../../core/tool-framework.js'; import { ToolRegistration, RequestContext } from '../../core/types.js'; import { promises as fs } from 'fs'; import path from 'path'; /** * Data Management Tools - 12-Factor MCP Implementation * * Implements Factor 2: Deterministic Execution with structured outputs * Implements Factor 3: Stateless Processes with RequestContext * Implements Factor 4: Structured Outputs for LLM consumption * * Handles data export/import, backup/restore, and data transformation tools */ // Input type interfaces interface ExportAllDataInput { includeConfig?: boolean; format?: 'json' | 'sql'; includeMedia?: boolean; } interface ImportDataInput { sourcePath: string; createBackup?: boolean; mergeStrategy?: 'overwrite' | 'skip_existing' | 'merge'; modules?: string[]; } interface SyncToProjectInput { modules?: string[]; format?: 'json' | 'markdown'; destination?: string; } interface ConfigureStorageInput { mode: 'project-local' | 'user-home' | 'custom'; customPath?: string; isolation?: 'project' | 'user' | 'global'; } interface ExportModuleDataInput { moduleName: string; format?: 'json' | 'markdown' | 'csv' | 'sql'; includeHistory?: boolean; dateRange?: { start?: string; end?: string; }; } interface ImportExternalDataInput { sourcePath: string; sourceType: 'csv' | 'json' | 'xml' | 'sql' | 'api'; targetModule: string; mapping?: Record<string, string>; validation?: { required?: string[]; schema?: any; }; } interface BackupDataInput { modules?: string[]; includeFiles?: boolean; compression?: boolean; encryption?: boolean; retentionDays?: number; } interface RestoreDataInput { backupPath: string; modules?: string[]; restorePoint?: string; verifyIntegrity?: boolean; } interface TransformDataInput { sourceModule: string; targetModule: string; transformationType: 'migrate' | 'duplicate' | 'merge' | 'convert'; mapping: Record<string, string>; customTransform?: string; } interface ValidateDataIntegrityInput { modules?: string[]; includeRelationships?: boolean; autoFix?: boolean; } /** * Export all Atlas data */ const exportAllDataTool = createTool<ExportAllDataInput, any>({ name: 'export_all_data', description: 'Export all Atlas data for backup or migration', category: 'data-management', inputSchema: { type: 'object', properties: { includeConfig: { type: 'boolean', default: true, description: 'Include configuration and metadata' }, format: { type: 'string', enum: ['json', 'sql'], default: 'json', description: 'Export format' }, includeMedia: { type: 'boolean', default: false, description: 'Include media files and attachments' } }, additionalProperties: false } as JSONSchema7, async execute(input: ExportAllDataInput, context: RequestContext) { try { const exportId = randomUUID(); const now = Date.now(); const timestamp = new Date(now).toISOString().replace(/[:.]/g, '-'); // Create export directory const exportDir = path.join(process.cwd(), `atlas-export-${timestamp}`); await fs.mkdir(exportDir, { recursive: true }); let totalExported = 0; const modules: string[] = []; const errors: string[] = []; // Export core data from database const tables = [ 'projects', 'agile_sprints', 'agile_stories', 'agile_epics', 'kanban_boards', 'documents', 'memories', 'business_plans', 'market_analyses', 'competitor_analyses', 'financial_projections', 'startup_assessments', 'pitch_decks', 'startup_metrics', 'business_reviews', 'deployment_configs', 'deployment_environments', 'deployments', 'security_scans', 'security_secrets' ]; for (const table of tables) { try { const result = await context.db.all(`SELECT * FROM ${table}`, []); if (result.success && result.data && result.data.length > 0) { const moduleData = { table, count: result.data.length, data: result.data, exportedAt: new Date(now).toISOString() }; const filePath = path.join(exportDir, `${table}.json`); await fs.writeFile(filePath, JSON.stringify(moduleData, null, 2), 'utf-8'); totalExported += result.data.length; modules.push(table); } } catch (error) { errors.push(`Failed to export ${table}: ${error instanceof Error ? error.message : 'Unknown error'}`); } } // Export configuration if requested if (input.includeConfig) { try { const configResult = await context.db.all( 'SELECT * FROM atlas_metadata', [] ); if (configResult.success && configResult.data) { const configPath = path.join(exportDir, 'config.json'); await fs.writeFile( configPath, JSON.stringify({ metadata: configResult.data, exportConfig: input, exportedAt: new Date(now).toISOString() }, null, 2), 'utf-8' ); } } catch (error) { errors.push(`Failed to export config: ${error instanceof Error ? error.message : 'Unknown error'}`); } } // Create export manifest const manifest = { exportId, version: '1.0.0', exportedAt: new Date(now).toISOString(), format: input.format, totalItems: totalExported, modules, includeConfig: input.includeConfig, includeMedia: input.includeMedia, errors: errors.length > 0 ? errors : undefined }; const manifestPath = path.join(exportDir, 'manifest.json'); await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf-8'); // Store export record const result = await context.db.run( `INSERT INTO data_exports (id, project_id, export_path, format, total_items, modules, include_config, include_media, status, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ exportId, context.projectId || 'default', exportDir, input.format || 'json', totalExported, JSON.stringify(modules), input.includeConfig !== false, input.includeMedia === true, errors.length > 0 ? 'completed_with_errors' : 'completed', now ] ); if (!result.success) { errors.push('Failed to record export in database'); } return createSuccessResult({ export: { id: exportId, path: exportDir, totalItems: totalExported, modules, format: input.format || 'json', errors: errors.length > 0 ? errors : undefined }, message: `Successfully exported ${totalExported} items from ${modules.length} modules`, filePath: exportDir }); } catch (error) { return createErrorResult({ code: 'EXPORT_ERROR', message: `Failed to export data: ${error instanceof Error ? error.message : 'Unknown error'}`, category: 'execution' }); } } }); /** * Import data from external source */ const importDataTool = createTool<ImportDataInput, any>({ name: 'import_data', description: 'Import data from Atlas export or external source', category: 'data-management', inputSchema: { type: 'object', properties: { sourcePath: { type: 'string', description: 'Path to data source (directory or file)', minLength: 1 }, createBackup: { type: 'boolean', default: true, description: 'Create backup before importing' }, mergeStrategy: { type: 'string', enum: ['overwrite', 'skip_existing', 'merge'], default: 'skip_existing', description: 'How to handle existing data' }, modules: { type: 'array', items: { type: 'string' }, description: 'Specific modules to import (empty for all)' } }, required: ['sourcePath'], additionalProperties: false } as JSONSchema7, async execute(input: ImportDataInput, context: RequestContext) { try { const importId = randomUUID(); const now = Date.now(); let backupPath: string | undefined; // Validate source path exists try { await fs.access(input.sourcePath); } catch { return createErrorResult({ code: 'SOURCE_NOT_FOUND', message: `Source path does not exist: ${input.sourcePath}`, category: 'validation' }); } // Create backup if requested if (input.createBackup) { try { const timestamp = new Date(now).toISOString().replace(/[:.]/g, '-'); backupPath = path.join(process.cwd(), `atlas-backup-${timestamp}`); await fs.mkdir(backupPath, { recursive: true }); // Export current data as backup const backupResult = await exportAllDataTool.execute({ includeConfig: true, format: 'json' }, context); if (!backupResult.success) { return createErrorResult({ code: 'BACKUP_FAILED', message: 'Failed to create backup before import', category: 'system' }); } } catch (error) { return createErrorResult({ code: 'BACKUP_ERROR', message: `Backup creation failed: ${error instanceof Error ? error.message : 'Unknown error'}`, category: 'system' }); } } let importedItems = 0; const importedModules: string[] = []; const errors: string[] = []; // Check if source is a directory or file const sourceStats = await fs.stat(input.sourcePath); if (sourceStats.isDirectory()) { // Import from Atlas export directory try { const manifestPath = path.join(input.sourcePath, 'manifest.json'); const manifestContent = await fs.readFile(manifestPath, 'utf-8'); const manifest = JSON.parse(manifestContent); // Read and import each module for (const module of manifest.modules) { if (input.modules && !input.modules.includes(module)) { continue; // Skip if not in selected modules } try { const modulePath = path.join(input.sourcePath, `${module}.json`); const moduleContent = await fs.readFile(modulePath, 'utf-8'); const moduleData = JSON.parse(moduleContent); // Import data based on merge strategy for (const item of moduleData.data) { try { const columns = Object.keys(item).join(', '); const placeholders = Object.keys(item).map(() => '?').join(', '); const values = Object.values(item); let query: string; switch (input.mergeStrategy) { case 'overwrite': query = `INSERT OR REPLACE INTO ${module} (${columns}) VALUES (${placeholders})`; break; case 'skip_existing': query = `INSERT OR IGNORE INTO ${module} (${columns}) VALUES (${placeholders})`; break; case 'merge': // For merge, we need to check if record exists and update query = `INSERT OR REPLACE INTO ${module} (${columns}) VALUES (${placeholders})`; break; default: query = `INSERT OR IGNORE INTO ${module} (${columns}) VALUES (${placeholders})`; } const result = await context.db.run(query, values); if (result.success) { importedItems++; } } catch (error) { errors.push(`Failed to import item in ${module}: ${error instanceof Error ? error.message : 'Unknown error'}`); } } importedModules.push(module); } catch (error) { errors.push(`Failed to import module ${module}: ${error instanceof Error ? error.message : 'Unknown error'}`); } } } catch (error) { return createErrorResult({ code: 'MANIFEST_ERROR', message: `Failed to read import manifest: ${error instanceof Error ? error.message : 'Unknown error'}`, category: 'validation' }); } } else { // Import from single file (JSON, CSV, etc.) try { const content = await fs.readFile(input.sourcePath, 'utf-8'); const data = JSON.parse(content); // Determine target table/module const targetModule = input.modules?.[0] || path.basename(input.sourcePath, path.extname(input.sourcePath)); // Import the data if (Array.isArray(data)) { for (const item of data) { try { const columns = Object.keys(item).join(', '); const placeholders = Object.keys(item).map(() => '?').join(', '); const values = Object.values(item); const query = `INSERT OR IGNORE INTO ${targetModule} (${columns}) VALUES (${placeholders})`; const result = await context.db.run(query, values); if (result.success) { importedItems++; } } catch (error) { errors.push(`Failed to import item: ${error instanceof Error ? error.message : 'Unknown error'}`); } } importedModules.push(targetModule); } } catch (error) { return createErrorResult({ code: 'FILE_IMPORT_ERROR', message: `Failed to import from file: ${error instanceof Error ? error.message : 'Unknown error'}`, category: 'execution' }); } } // Record import const recordResult = await context.db.run( `INSERT INTO data_imports (id, project_id, source_path, merge_strategy, imported_items, imported_modules, backup_path, status, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ importId, context.projectId || 'default', input.sourcePath, input.mergeStrategy || 'skip_existing', importedItems, JSON.stringify(importedModules), backupPath, errors.length > 0 ? 'completed_with_errors' : 'completed', now ] ); return createSuccessResult({ import: { id: importId, importedItems, importedModules, backupPath, errors: errors.length > 0 ? errors : undefined }, message: `Successfully imported ${importedItems} items from ${importedModules.length} modules` }); } catch (error) { return createErrorResult({ code: 'IMPORT_ERROR', message: `Failed to import data: ${error instanceof Error ? error.message : 'Unknown error'}`, category: 'execution' }); } } }); /** * Sync data to project directory */ const syncToProjectTool = createTool<SyncToProjectInput, any>({ name: 'sync_to_project', description: 'Sync Atlas data to project directory for version control', category: 'data-management', readOnly: true, inputSchema: { type: 'object', properties: { modules: { type: 'array', items: { type: 'string' }, default: ['agile_stories', 'kanban_boards', 'documents'], description: 'Modules to sync' }, format: { type: 'string', enum: ['json', 'markdown'], default: 'json', description: 'Output format' }, destination: { type: 'string', default: '.atlas-sync', description: 'Destination directory in project' } }, additionalProperties: false } as JSONSchema7, async execute(input: SyncToProjectInput, context: RequestContext) { try { const syncDir = path.join(process.cwd(), input.destination || '.atlas-sync'); await fs.mkdir(syncDir, { recursive: true }); const syncedFiles: string[] = []; const errors: string[] = []; let totalItems = 0; const modules = input.modules || ['agile_stories', 'kanban_boards', 'documents']; for (const module of modules) { try { const result = await context.db.all(`SELECT * FROM ${module}`, []); if (result.success && result.data && result.data.length > 0) { const fileName = `${module}.${input.format || 'json'}`; const filePath = path.join(syncDir, fileName); if (input.format === 'markdown') { const markdown = generateMarkdownFromData(module, result.data); await fs.writeFile(filePath, markdown, 'utf-8'); } else { const syncData = { module, count: result.data.length, data: result.data, syncedAt: new Date().toISOString() }; await fs.writeFile(filePath, JSON.stringify(syncData, null, 2), 'utf-8'); } syncedFiles.push(fileName); totalItems += result.data.length; } } catch (error) { errors.push(`Failed to sync ${module}: ${error instanceof Error ? error.message : 'Unknown error'}`); } } // Create sync manifest const manifest = { syncedAt: new Date().toISOString(), format: input.format || 'json', modules, totalItems, files: syncedFiles }; const manifestPath = path.join(syncDir, 'sync-manifest.json'); await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf-8'); return createSuccessResult({ sync: { destination: syncDir, files: syncedFiles, totalItems, format: input.format || 'json', errors: errors.length > 0 ? errors : undefined }, message: `Synced ${totalItems} items from ${modules.length} modules to ${syncDir}` }); } catch (error) { return createErrorResult({ code: 'SYNC_ERROR', message: `Failed to sync to project: ${error instanceof Error ? error.message : 'Unknown error'}`, category: 'execution' }); } } }); /** * Configure storage settings */ const configureStorageTool = createTool<ConfigureStorageInput, any>({ name: 'configure_storage', description: 'Configure Atlas data storage settings', category: 'data-management', inputSchema: { type: 'object', properties: { mode: { type: 'string', enum: ['project-local', 'user-home', 'custom'], description: 'Storage location mode' }, customPath: { type: 'string', description: 'Custom storage path (required for custom mode)', minLength: 1 }, isolation: { type: 'string', enum: ['project', 'user', 'global'], default: 'project', description: 'Data isolation level' } }, required: ['mode'], additionalProperties: false } as JSONSchema7, async execute(input: ConfigureStorageInput, context: RequestContext) { try { if (input.mode === 'custom' && !input.customPath) { return createErrorResult({ code: 'INVALID_CONFIG', message: 'Custom path is required when mode is "custom"', category: 'validation' }); } // Update storage configuration in database const configId = randomUUID(); const now = Date.now(); const result = await context.db.run( `INSERT OR REPLACE INTO storage_configs (id, project_id, mode, custom_path, isolation, active, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, [ configId, context.projectId || 'default', input.mode, input.customPath || null, input.isolation || 'project', true, now, now ] ); if (!result.success) { return createErrorResult({ code: 'CONFIG_SAVE_ERROR', message: 'Failed to save storage configuration', category: 'system' }); } // Deactivate other configs for this project await context.db.run( `UPDATE storage_configs SET active = FALSE WHERE project_id = ? AND id != ?`, [context.projectId || 'default', configId] ); return createSuccessResult({ config: { id: configId, mode: input.mode, customPath: input.customPath, isolation: input.isolation || 'project' }, message: `Storage configuration updated to ${input.mode} mode` }); } catch (error) { return createErrorResult({ code: 'CONFIG_ERROR', message: `Failed to configure storage: ${error instanceof Error ? error.message : 'Unknown error'}`, category: 'execution' }); } } }); /** * Export module-specific data */ const exportModuleDataTool = createTool<ExportModuleDataInput, any>({ name: 'export_module_data', description: 'Export data from a specific module with filtering options', category: 'data-management', readOnly: true, inputSchema: { type: 'object', properties: { moduleName: { type: 'string', description: 'Name of the module/table to export', minLength: 1 }, format: { type: 'string', enum: ['json', 'markdown', 'csv', 'sql'], default: 'json', description: 'Export format' }, includeHistory: { type: 'boolean', default: false, description: 'Include historical data and changes' }, dateRange: { type: 'object', properties: { start: { type: 'string', format: 'date-time' }, end: { type: 'string', format: 'date-time' } }, description: 'Filter by date range' } }, required: ['moduleName'], additionalProperties: false } as JSONSchema7, async execute(input: ExportModuleDataInput, context: RequestContext) { try { let query = `SELECT * FROM ${input.moduleName}`; const params: any[] = []; // Add date filtering if provided if (input.dateRange) { const conditions: string[] = []; if (input.dateRange.start) { conditions.push('created_at >= ?'); params.push(new Date(input.dateRange.start).getTime()); } if (input.dateRange.end) { conditions.push('created_at <= ?'); params.push(new Date(input.dateRange.end).getTime()); } if (conditions.length > 0) { query += ` WHERE ${conditions.join(' AND ')}`; } } const result = await context.db.all(query, params); if (!result.success || !result.data) { return createErrorResult({ code: 'QUERY_ERROR', message: `Failed to query module ${input.moduleName}`, category: 'execution' }); } const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const fileName = `${input.moduleName}-export-${timestamp}`; const exportDir = path.join(process.cwd(), 'exports'); await fs.mkdir(exportDir, { recursive: true }); let content: string; let fileExtension: string; switch (input.format) { case 'csv': content = generateCSVFromData(result.data); fileExtension = 'csv'; break; case 'markdown': content = generateMarkdownFromData(input.moduleName, result.data); fileExtension = 'md'; break; case 'sql': content = generateSQLFromData(input.moduleName, result.data); fileExtension = 'sql'; break; default: content = JSON.stringify({ module: input.moduleName, exportedAt: new Date().toISOString(), count: result.data.length, data: result.data }, null, 2); fileExtension = 'json'; } const filePath = path.join(exportDir, `${fileName}.${fileExtension}`); await fs.writeFile(filePath, content, 'utf-8'); // Record export await context.db.run( `INSERT INTO module_exports (id, project_id, module_name, format, file_path, item_count, include_history, date_range, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ randomUUID(), context.projectId || 'default', input.moduleName, input.format || 'json', filePath, result.data.length, input.includeHistory === true, JSON.stringify(input.dateRange || null), Date.now() ] ); return createSuccessResult({ export: { module: input.moduleName, format: input.format || 'json', filePath, itemCount: result.data.length, includeHistory: input.includeHistory, dateRange: input.dateRange }, message: `Exported ${result.data.length} items from ${input.moduleName}`, filePath }); } catch (error) { return createErrorResult({ code: 'MODULE_EXPORT_ERROR', message: `Failed to export module data: ${error instanceof Error ? error.message : 'Unknown error'}`, category: 'execution' }); } } }); // Helper functions function generateMarkdownFromData(module: string, data: any[]): string { let markdown = `# ${module.charAt(0).toUpperCase() + module.slice(1)} Export\n\n`; markdown += `Exported: ${new Date().toISOString()}\n`; markdown += `Total Items: ${data.length}\n\n`; if (data.length === 0) { markdown += 'No data to display.\n'; return markdown; } // Create table from data const headers = Object.keys(data[0]); markdown += `| ${headers.join(' | ')} |\n`; markdown += `| ${headers.map(() => '---').join(' | ')} |\n`; data.forEach(item => { const row = headers.map(header => { let value = item[header]; if (value === null || value === undefined) return ''; if (typeof value === 'object') return JSON.stringify(value); return String(value).replace(/\|/g, '\\|'); }); markdown += `| ${row.join(' | ')} |\n`; }); return markdown; } function generateCSVFromData(data: any[]): string { if (data.length === 0) return ''; const headers = Object.keys(data[0]); let csv = headers.map(h => `"${h}"`).join(',') + '\n'; data.forEach(item => { const row = headers.map(header => { let value = item[header]; if (value === null || value === undefined) return '""'; if (typeof value === 'object') value = JSON.stringify(value); return `"${String(value).replace(/"/g, '""')}"`; }); csv += row.join(',') + '\n'; }); return csv; } function generateSQLFromData(tableName: string, data: any[]): string { if (data.length === 0) return `-- No data in ${tableName}\n`; let sql = `-- Data export for ${tableName}\n`; sql += `-- Generated: ${new Date().toISOString()}\n\n`; data.forEach(item => { const columns = Object.keys(item).join(', '); const values = Object.values(item).map(v => { if (v === null || v === undefined) return 'NULL'; if (typeof v === 'string') return `'${v.replace(/'/g, "''")}'`; if (typeof v === 'object') return `'${JSON.stringify(v).replace(/'/g, "''")}'`; return String(v); }).join(', '); sql += `INSERT INTO ${tableName} (${columns}) VALUES (${values});\n`; }); return sql; } /** * Setup data management tools */ export async function setupDataManagementTools(): Promise<ToolRegistration> { return { module: 'data-management', tools: [ exportAllDataTool, importDataTool, syncToProjectTool, configureStorageTool, exportModuleDataTool ] }; }