@boundless-oss/atlas/src/storage/data-migration.ts

Atlas - MCP Server for comprehensive startup project management

/**
 * Data Migration System for Atlas
 * 
 * Handles migration from pre-SQLite JSON-based data to SQLite database
 * Ensures data integrity and provides rollback capabilities
 */

import { promises as fs } from 'fs';
import path from 'path';
import { randomUUID } from 'crypto';
import { getSQLiteManager } from './sqlite-manager.js';

export interface MigrationResult {
  success: boolean;
  migratedFiles: string[];
  errors: string[];
  backupPath?: string;
  summary: {
    epics: number;
    stories: number;
    sprints: number;
    other: number;
  };
}

export interface LegacyEpic {
  id: string;
  title: string;
  description: string;
  status: string;
  priority: string;
  goals: string[];
  createdAt: string;
  updatedAt: string;
  storyIds: string[];
  stories: any[];
  progress: number;
}

export interface LegacyStory {
  id: string;
  title: string;
  description: string;
  status: string;
  priority: string;
  storyPoints?: number;
  epicId?: string;
  sprintId?: string;
  acceptanceCriteria?: any[];
  assignee?: string;
  createdAt: string;
  updatedAt: string;
  tags?: string[];
}

export interface LegacySprint {
  id: string;
  name: string;
  goal: string;
  status: string;
  startDate: string;
  endDate: string;
  capacity?: number;
  storyIds: string[];
  createdAt: string;
  updatedAt: string;
}

export class DataMigrationManager {
  private atlasDir: string;
  private dataDir: string;
  private backupDir: string;

  constructor() {
    this.atlasDir = '.atlas';
    this.dataDir = path.join(this.atlasDir, 'data');
    this.backupDir = path.join(this.atlasDir, 'backups');
  }

  /**
   * Check if legacy data exists that needs migration
   */
  async detectLegacyData(): Promise<boolean> {
    try {
      const agileDir = path.join(this.dataDir, 'agile');
      const files = ['epics.json', 'stories.json', 'sprints.json'];
      
      for (const file of files) {
        const filePath = path.join(agileDir, file);
        try {
          await fs.access(filePath);
          return true; // Found at least one legacy file
        } catch {
          // File doesn't exist, continue
        }
      }
      
      return false;
    } catch {
      return false;
    }
  }

  /**
   * Create backup of existing data before migration
   */
  async createBackup(): Promise<string> {
    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
    const backupPath = path.join(this.backupDir, `pre-migration-${timestamp}`);
    
    await fs.mkdir(backupPath, { recursive: true });
    
    // Copy entire data directory
    await this.copyDirectory(this.dataDir, path.join(backupPath, 'data'));
    
    console.error(`✅ Backup created at: ${backupPath}`);
    return backupPath;
  }

  /**
   * Migrate all legacy data to SQLite
   */
  async migrateData(): Promise<MigrationResult> {
    const result: MigrationResult = {
      success: false,
      migratedFiles: [],
      errors: [],
      summary: { epics: 0, stories: 0, sprints: 0, other: 0 }
    };

    try {
      console.error('🔄 Starting data migration...');
      
      // Create backup first
      result.backupPath = await this.createBackup();
      
      // Get database connection
      const db = getSQLiteManager();
      
      // Ensure database is initialized
      if (!db.isReady()) {
        throw new Error('Database not ready for migration');
      }

      // Migrate agile data
      await this.migrateAgileData(result);
      
      // Migrate other data modules
      await this.migrateOtherData(result);
      
      result.success = result.errors.length === 0;
      
      if (result.success) {
        console.error('✅ Data migration completed successfully');
        console.error(`📊 Migrated: ${result.summary.epics} epics, ${result.summary.stories} stories, ${result.summary.sprints} sprints`);
      } else {
        console.warn('⚠️ Migration completed with errors:', result.errors);
      }
      
    } catch (error) {
      result.errors.push(`Migration failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
      console.error('❌ Migration failed:', error);
    }

    return result;
  }

  /**
   * Migrate agile data (epics, stories, sprints)
   */
  private async migrateAgileData(result: MigrationResult): Promise<void> {
    const agileDir = path.join(this.dataDir, 'agile');
    const db = getSQLiteManager();

    // Migrate epics
    try {
      const epicsPath = path.join(agileDir, 'epics.json');
      const epicsData = await this.readJsonFile<LegacyEpic[]>(epicsPath);
      
      if (epicsData && epicsData.length > 0) {
        console.error(`📋 Migrating ${epicsData.length} epics...`);
        
        for (const epic of epicsData) {
          await db.run(
            `INSERT OR REPLACE INTO agile_epics 
             (id, title, description, status, priority, goals, created_at, updated_at, story_ids, progress)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
            [
              epic.id,
              epic.title,
              epic.description,
              epic.status,
              epic.priority,
              JSON.stringify(epic.goals),
              new Date(epic.createdAt).getTime(),
              new Date(epic.updatedAt).getTime(),
              JSON.stringify(epic.storyIds),
              epic.progress
            ]
          );
        }
        
        result.summary.epics = epicsData.length;
        result.migratedFiles.push(epicsPath);
      }
    } catch (error) {
      result.errors.push(`Epic migration failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    // Migrate stories
    try {
      const storiesPath = path.join(agileDir, 'stories.json');
      const storiesData = await this.readJsonFile<LegacyStory[]>(storiesPath);
      
      if (storiesData && storiesData.length > 0) {
        console.log(`📝 Migrating ${storiesData.length} stories...`);
        
        for (const story of storiesData) {
          await db.run(
            `INSERT OR REPLACE INTO agile_stories 
             (id, title, description, status, priority, story_points, epic_id, sprint_id, 
              acceptance_criteria, assignee, created_at, updated_at, tags)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
            [
              story.id,
              story.title,
              story.description,
              story.status,
              story.priority,
              story.storyPoints || null,
              story.epicId || null,
              story.sprintId || null,
              JSON.stringify(story.acceptanceCriteria || []),
              story.assignee || null,
              new Date(story.createdAt).getTime(),
              new Date(story.updatedAt).getTime(),
              JSON.stringify(story.tags || [])
            ]
          );
        }
        
        result.summary.stories = storiesData.length;
        result.migratedFiles.push(storiesPath);
      }
    } catch (error) {
      result.errors.push(`Story migration failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    // Migrate sprints
    try {
      const sprintsPath = path.join(agileDir, 'sprints.json');
      const sprintsData = await this.readJsonFile<LegacySprint[]>(sprintsPath);
      
      if (sprintsData && sprintsData.length > 0) {
        console.log(`🏃 Migrating ${sprintsData.length} sprints...`);
        
        for (const sprint of sprintsData) {
          await db.run(
            `INSERT OR REPLACE INTO agile_sprints 
             (id, name, goal, status, start_date, end_date, capacity, story_ids, created_at, updated_at)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
            [
              sprint.id,
              sprint.name,
              sprint.goal,
              sprint.status,
              new Date(sprint.startDate).getTime(),
              new Date(sprint.endDate).getTime(),
              sprint.capacity || null,
              JSON.stringify(sprint.storyIds),
              new Date(sprint.createdAt).getTime(),
              new Date(sprint.updatedAt).getTime()
            ]
          );
        }
        
        result.summary.sprints = sprintsData.length;
        result.migratedFiles.push(sprintsPath);
      }
    } catch (error) {
      result.errors.push(`Sprint migration failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }
  }

  /**
   * Migrate other data modules (workspace, business, etc.)
   */
  private async migrateOtherData(result: MigrationResult): Promise<void> {
    // This can be extended for other data types as needed
    console.log('📂 Checking other data modules...');
    
    const otherModules = ['workspace', 'business', 'development'];
    
    for (const module of otherModules) {
      try {
        const moduleDir = path.join(this.dataDir, module);
        const files = await fs.readdir(moduleDir).catch(() => []);
        
        for (const file of files) {
          if (file.endsWith('.json')) {
            const filePath = path.join(moduleDir, file);
            result.migratedFiles.push(filePath);
            result.summary.other++;
          }
        }
      } catch {
        // Module directory doesn't exist, skip
      }
    }
  }

  /**
   * Archive migrated files (move to archive directory)
   */
  async archiveMigratedFiles(migratedFiles: string[]): Promise<void> {
    const archiveDir = path.join(this.atlasDir, 'archive');
    await fs.mkdir(archiveDir, { recursive: true });
    
    for (const filePath of migratedFiles) {
      try {
        const fileName = path.basename(filePath);
        const archivePath = path.join(archiveDir, fileName);
        await fs.rename(filePath, archivePath);
        console.log(`📦 Archived: ${fileName}`);
      } catch (error) {
        console.warn(`⚠️ Failed to archive ${filePath}:`, error);
      }
    }
  }

  /**
   * Read and parse JSON file
   */
  private async readJsonFile<T>(filePath: string): Promise<T | null> {
    try {
      const content = await fs.readFile(filePath, 'utf-8');
      return JSON.parse(content) as T;
    } catch {
      return null;
    }
  }

  /**
   * Copy directory recursively
   */
  private async copyDirectory(src: string, dest: string): Promise<void> {
    await fs.mkdir(dest, { recursive: true });
    const entries = await fs.readdir(src, { withFileTypes: true });
    
    for (const entry of entries) {
      const srcPath = path.join(src, entry.name);
      const destPath = path.join(dest, entry.name);
      
      if (entry.isDirectory()) {
        await this.copyDirectory(srcPath, destPath);
      } else {
        await fs.copyFile(srcPath, destPath);
      }
    }
  }
}

/**
 * Auto-migrate data on server startup if needed
 */
export async function autoMigrateOnStartup(): Promise<void> {
  const migrationManager = new DataMigrationManager();
  
  if (await migrationManager.detectLegacyData()) {
    console.log('🔍 Legacy data detected, starting automatic migration...');
    
    const result = await migrationManager.migrateData();
    
    if (result.success) {
      // Archive migrated files to prevent re-migration
      await migrationManager.archiveMigratedFiles(result.migratedFiles);
      console.log('✅ Legacy data migration completed successfully');
    } else {
      console.error('❌ Migration failed. Check backup at:', result.backupPath);
      throw new Error('Data migration failed: ' + result.errors.join(', '));
    }
  }
}