@boundless-oss/atlas/src/storage/migration.ts

Atlas - MCP Server for comprehensive startup project management

import { promises as fs } from 'fs';
import path from 'path';
import { SQLiteManager } from './sqlite-manager.js';
import { randomUUID } from 'crypto';

export interface MigrationResult {
  success: boolean;
  error?: string;
  migratedItems: number;
  errors: string[];
  duration: number;
  backupPath?: string;
}

export interface MigrationStatus {
  needsMigration: boolean;
  isRequired: boolean;
  hasLegacyData: boolean;
  backupExists: boolean;
  lastMigration?: number;
}

/**
 * Data Migration Manager
 * Handles migration from file-based JSON storage to SQLite
 */
export class DataMigration {
  private db: SQLiteManager;
  private legacyDataPath: string;
  private backupPath: string;
  private useInternalMethods: boolean;

  constructor(db: SQLiteManager, dataPath = '.atlas', useInternalMethods = false) {
    this.db = db;
    this.legacyDataPath = path.join(dataPath, 'data');
    this.backupPath = path.join(dataPath, 'backup');
    this.useInternalMethods = useInternalMethods;
  }

  /**
   * Get data using appropriate method (internal during initialization, public otherwise)
   */
  private async dbGet<T>(sql: string, params: any[] = []): Promise<{ success: boolean; data?: T; error?: string }> {
    if (this.useInternalMethods) {
      return this.db.getForMigration<T>(sql, params);
    } else {
      return this.db.get<T>(sql, params);
    }
  }

  /**
   * Run query using appropriate method (internal during initialization, public otherwise)
   */
  private async dbRun(sql: string, params: any[] = []): Promise<{ success: boolean; error?: string }> {
    if (this.useInternalMethods) {
      const result = await this.db.runForMigration(sql, params);
      return { success: result.success, error: result.error };
    } else {
      const result = await this.db.run(sql, params);
      return { success: result.success, error: result.error };
    }
  }

  /**
   * Check if migration is needed
   */
  async checkMigrationStatus(): Promise<MigrationStatus> {
    try {
      // Check if legacy data exists
      const hasLegacyData = await this.hasLegacyData();
      
      // Check if backup exists
      const backupExists = await this.pathExists(this.backupPath);
      
      // Check if migration was already completed
      const migrationRecord = await this.dbGet<{ value: string }>(
        'SELECT value FROM atlas_metadata WHERE key = ?',
        ['migration_status']
      );
      
      const lastMigrationRecord = await this.dbGet<{ value: string }>(
        'SELECT value FROM atlas_metadata WHERE key = ?',
        ['last_migration_timestamp']
      );

      // Check if database actually has migrated data
      const hasMigratedData = await this.hasDataInDatabase();
      
      const isRequired = hasLegacyData && (!hasMigratedData || migrationRecord.data?.value !== 'completed');
      const lastMigration = lastMigrationRecord.data?.value ? 
        parseInt(lastMigrationRecord.data.value) : undefined;

      return {
        needsMigration: isRequired,
        isRequired,
        hasLegacyData,
        backupExists,
        lastMigration
      };
    } catch (error) {
      console.error('Failed to check migration status:', error);
      return {
        needsMigration: false,
        isRequired: false,
        hasLegacyData: false,
        backupExists: false
      };
    }
  }

  /**
   * Perform the migration
   */
  async migrate(): Promise<MigrationResult> {
    const startTime = Date.now();
    const errors: string[] = [];
    let migratedItems = 0;
    let backupPath: string | undefined;

    try {
      console.error('🔄 Starting data migration from JSON files to SQLite...');

      // Disable foreign key constraints during migration
      await this.dbRun('PRAGMA foreign_keys = OFF');
      console.error('🔧 Disabled foreign key constraints for migration');

      // Create backup first
      backupPath = await this.createBackup();
      console.error(`📦 Backup created at: ${backupPath}`);

      // Migrate only active modules (skip deprecated/removed modules)
      // Core project data
      migratedItems += await this.migrateProjectData(errors);
      
      // Active 12-factor modules
      migratedItems += await this.migrateAgileData(errors);
      migratedItems += await this.migrateKanbanData(errors);
      migratedItems += await this.migrateDocumentationData(errors);
      migratedItems += await this.migrateMemoryData(errors);
      migratedItems += await this.migrateADRData(errors);
      migratedItems += await this.migrateDevelopmentData(errors);
      migratedItems += await this.migrateWorkspaceData(errors);
      migratedItems += await this.migrateProcessAutomationData(errors);
      migratedItems += await this.migrateProductRequirementsData(errors);
      migratedItems += await this.migrateDataManagementData(errors);
      migratedItems += await this.migrateRAGData(errors);
      
      // Skip deprecated/removed modules:
      console.error('⚠️  Skipping deprecated modules: approval-wrapper, deployment-management, deprecation, interaction-framework, memory, repo-setup, smart-estimation, workflow-recipes');
      console.error('⚠️  Skipping inactive modules: business-guidance, code-analysis, error-analysis, issue-tracking, product-roadmap, testing-framework, performance-monitoring, security');

      // Re-enable foreign key constraints
      await this.dbRun('PRAGMA foreign_keys = ON');
      console.log('🔧 Re-enabled foreign key constraints');

      // Update migration status
      await this.updateMigrationStatus('completed');

      const duration = Date.now() - startTime;
      console.log(`✅ Migration completed in ${duration}ms. Migrated ${migratedItems} items.`);

      return {
        success: true,
        migratedItems,
        errors,
        duration,
        backupPath
      };

    } catch (error) {
      // Re-enable foreign key constraints even on error
      try {
        await this.dbRun('PRAGMA foreign_keys = ON');
        console.log('🔧 Re-enabled foreign key constraints after error');
      } catch (pragmaError) {
        console.warn('⚠️ Failed to re-enable foreign key constraints:', pragmaError);
      }

      const duration = Date.now() - startTime;
      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
      errors.push(`Migration failed: ${errorMessage}`);

      console.error('❌ Migration failed:', error);

      return {
        success: false,
        migratedItems,
        errors,
        duration,
        backupPath
      };
    }
  }

  /**
   * Create backup of existing data
   */
  private async createBackup(): Promise<string> {
    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
    const backupDir = path.join(this.backupPath, `backup-${timestamp}`);
    
    await fs.mkdir(backupDir, { recursive: true });

    // Copy all JSON files to backup
    const files = await this.findJsonFiles(this.legacyDataPath);
    
    for (const file of files) {
      const relativePath = path.relative(this.legacyDataPath, file);
      const backupFile = path.join(backupDir, relativePath);
      
      await fs.mkdir(path.dirname(backupFile), { recursive: true });
      await fs.copyFile(file, backupFile);
    }

    return backupDir;
  }

  /**
   * Migrate agile management data
   */
  private async migrateAgileData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      // Ensure default project exists to avoid foreign key constraints
      await this.ensureDefaultProject();
      console.log('✅ Default project ensured for agile data');

      // Migrate sprints
      const sprintsFile = path.join(this.legacyDataPath, 'agile', 'sprints.json');
      if (await this.pathExists(sprintsFile)) {
        const sprints = await this.readJsonFile(sprintsFile, errors);
        if (Array.isArray(sprints)) {
          for (const sprint of sprints) {
            await this.migrateSprint(sprint);
            count++;
          }
        }
      }

      // Migrate stories
      const storiesFile = path.join(this.legacyDataPath, 'agile', 'stories.json');
      if (await this.pathExists(storiesFile)) {
        const stories = await this.readJsonFile(storiesFile, errors);
        if (Array.isArray(stories)) {
          for (const story of stories) {
            await this.migrateStory(story);
            count++;
          }
        }
      }

      // Migrate epics
      const epicsFile = path.join(this.legacyDataPath, 'agile', 'epics.json');
      if (await this.pathExists(epicsFile)) {
        const epics = await this.readJsonFile(epicsFile, errors);
        if (Array.isArray(epics)) {
          for (const epic of epics) {
            await this.migrateEpic(epic);
            count++;
          }
        }
      }

    } catch (error) {
      errors.push(`Agile data migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate kanban data
   */
  private async migrateKanbanData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      const boardsFile = path.join(this.legacyDataPath, 'kanban', 'boards.json');
      if (await this.pathExists(boardsFile)) {
        const boards = await this.readJsonFile(boardsFile);
        if (Array.isArray(boards)) {
          for (const board of boards) {
            await this.migrateKanbanBoard(board);
            count++;
          }
        }
      }
    } catch (error) {
      errors.push(`Kanban data migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate documentation data
   */
  private async migrateDocumentationData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      const docsFile = path.join(this.legacyDataPath, 'documentation', 'documents.json');
      if (await this.pathExists(docsFile)) {
        const docs = await this.readJsonFile(docsFile);
        if (Array.isArray(docs)) {
          for (const doc of docs) {
            await this.migrateDocument(doc);
            count++;
          }
        }
      }
    } catch (error) {
      errors.push(`Documentation migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate business guidance data
   */
  private async migrateBusinessData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      // Migrate business plans
      const businessFile = path.join(this.legacyDataPath, 'business', 'plans.json');
      if (await this.pathExists(businessFile)) {
        const plans = await this.readJsonFile(businessFile);
        if (Array.isArray(plans)) {
          for (const plan of plans) {
            await this.migrateBusinessPlan(plan);
            count++;
          }
        }
      }

      // Migrate market analyses
      const marketAnalysesFile = path.join(this.legacyDataPath, 'business', 'market-analyses.json');
      if (await this.pathExists(marketAnalysesFile)) {
        const analyses = await this.readJsonFile(marketAnalysesFile);
        if (Array.isArray(analyses)) {
          for (const analysis of analyses) {
            await this.migrateMarketAnalysis(analysis);
            count++;
          }
        }
      }

      // Migrate competitor analyses
      const competitorFile = path.join(this.legacyDataPath, 'business', 'competitor-analyses.json');
      if (await this.pathExists(competitorFile)) {
        const analyses = await this.readJsonFile(competitorFile);
        if (Array.isArray(analyses)) {
          for (const analysis of analyses) {
            await this.migrateCompetitorAnalysis(analysis);
            count++;
          }
        }
      }

      // Migrate financial projections
      const financialFile = path.join(this.legacyDataPath, 'business', 'financial-projections.json');
      if (await this.pathExists(financialFile)) {
        const projections = await this.readJsonFile(financialFile);
        if (Array.isArray(projections)) {
          for (const projection of projections) {
            await this.migrateFinancialProjection(projection);
            count++;
          }
        }
      }

      // Migrate startup assessments
      const assessmentFile = path.join(this.legacyDataPath, 'business', 'startup-assessments.json');
      if (await this.pathExists(assessmentFile)) {
        const assessments = await this.readJsonFile(assessmentFile);
        if (Array.isArray(assessments)) {
          for (const assessment of assessments) {
            await this.migrateStartupAssessment(assessment);
            count++;
          }
        }
      }

      // Migrate pitch decks
      const pitchDeckFile = path.join(this.legacyDataPath, 'business', 'pitch-decks.json');
      if (await this.pathExists(pitchDeckFile)) {
        const decks = await this.readJsonFile(pitchDeckFile);
        if (Array.isArray(decks)) {
          for (const deck of decks) {
            await this.migratePitchDeck(deck);
            count++;
          }
        }
      }

      // Migrate startup metrics
      const metricsFile = path.join(this.legacyDataPath, 'business', 'startup-metrics.json');
      if (await this.pathExists(metricsFile)) {
        const metrics = await this.readJsonFile(metricsFile);
        if (Array.isArray(metrics)) {
          for (const metric of metrics) {
            await this.migrateStartupMetrics(metric);
            count++;
          }
        }
      }

      // Migrate business reviews
      const reviewFile = path.join(this.legacyDataPath, 'business', 'business-reviews.json');
      if (await this.pathExists(reviewFile)) {
        const reviews = await this.readJsonFile(reviewFile);
        if (Array.isArray(reviews)) {
          for (const review of reviews) {
            await this.migrateBusinessReview(review);
            count++;
          }
        }
      }
    } catch (error) {
      errors.push(`Business data migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate memory data
   */
  private async migrateMemoryData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      const memoryFile = path.join(this.legacyDataPath, 'memory', 'memories.json');
      if (await this.pathExists(memoryFile)) {
        const memories = await this.readJsonFile(memoryFile);
        if (Array.isArray(memories)) {
          for (const memory of memories) {
            await this.migrateMemory(memory);
            count++;
          }
        }
      }
    } catch (error) {
      errors.push(`Memory data migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate individual sprint
   */
  private async migrateSprint(sprint: any): Promise<void> {
    const id = sprint.id || randomUUID();
    const projectId = sprint.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO agile_sprints 
       (id, project_id, name, goal, status, start_date, end_date, duration, team, 
        story_points_planned, story_points_completed, stories_total, stories_completed, 
        velocity, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        sprint.name || 'Unnamed Sprint',
        sprint.goal || null,
        sprint.status || 'planning',
        sprint.startDate ? new Date(sprint.startDate).getTime() : null,
        sprint.endDate ? new Date(sprint.endDate).getTime() : null,
        sprint.duration || 14,
        JSON.stringify(sprint.team || []),
        sprint.storyPointsPlanned || 0,
        sprint.storyPointsCompleted || 0,
        sprint.storiesTotal || 0,
        sprint.storiesCompleted || 0,
        sprint.velocity || 0,
        sprint.createdAt ? new Date(sprint.createdAt).getTime() : now,
        sprint.updatedAt ? new Date(sprint.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate individual story
   */
  private async migrateStory(story: any): Promise<void> {
    const id = story.id || randomUUID();
    const projectId = story.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO agile_stories 
       (id, project_id, sprint_id, epic_id, title, description, status, story_points, 
        priority, assignee, tags, acceptance_criteria, design_document_url, 
        implementation_document_url, documentation_status, documentation_last_reviewed, 
        documentation_reviewers, groomed_with_user_feedback, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        story.sprintId || null,
        story.epicId || story.epic || null,
        story.title || 'Untitled Story',
        story.description || null,
        story.status || 'todo',
        story.storyPoints || 0,
        story.priority || 'medium',
        story.assignee || null,
        JSON.stringify(story.tags || []),
        JSON.stringify(story.acceptanceCriteria || []),
        story.designDocumentUrl || null,
        story.implementationDocumentUrl || null,
        story.documentationStatus || 'pending',
        story.documentationLastReviewed ? new Date(story.documentationLastReviewed).getTime() : null,
        JSON.stringify(story.documentationReviewers || []),
        story.groomedWithUserFeedback || false,
        story.createdAt ? new Date(story.createdAt).getTime() : now,
        story.updatedAt ? new Date(story.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate individual epic
   */
  private async migrateEpic(epic: any): Promise<void> {
    const id = epic.id || randomUUID();
    const projectId = epic.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO agile_epics 
       (id, project_id, title, description, status, story_points, priority, 
        target_quarter, business_value, success_criteria, dependencies, risks, 
        created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        epic.title || 'Untitled Epic',
        epic.description || null,
        epic.status || 'planning',
        epic.storyPoints || 0,
        epic.priority || 'medium',
        epic.targetQuarter || null,
        epic.businessValue || null,
        JSON.stringify(epic.successCriteria || []),
        JSON.stringify(epic.dependencies || []),
        JSON.stringify(epic.risks || []),
        epic.createdAt ? new Date(epic.createdAt).getTime() : now,
        epic.updatedAt ? new Date(epic.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate kanban board
   */
  private async migrateKanbanBoard(board: any): Promise<void> {
    const id = board.id || randomUUID();
    const projectId = board.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO kanban_boards 
       (id, project_id, name, description, columns, cards, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        board.name || 'Untitled Board',
        board.description || null,
        JSON.stringify(board.columns || []),
        JSON.stringify(board.cards || []),
        board.createdAt ? new Date(board.createdAt).getTime() : now,
        board.updatedAt ? new Date(board.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate document
   */
  private async migrateDocument(doc: any): Promise<void> {
    const id = doc.id || randomUUID();
    const projectId = doc.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO documents 
       (id, project_id, title, content, type, path, tags, status, author, version, 
        created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        doc.title || 'Untitled Document',
        doc.content || '',
        doc.type || 'markdown',
        doc.path || null,
        JSON.stringify(doc.tags || []),
        doc.status || 'draft',
        doc.author || null,
        doc.version || 1,
        doc.createdAt ? new Date(doc.createdAt).getTime() : now,
        doc.updatedAt ? new Date(doc.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate business plan
   */
  private async migrateBusinessPlan(plan: any): Promise<void> {
    const id = plan.id || randomUUID();
    const projectId = plan.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO business_plans 
       (id, project_id, plan_type, content, metadata, status, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        plan.planType || 'general',
        JSON.stringify(plan.content || {}),
        JSON.stringify(plan.metadata || {}),
        plan.status || 'draft',
        plan.createdAt ? new Date(plan.createdAt).getTime() : now,
        plan.updatedAt ? new Date(plan.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate market analysis
   */
  private async migrateMarketAnalysis(analysis: any): Promise<void> {
    const id = analysis.id || randomUUID();
    const projectId = analysis.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO market_analyses 
       (id, project_id, industry, target_market, geographic_scope, market_size, 
        growth_rate, trends, opportunities, challenges, recommendations, 
        competitors, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        analysis.industry || '',
        analysis.targetMarket || '',
        analysis.geographicScope || 'global',
        analysis.marketSize || null,
        analysis.growthRate || null,
        JSON.stringify(analysis.trends || []),
        JSON.stringify(analysis.opportunities || []),
        JSON.stringify(analysis.challenges || []),
        JSON.stringify(analysis.recommendations || []),
        JSON.stringify(analysis.competitors || []),
        analysis.createdAt ? new Date(analysis.createdAt).getTime() : now,
        analysis.updatedAt ? new Date(analysis.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate competitor analysis
   */
  private async migrateCompetitorAnalysis(analysis: any): Promise<void> {
    const id = analysis.id || randomUUID();
    const projectId = analysis.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO competitor_analyses 
       (id, project_id, industry, business_idea, competition_level, 
        competitors, gaps, opportunities, recommendations, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        analysis.industry || '',
        analysis.businessIdea || '',
        analysis.competitionLevel || null,
        JSON.stringify(analysis.competitors || []),
        JSON.stringify(analysis.gaps || []),
        JSON.stringify(analysis.opportunities || []),
        JSON.stringify(analysis.recommendations || []),
        analysis.createdAt ? new Date(analysis.createdAt).getTime() : now,
        analysis.updatedAt ? new Date(analysis.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate financial projection
   */
  private async migrateFinancialProjection(projection: any): Promise<void> {
    const id = projection.id || randomUUID();
    const projectId = projection.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO financial_projections 
       (id, project_id, business_model, timeline, currency, revenue, expenses, 
        net_income, cash_flow, break_even_month, total_investment_needed, roi, 
        scenarios, assumptions, recommendations, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        projection.businessModel || '',
        projection.timeline || 36,
        projection.currency || 'USD',
        JSON.stringify(projection.revenue || {}),
        JSON.stringify(projection.expenses || {}),
        JSON.stringify(projection.netIncome || {}),
        JSON.stringify(projection.cashFlow || {}),
        projection.breakEvenMonth || null,
        projection.totalInvestmentNeeded || null,
        projection.roi || null,
        JSON.stringify(projection.scenarios || null),
        JSON.stringify(projection.assumptions || []),
        JSON.stringify(projection.recommendations || []),
        projection.createdAt ? new Date(projection.createdAt).getTime() : now,
        projection.updatedAt ? new Date(projection.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate startup assessment
   */
  private async migrateStartupAssessment(assessment: any): Promise<void> {
    const id = assessment.id || randomUUID();
    const projectId = assessment.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO startup_assessments 
       (id, project_id, stage, score, categories, strengths, weaknesses, 
        next_steps, stage_recommendations, overall, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        assessment.stage || 'idea',
        assessment.score || 0,
        JSON.stringify(assessment.categories || []),
        JSON.stringify(assessment.strengths || []),
        JSON.stringify(assessment.weaknesses || []),
        JSON.stringify(assessment.nextSteps || []),
        JSON.stringify(assessment.stageRecommendations || []),
        assessment.overall || null,
        assessment.createdAt ? new Date(assessment.createdAt).getTime() : now,
        assessment.updatedAt ? new Date(assessment.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate pitch deck
   */
  private async migratePitchDeck(deck: any): Promise<void> {
    const id = deck.id || randomUUID();
    const projectId = deck.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO pitch_decks 
       (id, project_id, business_idea, template, slides, presentation_tips, 
        key_messages, markdown, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        deck.businessIdea || '',
        deck.template || 'standard',
        JSON.stringify(deck.slides || []),
        JSON.stringify(deck.presentationTips || []),
        JSON.stringify(deck.keyMessages || []),
        deck.markdown || '',
        deck.createdAt ? new Date(deck.createdAt).getTime() : now,
        deck.updatedAt ? new Date(deck.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate startup metrics
   */
  private async migrateStartupMetrics(metrics: any): Promise<void> {
    const id = metrics.id || randomUUID();
    const projectId = metrics.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO startup_metrics 
       (id, project_id, metrics_type, current_metrics, goals, timeframe, 
        trends, benchmarks, recommendations, health_score, alerts, 
        created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        metrics.metricsType || 'saas',
        JSON.stringify(metrics.currentMetrics || {}),
        JSON.stringify(metrics.goals || {}),
        metrics.timeframe || 'monthly',
        JSON.stringify(metrics.trends || []),
        JSON.stringify(metrics.benchmarks || []),
        JSON.stringify(metrics.recommendations || []),
        metrics.healthScore || null,
        JSON.stringify(metrics.alerts || []),
        metrics.createdAt ? new Date(metrics.createdAt).getTime() : now,
        metrics.updatedAt ? new Date(metrics.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate business review
   */
  private async migrateBusinessReview(review: any): Promise<void> {
    const id = review.id || randomUUID();
    const projectId = review.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO business_reviews 
       (id, project_id, business_name, current_stage, overall_health_score, 
        strengths, gaps, strategic_paths, immediate_actions, data_collected, 
        review_date, next_review_date, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        review.businessName || '',
        review.currentStage || 'idea',
        review.overallHealthScore || 0,
        JSON.stringify(review.strengths || []),
        JSON.stringify(review.gaps || []),
        JSON.stringify(review.strategicPaths || []),
        JSON.stringify(review.immediateActions || []),
        JSON.stringify(review.dataCollected || {}),
        review.reviewDate || new Date().toISOString(),
        review.nextReviewDate || new Date(Date.now() + 90 * 24 * 60 * 60 * 1000).toISOString(),
        review.createdAt ? new Date(review.createdAt).getTime() : now,
        review.updatedAt ? new Date(review.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Migrate memory
   */
  private async migrateMemory(memory: any): Promise<void> {
    const id = memory.id || randomUUID();
    const projectId = memory.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO memories 
       (id, project_id, title, type, content, tags, metadata, importance, category, source, 
        created_by, embedding_vector, similarity_score, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        memory.title || null,
        memory.type || 'general',
        memory.content || '',
        JSON.stringify(memory.tags || []),
        JSON.stringify(memory.metadata || {}),
        memory.importance || 'medium',
        memory.category || null,
        memory.source || null,
        memory.createdBy || null,
        memory.embeddingVector || null,
        memory.similarityScore || null,
        memory.createdAt ? new Date(memory.createdAt).getTime() : now,
        memory.updatedAt ? new Date(memory.updatedAt).getTime() : now
      ]
    );
  }

  /**
   * Update migration status
   */
  private async updateMigrationStatus(status: string): Promise<void> {
    const now = Date.now();
    
    await this.dbRun(
      'INSERT OR REPLACE INTO atlas_metadata (key, value, updated_at) VALUES (?, ?, ?)',
      ['migration_status', status, now]
    );
    
    await this.dbRun(
      'INSERT OR REPLACE INTO atlas_metadata (key, value, updated_at) VALUES (?, ?, ?)',
      ['last_migration_timestamp', now.toString(), now]
    );
  }

  /**
   * Migrate ADR data
   */
  private async migrateADRData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      // Look for ADR files in various possible locations
      const adrPaths = [
        path.join(this.legacyDataPath, 'adr'),
        path.join(this.legacyDataPath, 'adr-management'),
        path.join(this.legacyDataPath, 'adrs')
      ];

      for (const adrPath of adrPaths) {
        if (await this.pathExists(adrPath)) {
          // Check for index file
          const indexFile = path.join(adrPath, 'index.json');
          if (await this.pathExists(indexFile)) {
            const index = await this.readJsonFile(indexFile);
            if (index && index.adrs && Array.isArray(index.adrs)) {
              for (const adrSummary of index.adrs) {
                // Read the full ADR file
                const adrFile = path.join(adrPath, `${adrSummary.id}.json`);
                if (await this.pathExists(adrFile)) {
                  const adr = await this.readJsonFile(adrFile);
                  if (adr) {
                    await this.migrateADR(adr);
                    count++;
                  }
                }
              }
            }
          }

          // Also check for individual ADR files without index
          const adrFiles = await this.findJsonFiles(adrPath);
          for (const adrFile of adrFiles) {
            if (!adrFile.endsWith('index.json')) {
              const adr = await this.readJsonFile(adrFile);
              if (adr && adr.id && adr.id.startsWith('ADR-')) {
                await this.migrateADR(adr);
                count++;
              }
            }
          }
        }
      }

    } catch (error) {
      errors.push(`ADR data migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate individual ADR
   */
  private async migrateADR(adr: any): Promise<void> {
    const id = adr.id || `ADR-${Date.now().toString().slice(-4)}`;
    const projectId = adr.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO adr_records 
       (id, project_id, title, status, date, deciders, template, context, decision, 
        consequences, tags, decision_drivers, considered_options, pros_and_cons, 
        supersedes, superseded_by, related_to, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        adr.title || 'Untitled ADR',
        adr.status || 'proposed',
        adr.date ? new Date(adr.date).getTime() : now,
        JSON.stringify(adr.deciders || []),
        adr.template || 'nygard',
        adr.context || '',
        adr.decision || '',
        adr.consequences || '',
        JSON.stringify(adr.tags || []),
        JSON.stringify(adr.decisionDrivers || []),
        JSON.stringify(adr.consideredOptions || []),
        JSON.stringify(adr.prosAndCons || []),
        JSON.stringify(adr.supersedes || []),
        adr.supersededBy || null,
        JSON.stringify(adr.relatedTo || []),
        adr.createdAt ? new Date(adr.createdAt).getTime() : now,
        adr.updatedAt ? new Date(adr.updatedAt).getTime() : now
      ]
    );

    // Migrate status history if available
    if (adr.statusHistory && Array.isArray(adr.statusHistory)) {
      for (const historyEntry of adr.statusHistory) {
        await this.dbRun(
          `INSERT OR REPLACE INTO adr_status_history 
           (id, adr_id, project_id, from_status, to_status, changed_at, changed_by, reason) 
           VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
          [
            randomUUID(),
            id,
            projectId,
            historyEntry.from || null,
            historyEntry.to || 'proposed',
            historyEntry.date ? new Date(historyEntry.date).getTime() : now,
            historyEntry.changedBy || 'unknown',
            historyEntry.reason || null
          ]
        );
      }
    }
  }

  /**
   * Migrate development module data
   */
  private async migrateDevelopmentData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      const devFile = path.join(this.legacyDataPath, 'development', 'development.json');
      if (await this.pathExists(devFile)) {
        const devData = await this.readJsonFile(devFile);
        if (devData) {
          // Migrate features
          if (devData.features) {
            for (const featureId in devData.features) {
              const feature = devData.features[featureId];
              await this.migrateDevelopmentFeature(feature);
              count++;
            }
          }

          // Migrate TDD sessions
          if (devData.sessions) {
            for (const sessionId in devData.sessions) {
              const session = devData.sessions[sessionId];
              await this.migrateTDDSession(session);
              count++;
            }
          }

          // Migrate TDD config
          if (devData.config) {
            await this.migrateTDDConfig(devData.config);
            count++;
          }
        }
      }
    } catch (error) {
      errors.push(`Development data migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate individual development feature
   */
  private async migrateDevelopmentFeature(feature: any): Promise<void> {
    const id = feature.id || randomUUID();
    const projectId = feature.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO development_features 
       (id, project_id, name, description, status, test_coverage, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        feature.name || 'Untitled Feature',
        feature.description || '',
        feature.status || 'planning',
        feature.testCoverage || 0.0,
        feature.createdAt ? new Date(feature.createdAt).getTime() : now,
        feature.updatedAt ? new Date(feature.updatedAt).getTime() : now
      ]
    );

    // Migrate test cases
    if (feature.testCases && Array.isArray(feature.testCases)) {
      for (const testCase of feature.testCases) {
        await this.dbRun(
          `INSERT OR REPLACE INTO development_test_cases 
           (id, feature_id, project_id, name, description, type, status, expected_behavior, 
            actual_behavior, error_message, created_at, updated_at) 
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
          [
            testCase.id || randomUUID(),
            id,
            projectId,
            testCase.name || 'Untitled Test',
            testCase.description || '',
            testCase.type || 'unit',
            testCase.status || 'pending',
            testCase.expectedBehavior || '',
            testCase.actualBehavior || null,
            testCase.errorMessage || null,
            testCase.createdAt ? new Date(testCase.createdAt).getTime() : now,
            testCase.updatedAt ? new Date(testCase.updatedAt).getTime() : now
          ]
        );
      }
    }
  }

  /**
   * Migrate TDD session
   */
  private async migrateTDDSession(session: any): Promise<void> {
    const id = session.id || randomUUID();
    const projectId = session.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO development_tdd_sessions 
       (id, feature_id, project_id, current_phase, started_at, completed_at, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        session.featureId || 'unknown',
        projectId,
        session.currentPhase || 'red',
        session.startedAt ? new Date(session.startedAt).getTime() : now,
        session.completedAt ? new Date(session.completedAt).getTime() : null,
        session.createdAt ? new Date(session.createdAt).getTime() : now,
        session.updatedAt ? new Date(session.updatedAt).getTime() : now
      ]
    );

    // Migrate phase history
    if (session.phaseHistory && Array.isArray(session.phaseHistory)) {
      for (const phase of session.phaseHistory) {
        await this.dbRun(
          `INSERT OR REPLACE INTO development_tdd_phase_history 
           (id, session_id, project_id, phase, started_at, completed_at, notes) 
           VALUES (?, ?, ?, ?, ?, ?, ?)`,
          [
            randomUUID(),
            id,
            projectId,
            phase.phase || 'red',
            phase.startedAt ? new Date(phase.startedAt).getTime() : now,
            phase.completedAt ? new Date(phase.completedAt).getTime() : null,
            phase.notes || null
          ]
        );
      }
    }
  }

  /**
   * Migrate TDD configuration
   */
  private async migrateTDDConfig(config: any): Promise<void> {
    const projectId = config.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO development_tdd_config 
       (project_id, enforce_test_first, minimum_test_coverage, 
        require_tests_before_implementation, auto_generate_test_templates, 
        created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?)`,
      [
        projectId,
        config.enforceTestFirst !== false,
        config.minimumTestCoverage || 80.0,
        config.requireTestsBeforeImplementation !== false,
        config.autoGenerateTestTemplates !== false,
        now,
        now
      ]
    );
  }

  /**
   * Migrate workspace module data
   */
  private async migrateWorkspaceData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      const workspaceFile = path.join(this.legacyDataPath, 'workspace', 'workspace.json');
      if (await this.pathExists(workspaceFile)) {
        const workspaceData = await this.readJsonFile(workspaceFile);
        if (workspaceData) {
          // Migrate workspaces
          if (workspaceData.workspaces) {
            for (const workspaceId in workspaceData.workspaces) {
              const workspace = workspaceData.workspaces[workspaceId];
              await this.migrateWorkspace(workspace, workspaceData.activeWorkspaceId === workspaceId);
              count++;
            }
          }
        }
      }
    } catch (error) {
      errors.push(`Workspace data migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate individual workspace
   */
  private async migrateWorkspace(workspace: any, isActive: boolean): Promise<void> {
    const id = workspace.id || `ws-${randomUUID()}`;
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO workspaces 
       (id, name, description, root_path, active, settings, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        workspace.name || 'Untitled Workspace',
        workspace.description || '',
        workspace.rootPath || process.cwd(),
        isActive,
        JSON.stringify(workspace.settings || {}),
        workspace.createdAt ? new Date(workspace.createdAt).getTime() : now,
        workspace.updatedAt ? new Date(workspace.updatedAt).getTime() : now
      ]
    );

    // Migrate repositories
    if (workspace.repositories && Array.isArray(workspace.repositories)) {
      for (const repo of workspace.repositories) {
        await this.dbRun(
          `INSERT OR REPLACE INTO workspace_repositories 
           (id, workspace_id, name, path, type, is_primary, remote_url, current_branch, 
            last_sync, dependencies, created_at, updated_at) 
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
          [
            repo.id || `repo-${randomUUID()}`,
            id,
            repo.name || 'Untitled Repository',
            repo.path || '',
            repo.type || 'local',
            repo.primary || false,
            repo.remote || null,
            repo.branch || null,
            repo.lastSync ? new Date(repo.lastSync).getTime() : null,
            JSON.stringify(repo.dependencies || []),
            repo.createdAt ? new Date(repo.createdAt).getTime() : now,
            repo.updatedAt ? new Date(repo.updatedAt).getTime() : now
          ]
        );
      }
    }
  }

  /**
   * Migrate process automation module data
   */
  private async migrateProcessAutomationData(errors: string[]): Promise<number> {
    let count = 0;

    try {
      const processesPath = path.join(this.legacyDataPath, 'process-automation', 'processes');
      const executionsPath = path.join(this.legacyDataPath, 'process-automation', 'executions');
      const templatesPath = path.join(this.legacyDataPath, 'process-automation', 'templates');

      // Migrate processes
      if (await this.pathExists(processesPath)) {
        const processFiles = await this.findJsonFiles(processesPath);
        for (const processFile of processFiles) {
          const process = await this.readJsonFile(processFile);
          if (process) {
            await this.migrateProcess(process);
            count++;
          }
        }
      }

      // Migrate executions
      if (await this.pathExists(executionsPath)) {
        const processDirs = await fs.readdir(executionsPath, { withFileTypes: true });
        for (const processDir of processDirs) {
          if (processDir.isDirectory()) {
            const executionFiles = await this.findJsonFiles(path.join(executionsPath, processDir.name));
            for (const executionFile of executionFiles) {
              const execution = await this.readJsonFile(executionFile);
              if (execution) {
                await this.migrateProcessExecution(execution);
                count++;
              }
            }
          }
        }
      }

      // Migrate templates
      if (await this.pathExists(templatesPath)) {
        const templateFiles = await this.findJsonFiles(templatesPath);
        for (const templateFile of templateFiles) {
          const template = await this.readJsonFile(templateFile);
          if (template) {
            await this.migrateProcessTemplate(template);
            count++;
          }
        }
      }
    } catch (error) {
      errors.push(`Process automation data migration error: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }

    return count;
  }

  /**
   * Migrate individual process
   */
  private async migrateProcess(process: any): Promise<void> {
    const id = process.id || randomUUID();
    const projectId = process.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO process_definitions 
       (id, project_id, name, description, version, persona, category, tags, 
        variables, on_success, on_failure, is_active, created_by, created_at, updated_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        projectId,
        process.name || 'Untitled Process',
        process.description || '',
        process.version || '1.0.0',
        process.persona || null,
        process.metadata?.category || null,
        JSON.stringify(process.metadata?.tags || []),
        JSON.stringify(process.variables || {}),
        null, // on_success - not in legacy format
        null, // on_failure - not in legacy format
        process.metadata?.isActive !== false,
        process.metadata?.createdBy || 'system',
        process.metadata?.createdAt ? new Date(process.metadata.createdAt).getTime() : now,
        now
      ]
    );

    // Migrate activities
    if (process.activities && Array.isArray(process.activities)) {
      for (let i = 0; i < process.activities.length; i++) {
        const activity = process.activities[i];
        await this.dbRun(
          `INSERT OR REPLACE INTO process_activities 
           (id, process_id, activity_order, activity_id, type, name, description, 
            config, retry_policy, timeout, required, created_at) 
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
          [
            randomUUID(),
            id,
            i,
            activity.id || randomUUID(),
            activity.type || 'tool',
            activity.name || 'Untitled Activity',
            activity.description || '',
            JSON.stringify(activity.config || {}),
            activity.retryPolicy ? JSON.stringify(activity.retryPolicy) : null,
            activity.timeout || null,
            activity.required !== false,
            now
          ]
        );
      }
    }

    // Migrate triggers
    if (process.triggers && Array.isArray(process.triggers)) {
      for (const trigger of process.triggers) {
        await this.dbRun(
          `INSERT OR REPLACE INTO process_triggers 
           (id, process_id, trigger_id, type, name, enabled, config, ai_suggested, 
            reasoning, created_at, updated_at) 
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
          [
            randomUUID(),
            id,
            trigger.id || randomUUID(),
            trigger.type || 'manual',
            trigger.name || 'Untitled Trigger',
            trigger.enabled !== false,
            JSON.stringify(trigger.config || {}),
            trigger.aiSuggested || false,
            trigger.reasoning || null,
            now,
            now
          ]
        );
      }
    }
  }

  /**
   * Migrate individual process execution
   */
  private async migrateProcessExecution(execution: any): Promise<void> {
    const id = execution.id || randomUUID();
    const projectId = execution.projectId || 'default-project';
    const now = Date.now();

    await this.dbRun(
      `INSERT OR REPLACE INTO process_executions 
       (id, process_id, project_id, process_version, status, triggered_by, 
        trigger_type, variables, error_message, error_code, started_at, 
        completed_at, duration, created_at) 
       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
      [
        id,
        execution.processId || 'unknown',
        projectId,
        execution.processVersion || '1.0.0',
        execution.status || 'pending',
        execution.triggeredBy || 'system',
        execution.triggerType || 'manual',
        JSON.stringify(execution.variables || {}),
        execution.error?.message || null,
        execution.error?.code || null,
        execution.startedAt ? new Date(execution.startedAt).getTime() : now,
        execution.completedAt ? new Date(execution.completedAt).getTime() : null,
        execution.duration || null,
        now
      ]
    );

    // Migrate activity results
    if (execution.activityResults) {
      for (const [activityId, result] of Object.entries(execution.activityResults as any)) {
        const typedResult = result as any;
        await this.dbRun(
          `INSERT OR REPLACE INTO process_activity_results 
           (id, execution_id, ac