@claude-vector/core/src/index-integrity-manager.js

Core vector search engine for code intelligence

/**
 * IndexIntegrityManager - インデックス整合性チェックと自動復旧システム
 * 
 * 機能:
 * - インデックスファイルの整合性チェック
 * - 破損検出と診断
 * - 自動バックアップ管理
 * - 段階的復旧プロセス
 * - 復旧進捗とログ
 * - インデックス品質評価
 */

import fs from 'fs/promises';
import fsSync from 'fs';
import path from 'path';
import crypto from 'crypto';
import { EventEmitter } from 'events';

export class IndexIntegrityManager extends EventEmitter {
  constructor(options = {}) {
    super();
    
    this.options = {
      // バックアップ設定
      enableAutoBackup: options.enableAutoBackup ?? true,
      maxBackups: options.maxBackups ?? 10,
      backupInterval: options.backupInterval ?? 24 * 60 * 60 * 1000, // 24時間
      backupCompressionLevel: options.backupCompressionLevel ?? 6,
      
      // 整合性チェック設定
      enablePeriodicCheck: options.enablePeriodicCheck ?? true,
      checkInterval: options.checkInterval ?? 60 * 60 * 1000, // 1時間
      checksumAlgorithm: options.checksumAlgorithm ?? 'sha256',
      
      // 復旧設定
      autoRecoveryEnabled: options.autoRecoveryEnabled ?? true,
      maxRecoveryAttempts: options.maxRecoveryAttempts ?? 3,
      recoveryTimeout: options.recoveryTimeout ?? 300000, // 5分
      
      // ログ設定
      verboseLogging: options.verboseLogging ?? false,
      logFile: options.logFile ?? './.claude-integrity.log',
      
      ...options
    };
    
    // 状態管理
    this.state = {
      isChecking: false,
      isRecovering: false,
      lastCheckTime: null,
      lastBackupTime: null,
      corruptionDetected: false,
      recoveryAttempts: 0
    };
    
    // 統計情報
    this.stats = {
      totalChecks: 0,
      corruptionsDetected: 0,
      successfulRecoveries: 0,
      failedRecoveries: 0,
      backupsCreated: 0,
      averageCheckTime: 0
    };
    
    // タイマー
    this.checkTimer = null;
    this.backupTimer = null;
    
    // 初期化
    this.initialize();
  }

  /**
   * 初期化
   */
  async initialize() {
    if (this.options.enablePeriodicCheck) {
      this.startPeriodicChecks();
    }
    
    if (this.options.enableAutoBackup) {
      this.startAutoBackup();
    }
    
    this.emit('initialized', this.options);
  }

  /**
   * インデックス整合性チェック
   */
  async checkIndexIntegrity(indexPaths, metadata = {}) {
    if (this.state.isChecking) {
      throw new Error('Integrity check already in progress');
    }
    
    this.state.isChecking = true;
    this.state.lastCheckTime = Date.now();
    this.stats.totalChecks++;
    
    const startTime = Date.now();
    const checkResult = {
      timestamp: new Date().toISOString(),
      indexPaths,
      metadata,
      isValid: true,
      issues: [],
      corruptionLevel: 'none', // none, minor, major, critical
      checksums: {},
      recommendations: []
    };
    
    try {
      this.emit('check-started', { indexPaths, metadata });
      
      // 各インデックスファイルのチェック
      for (const indexPath of indexPaths) {
        await this.checkSingleIndex(indexPath, checkResult);
      }
      
      // 相互整合性チェック
      if (indexPaths.length > 1) {
        await this.checkCrossIndexConsistency(indexPaths, checkResult);
      }
      
      // メタデータ整合性チェック
      await this.checkMetadataConsistency(indexPaths, checkResult);
      
      // 総合評価
      this.evaluateOverallIntegrity(checkResult);
      
      // 統計更新
      const checkTime = Date.now() - startTime;
      this.updateCheckStats(checkTime);
      
      this.emit('check-completed', checkResult);
      
      if (!checkResult.isValid) {
        this.state.corruptionDetected = true;
        this.stats.corruptionsDetected++;
        
        // 自動復旧の開始
        if (this.options.autoRecoveryEnabled) {
          setTimeout(() => {
            this.recoverIndex(indexPaths, checkResult);
          }, 1000);
        }
      }
      
      return checkResult;
      
    } catch (error) {
      checkResult.isValid = false;
      checkResult.issues.push({
        type: 'CHECK_ERROR',
        severity: 'critical',
        message: `Integrity check failed: ${error.message}`,
        timestamp: Date.now()
      });
      
      this.emit('check-error', { error, checkResult });
      throw error;
      
    } finally {
      this.state.isChecking = false;
    }
  }

  /**
   * 単一インデックスファイルのチェック
   */
  async checkSingleIndex(indexPath, checkResult) {
    try {
      // ファイル存在チェック
      if (!fsSync.existsSync(indexPath)) {
        checkResult.issues.push({
          type: 'FILE_MISSING',
          severity: 'critical',
          file: indexPath,
          message: `Index file does not exist: ${indexPath}`,
          timestamp: Date.now()
        });
        checkResult.isValid = false;
        return;
      }
      
      // ファイルサイズチェック
      const stats = await fs.stat(indexPath);
      if (stats.size === 0) {
        checkResult.issues.push({
          type: 'EMPTY_FILE',
          severity: 'critical',
          file: indexPath,
          message: `Index file is empty: ${indexPath}`,
          timestamp: Date.now()
        });
        checkResult.isValid = false;
        return;
      }
      
      // チェックサム計算
      const checksum = await this.calculateChecksum(indexPath);
      checkResult.checksums[indexPath] = checksum;
      
      // JSON構造チェック
      await this.validateJsonStructure(indexPath, checkResult);
      
      // データ品質チェック
      await this.validateDataQuality(indexPath, checkResult);
      
    } catch (error) {
      checkResult.issues.push({
        type: 'FILE_ACCESS_ERROR',
        severity: 'major',
        file: indexPath,
        message: `Cannot access file: ${error.message}`,
        timestamp: Date.now()
      });
      checkResult.isValid = false;
    }
  }

  /**
   * JSON構造検証
   */
  async validateJsonStructure(indexPath, checkResult) {
    try {
      const content = await fs.readFile(indexPath, 'utf8');
      const data = JSON.parse(content);
      
      // ファイルタイプ別検証
      if (indexPath.includes('embeddings')) {
        this.validateEmbeddingsStructure(data, indexPath, checkResult);
      } else if (indexPath.includes('chunks')) {
        this.validateChunksStructure(data, indexPath, checkResult);
      } else if (indexPath.includes('metadata')) {
        this.validateMetadataStructure(data, indexPath, checkResult);
      }
      
    } catch (error) {
      if (error instanceof SyntaxError) {
        checkResult.issues.push({
          type: 'INVALID_JSON',
          severity: 'critical',
          file: indexPath,
          message: `Invalid JSON format: ${error.message}`,
          timestamp: Date.now()
        });
        checkResult.isValid = false;
      } else {
        throw error;
      }
    }
  }

  /**
   * エンベディング構造検証
   */
  validateEmbeddingsStructure(data, filePath, checkResult) {
    if (!Array.isArray(data)) {
      checkResult.issues.push({
        type: 'INVALID_STRUCTURE',
        severity: 'critical',
        file: filePath,
        message: 'Embeddings file should contain an array',
        timestamp: Date.now()
      });
      checkResult.isValid = false;
      return;
    }
    
    // サンプル検証（最初の10個）
    const samples = data.slice(0, Math.min(10, data.length));
    for (let i = 0; i < samples.length; i++) {
      const embedding = samples[i];
      
      if (!Array.isArray(embedding)) {
        checkResult.issues.push({
          type: 'INVALID_EMBEDDING',
          severity: 'major',
          file: filePath,
          message: `Embedding at index ${i} is not an array`,
          timestamp: Date.now()
        });
        checkResult.isValid = false;
      } else if (embedding.length === 0) {
        checkResult.issues.push({
          type: 'EMPTY_EMBEDDING',
          severity: 'major',
          file: filePath,
          message: `Embedding at index ${i} is empty`,
          timestamp: Date.now()
        });
        checkResult.isValid = false;
      } else if (embedding.some(val => typeof val !== 'number' || isNaN(val))) {
        checkResult.issues.push({
          type: 'INVALID_EMBEDDING_VALUES',
          severity: 'major',
          file: filePath,
          message: `Embedding at index ${i} contains invalid values`,
          timestamp: Date.now()
        });
        checkResult.isValid = false;
      }
    }
  }

  /**
   * チャンク構造検証
   */
  validateChunksStructure(data, filePath, checkResult) {
    if (!Array.isArray(data)) {
      checkResult.issues.push({
        type: 'INVALID_STRUCTURE',
        severity: 'critical',
        file: filePath,
        message: 'Chunks file should contain an array',
        timestamp: Date.now()
      });
      checkResult.isValid = false;
      return;
    }
    
    // サンプル検証
    const samples = data.slice(0, Math.min(10, data.length));
    for (let i = 0; i < samples.length; i++) {
      const chunk = samples[i];
      
      if (!chunk || typeof chunk !== 'object') {
        checkResult.issues.push({
          type: 'INVALID_CHUNK',
          severity: 'major',
          file: filePath,
          message: `Chunk at index ${i} is not an object`,
          timestamp: Date.now()
        });
        checkResult.isValid = false;
        continue;
      }
      
      // 必須フィールドチェック
      const requiredFields = ['content', 'metadata'];
      for (const field of requiredFields) {
        if (!(field in chunk)) {
          checkResult.issues.push({
            type: 'MISSING_FIELD',
            severity: 'major',
            file: filePath,
            message: `Chunk at index ${i} missing required field: ${field}`,
            timestamp: Date.now()
          });
          checkResult.isValid = false;
        }
      }
      
      // コンテンツの有効性チェック
      if (chunk.content && typeof chunk.content !== 'string') {
        checkResult.issues.push({
          type: 'INVALID_CONTENT',
          severity: 'minor',
          file: filePath,
          message: `Chunk at index ${i} content is not a string`,
          timestamp: Date.now()
        });
      }
    }
  }

  /**
   * メタデータ構造検証
   */
  validateMetadataStructure(data, filePath, checkResult) {
    if (!data || typeof data !== 'object') {
      checkResult.issues.push({
        type: 'INVALID_STRUCTURE',
        severity: 'critical',
        file: filePath,
        message: 'Metadata file should contain an object',
        timestamp: Date.now()
      });
      checkResult.isValid = false;
      return;
    }
    
    // 基本フィールドのチェック
    const expectedFields = ['version', 'timestamp', 'totalChunks', 'indexConfig'];
    for (const field of expectedFields) {
      if (!(field in data)) {
        checkResult.issues.push({
          type: 'MISSING_METADATA_FIELD',
          severity: 'minor',
          file: filePath,
          message: `Missing metadata field: ${field}`,
          timestamp: Date.now()
        });
      }
    }
  }

  /**
   * データ品質チェック
   */
  async validateDataQuality(indexPath, checkResult) {
    try {
      const content = await fs.readFile(indexPath, 'utf8');
      const data = JSON.parse(content);
      
      if (Array.isArray(data)) {
        // 配列の場合（embeddings, chunks）
        if (data.length === 0) {
          checkResult.issues.push({
            type: 'EMPTY_DATA',
            severity: 'major',
            file: indexPath,
            message: 'Index file contains no data',
            timestamp: Date.now()
          });
          checkResult.isValid = false;
        }
        
        // 重複チェック（サンプル）
        if (data.length > 100) {
          const sample = data.slice(0, 100);
          const unique = new Set(sample.map(item => JSON.stringify(item)));
          if (unique.size < sample.length * 0.9) { // 10%以上重複
            checkResult.issues.push({
              type: 'HIGH_DUPLICATION',
              severity: 'minor',
              file: indexPath,
              message: 'High level of data duplication detected',
              timestamp: Date.now()
            });
          }
        }
      }
      
    } catch (error) {
      // JSON parsing errorは別でキャッチされるのでここではスキップ
      if (!(error instanceof SyntaxError)) {
        throw error;
      }
    }
  }

  /**
   * 相互整合性チェック
   */
  async checkCrossIndexConsistency(indexPaths, checkResult) {
    try {
      const loadedData = {};
      
      // 全ファイルを読み込み
      for (const indexPath of indexPaths) {
        if (fsSync.existsSync(indexPath)) {
          const content = await fs.readFile(indexPath, 'utf8');
          const data = JSON.parse(content);
          loadedData[indexPath] = data;
        }
      }
      
      // エンベディングとチャンクの数の一致チェック
      const embeddingsFile = indexPaths.find(p => p.includes('embeddings'));
      const chunksFile = indexPaths.find(p => p.includes('chunks'));
      
      if (embeddingsFile && chunksFile && 
          loadedData[embeddingsFile] && loadedData[chunksFile]) {
        
        const embeddingsCount = loadedData[embeddingsFile].length;
        const chunksCount = loadedData[chunksFile].length;
        
        if (embeddingsCount !== chunksCount) {
          checkResult.issues.push({
            type: 'COUNT_MISMATCH',
            severity: 'critical',
            message: `Embeddings count (${embeddingsCount}) does not match chunks count (${chunksCount})`,
            files: [embeddingsFile, chunksFile],
            timestamp: Date.now()
          });
          checkResult.isValid = false;
        }
      }
      
    } catch (error) {
      checkResult.issues.push({
        type: 'CONSISTENCY_CHECK_ERROR',
        severity: 'major',
        message: `Cross-index consistency check failed: ${error.message}`,
        timestamp: Date.now()
      });
    }
  }

  /**
   * メタデータ整合性チェック
   */
  async checkMetadataConsistency(indexPaths, checkResult) {
    const metadataFile = indexPaths.find(p => p.includes('metadata'));
    if (!metadataFile || !fsSync.existsSync(metadataFile)) {
      return; // メタデータファイルがない場合はスキップ
    }
    
    try {
      const content = await fs.readFile(metadataFile, 'utf8');
      const metadata = JSON.parse(content);
      
      // 実際のデータとメタデータの整合性チェック
      const chunksFile = indexPaths.find(p => p.includes('chunks'));
      if (chunksFile && fsSync.existsSync(chunksFile)) {
        const chunksContent = await fs.readFile(chunksFile, 'utf8');
        const chunks = JSON.parse(chunksContent);
        
        if (metadata.totalChunks && metadata.totalChunks !== chunks.length) {
          checkResult.issues.push({
            type: 'METADATA_MISMATCH',
            severity: 'minor',
            message: `Metadata totalChunks (${metadata.totalChunks}) does not match actual chunks (${chunks.length})`,
            files: [metadataFile, chunksFile],
            timestamp: Date.now()
          });
        }
      }
      
    } catch (error) {
      checkResult.issues.push({
        type: 'METADATA_CHECK_ERROR',
        severity: 'minor',
        message: `Metadata consistency check failed: ${error.message}`,
        timestamp: Date.now()
      });
    }
  }

  /**
   * 総合整合性評価
   */
  evaluateOverallIntegrity(checkResult) {
    const issues = checkResult.issues;
    const criticalIssues = issues.filter(i => i.severity === 'critical').length;
    const majorIssues = issues.filter(i => i.severity === 'major').length;
    const minorIssues = issues.filter(i => i.severity === 'minor').length;
    
    if (criticalIssues > 0) {
      checkResult.corruptionLevel = 'critical';
      checkResult.recommendations.push('Immediate index rebuild required');
      checkResult.recommendations.push('Consider restoring from backup');
    } else if (majorIssues > 2) {
      checkResult.corruptionLevel = 'major';
      checkResult.recommendations.push('Index rebuild recommended');
      checkResult.recommendations.push('Monitor for further corruption');
    } else if (majorIssues > 0 || minorIssues > 5) {
      checkResult.corruptionLevel = 'minor';
      checkResult.recommendations.push('Consider index refresh');
      checkResult.recommendations.push('Schedule maintenance rebuild');
    } else {
      checkResult.corruptionLevel = 'none';
      checkResult.recommendations.push('Index appears healthy');
    }
  }

  /**
   * チェックサム計算
   */
  async calculateChecksum(filePath) {
    const content = await fs.readFile(filePath);
    return crypto.createHash(this.options.checksumAlgorithm)
                 .update(content)
                 .digest('hex');
  }

  /**
   * インデックス復旧
   */
  async recoverIndex(indexPaths, integrityResult) {
    if (this.state.isRecovering) {
      throw new Error('Recovery already in progress');
    }
    
    if (this.state.recoveryAttempts >= this.options.maxRecoveryAttempts) {
      throw new Error(`Maximum recovery attempts (${this.options.maxRecoveryAttempts}) exceeded`);
    }
    
    this.state.isRecovering = true;
    this.state.recoveryAttempts++;
    
    const recoveryResult = {
      timestamp: new Date().toISOString(),
      indexPaths,
      integrityResult,
      success: false,
      strategy: null,
      steps: [],
      restoredFiles: [],
      errors: []
    };
    
    try {
      this.emit('recovery-started', { indexPaths, integrityResult });
      
      // 復旧戦略の決定
      const strategy = this.determineRecoveryStrategy(integrityResult);
      recoveryResult.strategy = strategy;
      
      this.emit('recovery-strategy-determined', { strategy, integrityResult });
      
      switch (strategy) {
        case 'backup_restore':
          await this.executeBackupRestore(indexPaths, recoveryResult);
          break;
        case 'partial_repair':
          await this.executePartialRepair(indexPaths, recoveryResult);
          break;
        case 'full_rebuild':
          await this.executeFullRebuild(indexPaths, recoveryResult);
          break;
        default:
          throw new Error(`Unknown recovery strategy: ${strategy}`);
      }
      
      // 復旧後の整合性チェック
      const postRecoveryCheck = await this.checkIndexIntegrity(indexPaths, {
        type: 'post_recovery_verification'
      });
      
      if (postRecoveryCheck.isValid) {
        recoveryResult.success = true;
        this.stats.successfulRecoveries++;
        this.state.corruptionDetected = false;
        this.state.recoveryAttempts = 0;
      } else {
        recoveryResult.success = false;
        recoveryResult.errors.push('Post-recovery integrity check failed');
        this.stats.failedRecoveries++;
      }
      
      this.emit('recovery-completed', recoveryResult);
      return recoveryResult;
      
    } catch (error) {
      recoveryResult.success = false;
      recoveryResult.errors.push(error.message);
      this.stats.failedRecoveries++;
      
      this.emit('recovery-error', { error, recoveryResult });
      throw error;
      
    } finally {
      this.state.isRecovering = false;
    }
  }

  /**
   * 復旧戦略決定
   */
  determineRecoveryStrategy(integrityResult) {
    const { corruptionLevel, issues } = integrityResult;
    
    // クリティカルな破損の場合
    if (corruptionLevel === 'critical') {
      const hasMissingFiles = issues.some(i => i.type === 'FILE_MISSING');
      const hasInvalidJson = issues.some(i => i.type === 'INVALID_JSON');
      
      if (hasMissingFiles || hasInvalidJson) {
        return 'backup_restore';
      } else {
        return 'full_rebuild';
      }
    }
    
    // メジャーな破損の場合
    if (corruptionLevel === 'major') {
      const hasStructuralIssues = issues.some(i => 
        i.type === 'INVALID_STRUCTURE' || i.type === 'COUNT_MISMATCH'
      );
      
      if (hasStructuralIssues) {
        return 'backup_restore';
      } else {
        return 'partial_repair';
      }
    }
    
    // マイナーな破損の場合
    return 'partial_repair';
  }

  /**
   * バックアップからの復元
   */
  async executeBackupRestore(indexPaths, recoveryResult) {
    recoveryResult.steps.push({
      step: 'backup_restore_start',
      timestamp: Date.now(),
      message: 'Starting backup restore process'
    });
    
    // バックアップディレクトリの検索
    const backupDir = path.join(path.dirname(indexPaths[0]), '.backups');
    if (!fsSync.existsSync(backupDir)) {
      throw new Error('No backup directory found');
    }
    
    // 最新のバックアップを検索
    const backups = await this.findAvailableBackups(backupDir);
    if (backups.length === 0) {
      throw new Error('No backups available');
    }
    
    const latestBackup = backups[0]; // 最新のバックアップ
    
    recoveryResult.steps.push({
      step: 'backup_selected',
      timestamp: Date.now(),
      message: `Selected backup: ${latestBackup.timestamp}`,
      backup: latestBackup
    });
    
    // バックアップからファイルを復元
    for (const indexPath of indexPaths) {
      const fileName = path.basename(indexPath);
      const backupFilePath = path.join(latestBackup.path, fileName);
      
      if (fsSync.existsSync(backupFilePath)) {
        await fs.copyFile(backupFilePath, indexPath);
        recoveryResult.restoredFiles.push(indexPath);
        
        recoveryResult.steps.push({
          step: 'file_restored',
          timestamp: Date.now(),
          message: `Restored file: ${fileName}`,
          file: indexPath
        });
      }
    }
    
    recoveryResult.steps.push({
      step: 'backup_restore_complete',
      timestamp: Date.now(),
      message: 'Backup restore completed'
    });
  }

  /**
   * 部分修復
   */
  async executePartialRepair(indexPaths, recoveryResult) {
    recoveryResult.steps.push({
      step: 'partial_repair_start',
      timestamp: Date.now(),
      message: 'Starting partial repair process'
    });
    
    // 修復可能な問題の特定と修正
    const issues = recoveryResult.integrityResult.issues;
    
    for (const issue of issues) {
      try {
        switch (issue.type) {
          case 'METADATA_MISMATCH':
            await this.repairMetadataMismatch(issue, recoveryResult);
            break;
          case 'HIGH_DUPLICATION':
            await this.repairDuplication(issue, recoveryResult);
            break;
          case 'EMPTY_DATA':
            recoveryResult.steps.push({
              step: 'repair_skip',
              timestamp: Date.now(),
              message: `Cannot repair empty data issue: ${issue.message}`,
              issue: issue.type
            });
            break;
          default:
            recoveryResult.steps.push({
              step: 'repair_skip',
              timestamp: Date.now(),
              message: `No repair strategy for issue: ${issue.type}`,
              issue: issue.type
            });
        }
      } catch (error) {
        recoveryResult.errors.push(`Failed to repair ${issue.type}: ${error.message}`);
      }
    }
    
    recoveryResult.steps.push({
      step: 'partial_repair_complete',
      timestamp: Date.now(),
      message: 'Partial repair completed'
    });
  }

  /**
   * フル再構築
   */
  async executeFullRebuild(indexPaths, recoveryResult) {
    recoveryResult.steps.push({
      step: 'full_rebuild_start',
      timestamp: Date.now(),
      message: 'Starting full index rebuild'
    });
    
    // この実装では、実際の再構築プロセスは呼び出し元に委ねる
    // イベントを発火して外部システムに再構築を依頼
    this.emit('rebuild-requested', {
      indexPaths,
      reason: 'corruption_recovery',
      recoveryResult
    });
    
    recoveryResult.steps.push({
      step: 'rebuild_requested',
      timestamp: Date.now(),
      message: 'Index rebuild has been requested from external system'
    });
    
    // 実際の再構築は外部で行われるため、ここでは完了とする
    recoveryResult.steps.push({
      step: 'full_rebuild_delegated',
      timestamp: Date.now(),
      message: 'Full rebuild delegated to external system'
    });
  }

  /**
   * メタデータ不整合修復
   */
  async repairMetadataMismatch(issue, recoveryResult) {
    // 実際のチャンクファイルからメタデータを再生成
    if (issue.files && issue.files.length >= 2) {
      const metadataFile = issue.files.find(f => f.includes('metadata'));
      const chunksFile = issue.files.find(f => f.includes('chunks'));
      
      if (metadataFile && chunksFile && fsSync.existsSync(chunksFile)) {
        const chunksContent = await fs.readFile(chunksFile, 'utf8');
        const chunks = JSON.parse(chunksContent);
        
        let metadata = {};
        if (fsSync.existsSync(metadataFile)) {
          const metadataContent = await fs.readFile(metadataFile, 'utf8');
          metadata = JSON.parse(metadataContent);
        }
        
        // メタデータを実際のデータに合わせて更新
        metadata.totalChunks = chunks.length;
        metadata.lastUpdated = new Date().toISOString();
        metadata.repairedAt = Date.now();
        
        await fs.writeFile(metadataFile, JSON.stringify(metadata, null, 2));
        
        recoveryResult.steps.push({
          step: 'metadata_repaired',
          timestamp: Date.now(),
          message: `Repaired metadata mismatch: updated totalChunks to ${chunks.length}`,
          file: metadataFile
        });
      }
    }
  }

  /**
   * 重複データ修復
   */
  async repairDuplication(issue, recoveryResult) {
    if (issue.file) {
      const content = await fs.readFile(issue.file, 'utf8');
      const data = JSON.parse(content);
      
      if (Array.isArray(data)) {
        const originalLength = data.length;
        const uniqueData = [];
        const seen = new Set();
        
        for (const item of data) {
          const key = JSON.stringify(item);
          if (!seen.has(key)) {
            seen.add(key);
            uniqueData.push(item);
          }
        }
        
        if (uniqueData.length < originalLength) {
          await fs.writeFile(issue.file, JSON.stringify(uniqueData, null, 2));
          
          recoveryResult.steps.push({
            step: 'duplication_repaired',
            timestamp: Date.now(),
            message: `Removed ${originalLength - uniqueData.length} duplicate entries`,
            file: issue.file,
            originalLength,
            newLength: uniqueData.length
          });
        }
      }
    }
  }

  /**
   * 利用可能なバックアップの検索
   */
  async findAvailableBackups(backupDir) {
    try {
      const entries = await fs.readdir(backupDir);
      const backups = [];
      
      for (const entry of entries) {
        const entryPath = path.join(backupDir, entry);
        const stat = await fs.stat(entryPath);
        
        if (stat.isDirectory()) {
          // タイムスタンプからバックアップ情報を抽出
          const timestamp = entry.replace('backup_', '').replace(/[^0-9]/g, '');
          if (timestamp.length >= 8) { // YYYYMMDD形式以上
            backups.push({
              timestamp: entry,
              path: entryPath,
              created: stat.ctime,
              size: await this.calculateDirectorySize(entryPath)
            });
          }
        }
      }
      
      // 新しい順にソート
      return backups.sort((a, b) => b.created.getTime() - a.created.getTime());
      
    } catch (error) {
      return [];
    }
  }

  /**
   * ディレクトリサイズ計算
   */
  async calculateDirectorySize(dirPath) {
    let totalSize = 0;
    
    try {
      const entries = await fs.readdir(dirPath);
      
      for (const entry of entries) {
        const entryPath = path.join(dirPath, entry);
        const stat = await fs.stat(entryPath);
        
        if (stat.isFile()) {
          totalSize += stat.size;
        } else if (stat.isDirectory()) {
          totalSize += await this.calculateDirectorySize(entryPath);
        }
      }
    } catch (error) {
      // エラーの場合は0を返す
    }
    
    return totalSize;
  }

  /**
   * 自動バックアップ作成
   */
  async createBackup(indexPaths, metadata = {}) {
    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
    const backupDir = path.join(path.dirname(indexPaths[0]), '.backups', `backup_${timestamp}`);
    
    // バックアップディレクトリ作成
    await fs.mkdir(backupDir, { recursive: true });
    
    const backupResult = {
      timestamp,
      backupDir,
      files: [],
      metadata,
      size: 0
    };
    
    try {
      for (const indexPath of indexPaths) {
        if (fsSync.existsSync(indexPath)) {
          const fileName = path.basename(indexPath);
          const backupFilePath = path.join(backupDir, fileName);
          
          await fs.copyFile(indexPath, backupFilePath);
          
          const stat = await fs.stat(backupFilePath);
          backupResult.files.push({
            original: indexPath,
            backup: backupFilePath,
            size: stat.size
          });
          backupResult.size += stat.size;
        }
      }
      
      // バックアップメタデータ保存
      const backupMetadata = {
        timestamp,
        created: new Date().toISOString(),
        files: backupResult.files,
        totalSize: backupResult.size,
        metadata
      };
      
      await fs.writeFile(
        path.join(backupDir, 'backup_metadata.json'),
        JSON.stringify(backupMetadata, null, 2)
      );
      
      this.stats.backupsCreated++;
      this.state.lastBackupTime = Date.now();
      
      // 古いバックアップの削除
      await this.cleanupOldBackups(path.dirname(backupDir));
      
      this.emit('backup-created', backupResult);
      return backupResult;
      
    } catch (error) {
      // 失敗した場合はバックアップディレクトリを削除
      try {
        await fs.rmdir(backupDir, { recursive: true });
      } catch (cleanupError) {
        // クリーンアップエラーは無視
      }
      throw error;
    }
  }

  /**
   * 古いバックアップのクリーンアップ
   */
  async cleanupOldBackups(backupsDir) {
    try {
      const backups = await this.findAvailableBackups(backupsDir);
      
      if (backups.length > this.options.maxBackups) {
        const toDelete = backups.slice(this.options.maxBackups);
        
        for (const backup of toDelete) {
          await fs.rmdir(backup.path, { recursive: true });
        }
      }
    } catch (error) {
      // クリーンアップエラーはログのみ
      if (this.options.verboseLogging) {
        console.warn('Failed to cleanup old backups:', error.message);
      }
    }
  }

  /**
   * 定期的整合性チェック開始
   */
  startPeriodicChecks() {
    if (this.checkTimer) {
      clearInterval(this.checkTimer);
    }
    
    this.checkTimer = setInterval(() => {
      this.emit('periodic-check-due');
    }, this.options.checkInterval);
  }

  /**
   * 自動バックアップ開始
   */
  startAutoBackup() {
    if (this.backupTimer) {
      clearInterval(this.backupTimer);
    }
    
    this.backupTimer = setInterval(() => {
      this.emit('backup-due');
    }, this.options.backupInterval);
  }

  /**
   * チェック統計更新
   */
  updateCheckStats(checkTime) {
    const alpha = 0.1;
    this.stats.averageCheckTime = this.stats.averageCheckTime * (1 - alpha) + checkTime * alpha;
  }

  /**
   * 統計情報取得
   */
  getStats() {
    return {
      ...this.stats,
      state: this.state,
      timers: {
        periodicCheckEnabled: !!this.checkTimer,
        autoBackupEnabled: !!this.backupTimer
      }
    };
  }

  /**
   * クリーンアップ
   */
  cleanup() {
    if (this.checkTimer) {
      clearInterval(this.checkTimer);
      this.checkTimer = null;
    }
    
    if (this.backupTimer) {
      clearInterval(this.backupTimer);
      this.backupTimer = null;
    }
    
    this.emit('cleanup');
  }
}

// デフォルトインスタンス
export const defaultIndexIntegrityManager = new IndexIntegrityManager();