@sailboat-computer/data-storage/src/storage/UnifiedStorageManagerImpl.ts

Shared data storage library for sailboat computer v3

/**
 * Unified storage manager implementation
 */

import {
  UnifiedStorageManager,
  UnifiedStorageManagerOptions,
  StorageManager,
  StorageConfig,
  StorageStatus,
  LocalStorageStatus,
  ClearResult,
  SyncResult,
  ResilienceMetrics,
  DataQuery,
  StoredData,
  StorageTier,
  StorageTierType,
  CleanupResult,
  StorageOptions,
  HealthStatus,
  AlertSeverity,
  BatchPriority,
  BatchPriorityType
} from '../types';
import { StorageError, StorageErrorCode } from '../utils/errors';
import { createStorageManager } from './manager';
import { createLocalFileProvider } from './providers/local-file';
import { LocalFileProvider } from './providers/local-file';

/**
 * Default options for unified storage manager
 */
const DEFAULT_OPTIONS: UnifiedStorageManagerOptions = {
  localStorage: {
    directory: './data',
    maxSizeBytes: 1024 * 1024 * 100, // 100 MB
    encrypt: false,
    compressionLevel: 0
  },
  sync: {
    enabled: true,
    intervalMs: 60000, // 1 minute
    batchSize: 100
  }
};

/**
 * Unified storage manager implementation
 */
export class UnifiedStorageManagerImpl implements UnifiedStorageManager {
  private initialized = false;
  private remoteStorageManager: StorageManager;
  private localStorageProvider: LocalFileProvider;
  private syncInterval?: NodeJS.Timeout;
  private offlineBuffer: StoredData[] = [];
  private lastOperation: Date = new Date();
  private metrics = {
    operationsPerSecond: 0,
    averageLatency: 0,
    errorRate: 0,
    operationCount: 0,
    errorCount: 0,
    totalLatency: 0,
    syncCount: 0,
    syncSuccessCount: 0,
    syncFailureCount: 0,
    syncItemCount: 0,
    syncConflictCount: 0
  };
  
  /**
   * Create a new unified storage manager
   * 
   * @param config - Storage configuration
   * @param options - Storage manager options
   */
  constructor(
    private config: StorageConfig,
    private options: UnifiedStorageManagerOptions = DEFAULT_OPTIONS
  ) {
    // Set default options with non-nullable properties
    const defaultedOptions = {
      ...DEFAULT_OPTIONS,
      ...options
    };
    
    // Ensure localStorage and sync are not undefined
    this.options = {
      ...defaultedOptions,
      localStorage: {
        ...DEFAULT_OPTIONS.localStorage,
        ...(options?.localStorage || {})
      },
      sync: {
        ...DEFAULT_OPTIONS.sync,
        ...(options?.sync || {})
      }
    };
    
    // Create remote storage manager
    this.remoteStorageManager = createStorageManager(config, {
      retryAttempts: this.options.resilience?.retry?.maxRetries || 3,
      retryDelayMs: this.options.resilience?.retry?.baseDelayMs || 100,
      enableMetrics: true
    });
    
    // Create local storage provider
    this.localStorageProvider = createLocalFileProvider({
      directory: this.options.localStorage.directory,
      maxSizeBytes: this.options.localStorage.maxSizeBytes,
      encrypt: this.options.localStorage.encrypt,
      compressionLevel: this.options.localStorage.compressionLevel
    }) as LocalFileProvider;
  }
  
  /**
   * Initialize the storage manager
   */
  async initialize(): Promise<void> {
    if (this.initialized) {
      return;
    }
    
    try {
      // Initialize local storage provider
      await this.localStorageProvider.initialize();
      
      // Initialize remote storage manager
      try {
        await this.remoteStorageManager.initialize();
      } catch (error) {
        console.warn('Failed to initialize remote storage manager:', error);
        // Continue with local storage only
      }
      
      // Start sync interval if enabled
      if (this.options.sync.enabled) {
        this.startSyncInterval();
      }
      
      this.initialized = true;
      console.log('Unified storage manager initialized');
    } catch (error) {
      console.error('Failed to initialize unified storage manager:', error);
      throw new StorageError(
        StorageErrorCode.CONNECTION_FAILED,
        'Failed to initialize unified storage manager',
        { error }
      );
    }
  }
  
  /**
   * Store data
   * 
   * @param data - Data to store
   * @param category - Data category
   * @param options - Storage options
   * @returns Data ID
   */
  async store(data: any, category: string, options: StorageOptions = {}): Promise<string> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // Try to store in remote storage first
      try {
        const id = await this.remoteStorageManager.store(data, category, options);
        this.updateMetrics(Date.now() - startTime);
        return id;
      } catch (remoteError) {
        console.warn('Failed to store data in remote storage:', remoteError);
        
        // Store in local storage as fallback
        const id = await this.localStorageProvider.store(data, {
          category,
          timestamp: options.timestamp ? options.timestamp.toISOString() : new Date().toISOString(),
          tags: options.tags || {}
        });
        
        // Add to offline buffer for later sync
        this.offlineBuffer.push({
          data,
          metadata: {
            id,
            category,
            timestamp: options.timestamp ? options.timestamp.toISOString() : new Date().toISOString(),
            tags: options.tags || {}
          }
        });
        
        this.updateMetrics(Date.now() - startTime);
        return id;
      }
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.STORE_FAILED,
        'Failed to store data',
        { category, error }
      );
    }
  }
  
  /**
   * Retrieve data
   * 
   * @param query - Data query
   * @param tier - Storage tier to query (optional)
   * @returns Stored data
   */
  async retrieve(query: DataQuery, tier?: StorageTierType): Promise<StoredData[]> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // If tier is specified, query only that tier
      if (tier) {
        if (tier === StorageTier.HOT) {
          // Query local storage
          const results = await this.localStorageProvider.retrieve(query);
          this.updateMetrics(Date.now() - startTime);
          return results;
        } else {
          // Query remote storage
          try {
            const results = await this.remoteStorageManager.retrieve(query, tier);
            this.updateMetrics(Date.now() - startTime);
            return results;
          } catch (remoteError) {
            console.warn('Failed to retrieve data from remote storage:', remoteError);
            this.updateMetrics(Date.now() - startTime, true);
            return [];
          }
        }
      }
      
      // Query both local and remote storage
      let localResults: StoredData[] = [];
      let remoteResults: StoredData[] = [];
      
      // Query local storage
      localResults = await this.localStorageProvider.retrieve(query);
      
      // Query remote storage
      try {
        remoteResults = await this.remoteStorageManager.retrieve(query);
      } catch (remoteError) {
        console.warn('Failed to retrieve data from remote storage:', remoteError);
        // Continue with local results only
      }
      
      // Merge results, preferring local results for the same ID
      const resultsMap = new Map<string, StoredData>();
      
      // Add remote results first
      for (const result of remoteResults) {
        if (result.metadata.id) {
          resultsMap.set(result.metadata.id, result);
        }
      }
      
      // Add local results, overwriting remote results with the same ID
      for (const result of localResults) {
        if (result.metadata.id) {
          resultsMap.set(result.metadata.id, result);
        }
      }
      
      // Convert map to array
      const results = Array.from(resultsMap.values());
      
      // Apply sorting if specified
      if (query.sort) {
        results.sort((a, b) => {
          const aValue = query.sort?.field === 'timestamp'
            ? new Date(a.metadata.timestamp).getTime()
            : a.data[query.sort?.field || ''];
          
          const bValue = query.sort?.field === 'timestamp'
            ? new Date(b.metadata.timestamp).getTime()
            : b.data[query.sort?.field || ''];
          
          if (query.sort?.order === 'desc') {
            return bValue - aValue;
          } else {
            return aValue - bValue;
          }
        });
      }
      
      // Apply pagination if specified
      if (query.limit) {
        const offset = query.offset || 0;
        const limit = query.limit;
        
        this.updateMetrics(Date.now() - startTime);
        return results.slice(offset, offset + limit);
      }
      
      this.updateMetrics(Date.now() - startTime);
      return results;
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.RETRIEVE_FAILED,
        'Failed to retrieve data',
        { query, tier, error }
      );
    }
  }
  
  /**
   * Update data
   * 
   * @param data - Data to update
   */
  async update(data: StoredData): Promise<void> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // Try to update in remote storage first
      try {
        await this.remoteStorageManager.update(data);
      } catch (remoteError) {
        console.warn('Failed to update data in remote storage:', remoteError);
        
        // Update in local storage as fallback
        await this.localStorageProvider.update(data);
        
        // Add to offline buffer for later sync
        this.offlineBuffer.push(data);
      }
      
      // Also update in local storage to keep it in sync
      try {
        await this.localStorageProvider.update(data);
      } catch (localError) {
        console.warn('Failed to update data in local storage:', localError);
      }
      
      this.updateMetrics(Date.now() - startTime);
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.UPDATE_FAILED,
        'Failed to update data',
        { id: data.metadata.id, error }
      );
    }
  }
  
  /**
   * Delete data
   * 
   * @param id - Data ID
   */
  async delete(id: string): Promise<void> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // Try to delete from remote storage first
      try {
        await this.remoteStorageManager.delete(id);
      } catch (remoteError) {
        console.warn('Failed to delete data from remote storage:', remoteError);
      }
      
      // Also delete from local storage
      try {
        await this.localStorageProvider.delete(id);
      } catch (localError) {
        console.warn('Failed to delete data from local storage:', localError);
      }
      
      // Remove from offline buffer if present
      this.offlineBuffer = this.offlineBuffer.filter(item => item.metadata.id !== id);
      
      this.updateMetrics(Date.now() - startTime);
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.DELETE_FAILED,
        'Failed to delete data',
        { id, error }
      );
    }
  }
  
  /**
   * Store multiple data items
   * 
   * @param items - Data items to store
   * @returns Data IDs
   */
  async storeBatch(items: Array<{data: any, category: string, options?: StorageOptions}>): Promise<string[]> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // Try to store in remote storage first
      try {
        const ids = await this.remoteStorageManager.storeBatch(items);
        this.updateMetrics(Date.now() - startTime);
        return ids;
      } catch (remoteError) {
        console.warn('Failed to store batch data in remote storage:', remoteError);
        
        // Store in local storage as fallback
        const ids: string[] = [];
        
        for (const item of items) {
          const id = await this.localStorageProvider.store(item.data, {
            category: item.category,
            timestamp: item.options?.timestamp ? item.options.timestamp.toISOString() : new Date().toISOString(),
            tags: item.options?.tags || {}
          });
          
          ids.push(id);
          
          // Add to offline buffer for later sync
          this.offlineBuffer.push({
            data: item.data,
            metadata: {
              id,
              category: item.category,
              timestamp: item.options?.timestamp ? item.options.timestamp.toISOString() : new Date().toISOString(),
              tags: item.options?.tags || {}
            }
          });
        }
        
        this.updateMetrics(Date.now() - startTime);
        return ids;
      }
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.STORE_FAILED,
        'Failed to store batch data',
        { itemCount: items.length, error }
      );
    }
  }
  
  /**
   * Retrieve multiple data sets
   * 
   * @param queries - Data queries
   * @param tier - Storage tier to query (optional)
   * @returns Stored data sets
   */
  async retrieveBatch(queries: DataQuery[], tier?: StorageTierType): Promise<StoredData[][]> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      const results: StoredData[][] = [];
      
      for (const query of queries) {
        const queryResults = await this.retrieve(query, tier);
        results.push(queryResults);
      }
      
      this.updateMetrics(Date.now() - startTime);
      return results;
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.RETRIEVE_FAILED,
        'Failed to retrieve batch data',
        { queryCount: queries.length, error }
      );
    }
  }
  
  /**
   * Clean up old data
   * 
   * @param category - Data category
   * @param tier - Storage tier
   * @param retentionDays - Retention period in days
   * @returns Cleanup result
   */
  async cleanup(category: string, tier: StorageTierType, retentionDays: number): Promise<CleanupResult> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      if (tier === StorageTier.HOT) {
        // Clean up local storage
        return await this.localStorageProvider.cleanup(category, retentionDays);
      } else {
        // Clean up remote storage
        return await this.remoteStorageManager.cleanup(category, tier, retentionDays);
      }
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.CLEANUP_FAILED,
        'Failed to clean up data',
        { category, tier, retentionDays, error }
      );
    }
  }
  
  /**
   * Migrate data between tiers
   * 
   * @param id - Data ID
   * @param fromTier - Source tier
   * @param toTier - Destination tier
   */
  async migrate(id: string, fromTier: StorageTierType, toTier: StorageTierType): Promise<void> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      if (fromTier === StorageTier.HOT && toTier !== StorageTier.HOT) {
        // Migrate from local to remote
        const data = await this.localStorageProvider.retrieve({ id } as DataQuery);
        
        if (data.length === 0) {
          throw new Error(`Data with ID ${id} not found in local storage`);
        }
        
        // Store in remote storage
        await this.remoteStorageManager.store(data[0].data, data[0].metadata.category, {
          timestamp: new Date(data[0].metadata.timestamp),
          tags: data[0].metadata.tags,
          tier: toTier
        });
        
        // Delete from local storage
        await this.localStorageProvider.delete(id);
      } else if (fromTier !== StorageTier.HOT && toTier === StorageTier.HOT) {
        // Migrate from remote to local
        const data = await this.remoteStorageManager.retrieve({ id } as DataQuery, fromTier);
        
        if (data.length === 0) {
          throw new Error(`Data with ID ${id} not found in remote storage`);
        }
        
        // Store in local storage
        await this.localStorageProvider.store(data[0].data, {
          id,
          category: data[0].metadata.category,
          timestamp: data[0].metadata.timestamp,
          tags: data[0].metadata.tags
        });
        
        // Delete from remote storage
        await this.remoteStorageManager.delete(id);
      } else if (fromTier !== StorageTier.HOT && toTier !== StorageTier.HOT) {
        // Migrate between remote tiers
        await this.remoteStorageManager.migrate(id, fromTier, toTier);
      } else {
        // No migration needed
        console.log(`No migration needed for ${id} from ${fromTier} to ${toTier}`);
      }
      
      this.updateMetrics(Date.now() - startTime);
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.MIGRATION_FAILED,
        'Failed to migrate data',
        { id, fromTier, toTier, error }
      );
    }
  }
  
  /**
   * Shutdown the storage manager
   */
  async shutdown(): Promise<void> {
    if (!this.initialized) {
      return;
    }
    
    try {
      // Stop sync interval
      if (this.syncInterval) {
        clearInterval(this.syncInterval);
        this.syncInterval = undefined;
      }
      
      // Shutdown local storage provider
      await this.localStorageProvider.shutdown();
      
      // Shutdown remote storage manager
      await this.remoteStorageManager.shutdown();
      
      this.initialized = false;
      console.log('Unified storage manager closed');
    } catch (error) {
      console.error('Error closing unified storage manager:', error);
      throw new StorageError(
        StorageErrorCode.SYSTEM_ERROR,
        'Failed to close unified storage manager',
        { error }
      );
    }
  }
  
  /**
   * Get storage status
   * 
   * @returns Storage status
   */
  async getStatus(): Promise<StorageStatus> {
    this.ensureInitialized();
    
    try {
      // Get local storage status
      const localStatus = await this.getLocalStatus();
      
      // Get remote storage status
      let remoteStatus: StorageStatus | undefined;
      try {
        remoteStatus = await this.remoteStorageManager.getStatus();
      } catch (error) {
        console.warn('Failed to get remote storage status:', error);
      }
      
      // Combine statuses
      return {
        connected: remoteStatus?.connected || false,
        healthy: localStatus.healthy && (remoteStatus?.healthy || false),
        lastOperation: this.lastOperation,
        metrics: {
          operationsPerSecond: this.metrics.operationsPerSecond,
          averageLatency: this.metrics.averageLatency,
          errorRate: this.metrics.errorRate,
          storageUsed: localStatus.usedSizeBytes + (remoteStatus?.metrics?.storageUsed || 0),
          storageAvailable: localStatus.freeSizeBytes + (remoteStatus?.metrics?.storageAvailable || 0)
        },
        queryPerformance: {
          hot: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          warm: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          cold: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          pendingItems: this.offlineBuffer.length,
          averageBatchSize: 0,
          totalBatches: 0,
          failedBatches: 0,
          itemsRemoved: 0,
          bytesFreed: 0,
          duration: 0,
          hotStorageUsage: localStatus.usedSizeBytes,
          warmStorageUsage: 0,
          coldStorageUsage: 0,
          totalStorageUsage: localStatus.usedSizeBytes + (remoteStatus?.metrics?.storageUsed || 0),
          hotStorageCapacity: localStatus.totalSizeBytes,
          warmStorageCapacity: 0,
          coldStorageCapacity: 0,
          totalStorageCapacity: localStatus.totalSizeBytes + (remoteStatus?.metrics?.storageAvailable || 0),
          operationsPerSecond: this.metrics.operationsPerSecond,
          averageLatency: this.metrics.averageLatency,
          errorRate: this.metrics.errorRate,
          queryPerformance: {
            hot: {
              averageQueryTime: 0,
              queriesPerSecond: 0
            },
            warm: {
              averageQueryTime: 0,
              queriesPerSecond: 0
            },
            cold: {
              averageQueryTime: 0,
              queriesPerSecond: 0
            },
            overall: HealthStatus.HEALTHY,
            providers: {
              hot: HealthStatus.HEALTHY,
              warm: HealthStatus.HEALTHY,
              cold: HealthStatus.HEALTHY
            },
            issues: {
              severity: AlertSeverity.INFO,
              message: 'Unified storage manager status',
              component: 'unified-storage-manager',
              timestamp: new Date()
            },
            circuitBreaker: {
              state: {},
              failures: {},
              lastFailure: {}
            },
            retry: {
              retryCount: {},
              successCount: {},
              failureCount: {},
              averageDelay: {}
            },
            fallback: {
              fallbackCount: {},
              successCount: {},
              failureCount: {}
            },
            timeout: {
              timeoutCount: {},
              averageDuration: {}
            },
            bulkhead: {
              concurrentOperations: {},
              queueSize: {},
              rejectionCount: {}
            },
            rateLimit: {
              limitedCount: {},
              bypassCount: {},
              waitTimeMs: {},
              avgTokensPerSecond: {},
              currentTokens: {}
            }
          }
        }
      };
    } catch (error) {
      console.error('Failed to get unified storage status:', error);
      return {
        connected: false,
        healthy: false,
        error: `Failed to get unified storage status: ${error}`,
        queryPerformance: {
          hot: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          warm: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          cold: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          pendingItems: 0,
          averageBatchSize: 0,
          totalBatches: 0,
          failedBatches: 0,
          itemsRemoved: 0,
          bytesFreed: 0,
          duration: 0,
          hotStorageUsage: 0,
          warmStorageUsage: 0,
          coldStorageUsage: 0,
          totalStorageUsage: 0,
          hotStorageCapacity: 0,
          warmStorageCapacity: 0,
          coldStorageCapacity: 0,
          totalStorageCapacity: 0,
          operationsPerSecond: 0,
          averageLatency: 0,
          errorRate: 0,
          queryPerformance: {
            hot: {
              averageQueryTime: 0,
              queriesPerSecond: 0
            },
            warm: {
              averageQueryTime: 0,
              queriesPerSecond: 0
            },
            cold: {
              averageQueryTime: 0,
              queriesPerSecond: 0
            },
            overall: HealthStatus.UNHEALTHY,
            providers: {
              hot: HealthStatus.UNHEALTHY,
              warm: HealthStatus.UNHEALTHY,
              cold: HealthStatus.UNHEALTHY
            },
            issues: {
              severity: AlertSeverity.ALARM,
              message: `Failed to get unified storage status: ${error}`,
              component: 'unified-storage-manager',
              timestamp: new Date()
            },
            circuitBreaker: {
              state: {},
              failures: {},
              lastFailure: {}
            },
            retry: {
              retryCount: {},
              successCount: {},
              failureCount: {},
              averageDelay: {}
            },
            fallback: {
              fallbackCount: {},
              successCount: {},
              failureCount: {}
            },
            timeout: {
              timeoutCount: {},
              averageDuration: {}
            },
            bulkhead: {
              concurrentOperations: {},
              queueSize: {},
              rejectionCount: {}
            },
            rateLimit: {
              limitedCount: {},
              bypassCount: {},
              waitTimeMs: {},
              avgTokensPerSecond: {},
              currentTokens: {}
            }
          }
        }
      };
    }
  }
  
  /**
   * Get local storage status
   * 
   * @returns Local storage status
   */
  async getLocalStatus(): Promise<LocalStorageStatus> {
    this.ensureInitialized();
    
    try {
      // In a real implementation, this would get disk usage statistics
      
      // Get the maxSizeBytes with a default value to avoid undefined
      const maxSizeBytes = this.options.localStorage.maxSizeBytes || DEFAULT_OPTIONS.localStorage.maxSizeBytes || 0;
      
      return {
        available: true,
        healthy: true,
        totalSizeBytes: maxSizeBytes,
        usedSizeBytes: 0,
        freeSizeBytes: maxSizeBytes,
        itemCount: 0
      };
    } catch (error) {
      console.error('Failed to get local storage status:', error);
      return {
        available: false,
        healthy: false,
        totalSizeBytes: 0,
        usedSizeBytes: 0,
        freeSizeBytes: 0,
        itemCount: 0,
        lastError: `Failed to get local storage status: ${error}`
      };
    }
  }
  
  /**
   * Clear local cache
   * 
   * @returns Clear result
   */
  async clearLocalCache(): Promise<ClearResult> {
    this.ensureInitialized();
    
    try {
      // In a real implementation, this would delete all files from the local storage directory
      
      return {
        success: true,
        itemsRemoved: 0,
        bytesFreed: 0,
        durationMs: 0
      };
    } catch (error) {
      console.error('Failed to clear local cache:', error);
      return {
        success: false,
        itemsRemoved: 0,
        bytesFreed: 0,
        durationMs: 0,
        error: `Failed to clear local cache: ${error}`
      };
    }
  }
  
  /**
   * Synchronize local and remote storage
   * 
   * @returns Sync result
   */
  async synchronize(): Promise<SyncResult> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    
    try {
      // Check if there are items in the offline buffer
      if (this.offlineBuffer.length === 0) {
        return {
          success: true,
          itemCount: 0,
          conflictCount: 0,
          failedCount: 0,
          durationMs: Date.now() - startTime
        };
      }
      
      // Get items to sync
      const batchSize = this.options.sync.batchSize;
      const itemsToSync = this.offlineBuffer.slice(0, batchSize);
      
      // Track sync metrics
      this.metrics.syncCount++;
      this.metrics.syncItemCount += itemsToSync.length;
      
      // Sync each item
      let successCount = 0;
      let conflictCount = 0;
      let failedCount = 0;
      
      for (const item of itemsToSync) {
        try {
          // Check if item exists in remote storage
          const remoteItems = await this.remoteStorageManager.retrieve({ id: item.metadata.id } as DataQuery);
          
          if (remoteItems.length > 0) {
            // Item exists in remote storage, check for conflicts
            const remoteItem = remoteItems[0];
                        const remoteTimestamp = new Date(remoteItem.metadata.timestamp).getTime();
            const localTimestamp = new Date(item.metadata.timestamp).getTime();
            
            if (remoteTimestamp > localTimestamp) {
              // Remote item is newer, keep it
              conflictCount++;
              continue;
            }
            
            // Update remote item with local data
            await this.remoteStorageManager.update(item);
            successCount++;
          } else {
            // Item doesn't exist in remote storage, create it
            await this.remoteStorageManager.store(item.data, item.metadata.category, {
              timestamp: new Date(item.metadata.timestamp),
              tags: item.metadata.tags
            });
            successCount++;
          }
          
          // Remove from offline buffer
          this.offlineBuffer = this.offlineBuffer.filter(bufferItem => 
            bufferItem.metadata.id !== item.metadata.id
          );
        } catch (error) {
          console.error('Failed to sync item:', error);
          failedCount++;
        }
      }
      
      // Update sync metrics
      this.metrics.syncSuccessCount += successCount;
      this.metrics.syncFailureCount += failedCount;
      this.metrics.syncConflictCount += conflictCount;
      
      return {
        success: failedCount === 0,
        itemCount: itemsToSync.length,
        conflictCount,
        failedCount,
        durationMs: Date.now() - startTime
      };
    } catch (error) {
      console.error('Failed to synchronize storage:', error);
      return {
        success: false,
        itemCount: 0,
        conflictCount: 0,
        failedCount: 0,
        durationMs: Date.now() - startTime,
        error: `Failed to synchronize storage: ${error}`
      };
    }
  }
  
  /**
   * Start sync interval
   */
  private startSyncInterval(): void {
    if (this.syncInterval) {
      clearInterval(this.syncInterval);
    }
    
    const intervalMs = this.options.sync.intervalMs;
    
    this.syncInterval = setInterval(async () => {
      try {
        await this.synchronize();
      } catch (error) {
        console.error('Error during scheduled sync:', error);
      }
    }, intervalMs);
    
    console.log(`Sync interval started (${intervalMs}ms)`);
  }
  
  /**
   * Update metrics
   * 
   * @param latencyMs - Operation latency in milliseconds
   * @param isError - Whether the operation resulted in an error
   */
  private updateMetrics(latencyMs: number, isError: boolean = false): void {
    this.lastOperation = new Date();
    this.metrics.operationCount++;
    this.metrics.totalLatency += latencyMs;
    
    if (isError) {
      this.metrics.errorCount++;
    }
    
    // Calculate averages
    this.metrics.averageLatency = this.metrics.totalLatency / this.metrics.operationCount;
    this.metrics.errorRate = this.metrics.errorCount / this.metrics.operationCount;
    
    // Calculate operations per second (over the last minute)
    const oneMinuteAgo = Date.now() - 60000;
    this.metrics.operationsPerSecond = this.metrics.operationCount / 60;
  }
  
  /**
   * Get resilience metrics
   * 
   * @returns Resilience metrics
   */
  getResilienceMetrics(): ResilienceMetrics {
    return {
      circuitBreaker: {
        state: {},
        failures: {},
        lastFailure: {}
      },
      retry: {
        retryCount: {},
        successCount: {},
        failureCount: {},
        averageDelay: {}
      },
      fallback: {
        fallbackCount: {},
        successCount: {},
        failureCount: {}
      },
      timeout: {
        timeoutCount: {},
        averageDuration: {}
      },
      bulkhead: {
        concurrentOperations: {},
        queueSize: {},
        rejectionCount: {}
      },
      rateLimit: {
        limitedCount: {},
        bypassCount: {},
        waitTimeMs: {},
        avgTokensPerSecond: {},
        currentTokens: {}
      }
    };
  }
  
  /**
   * Ensure provider is initialized
   * 
   * @throws StorageError if not initialized
   */
  private ensureInitialized(): void {
    if (!this.initialized) {
      throw new StorageError(
        StorageErrorCode.CONNECTION_FAILED,
        'Unified storage manager not initialized',
        {}
      );
    }
  }
}

/**
 * Create a new unified storage manager
 * 
 * @param config - Storage configuration
 * @param options - Storage manager options
 * @returns Unified storage manager
 */
export function createUnifiedStorageManager(
  config: StorageConfig,
  options?: UnifiedStorageManagerOptions
): UnifiedStorageManager {
  return new UnifiedStorageManagerImpl(config, options);
}