@sailboat-computer/data-storage/src/storage/manager.ts

Shared data storage library for sailboat computer v3

/**
 * Storage manager implementation
 */

import { v4 as uuidv4 } from 'uuid';
import {
  StorageManager,
  StorageProvider,
  StorageConfig,
  StorageOptions,
  StorageStatus,
  StorageTier,
  StorageTierType,
  DataQuery,
  StoredData,
  CleanupResult,
  StorageManagerOptions,
  BatchPriority,
  BatchPriorityType,
  HealthStatus,
  AlertSeverity
} from '../types';
import { StorageError, StorageErrorCode, withRetry } from '../utils/errors';
import { createRedisProvider } from './providers/redis';
import { createInfluxDBProvider } from './providers/influxdb';
import { createPostgreSQLProvider } from './providers/postgresql';

/**
 * Default storage manager options
 */
const DEFAULT_OPTIONS: StorageManagerOptions = {
  retryAttempts: 3,
  retryDelayMs: 100,
  enableMetrics: true,
  metricsInterval: 60000, // 1 minute
  logger: {
    level: 'info',
    console: true
  }
};

/**
 * Storage manager implementation
 */
export class StorageManagerImpl implements StorageManager {
  private providers: Partial<Record<StorageTierType, StorageProvider>> = {};
  private initialized = false;
  private metrics = {
    operationsPerSecond: 0,
    averageLatency: 0,
    errorRate: 0,
    operationCount: 0,
    errorCount: 0,
    totalLatency: 0
  };
  private metricsInterval?: NodeJS.Timeout;
  
  /**
   * Create a new storage manager
   * 
   * @param config - Storage configuration
   * @param options - Storage manager options
   */
  constructor(
    private config: StorageConfig,
    private options: StorageManagerOptions = DEFAULT_OPTIONS
  ) {}
  
  /**
   * Initialize the storage manager
   */
  async initialize(): Promise<void> {
    if (this.initialized) {
      return;
    }
    
    try {
      // Initialize providers
      if (this.config.providers.hot) {
        this.providers[StorageTier.HOT] = createRedisProvider(this.config.providers.hot.config);
      }
      
      if (this.config.providers.warm) {
        this.providers[StorageTier.WARM] = createInfluxDBProvider(this.config.providers.warm.config);
      }
      
      if (this.config.providers.cold) {
        this.providers[StorageTier.COLD] = createPostgreSQLProvider(this.config.providers.cold.config);
      }
      
      // Initialize each provider
      for (const [tier, provider] of Object.entries(this.providers)) {
        try {
          await provider.initialize();
          console.log(`Initialized ${tier} storage provider`);
        } catch (error) {
          console.error(`Failed to initialize ${tier} storage provider:`, error);
          throw new StorageError(
            StorageErrorCode.CONNECTION_FAILED,
            `Failed to initialize ${tier} storage provider`,
            { tier, error }
          );
        }
      }
      
      // Start metrics collection if enabled
      if (this.options.enableMetrics) {
        this.startMetricsCollection();
      }
      
      this.initialized = true;
      console.log('Storage manager initialized');
    } catch (error) {
      console.error('Failed to initialize storage manager:', error);
      throw new StorageError(
        StorageErrorCode.CONNECTION_FAILED,
        'Failed to initialize storage manager',
        { error }
      );
    }
  }
  
  /**
   * Store data
   * 
   * @param data - Data to store
   * @param category - Data category
   * @param options - Storage options
   * @returns Data ID
   */
  async store(data: any, category: string, options: StorageOptions = {}): Promise<string> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // Determine storage tier
      const tier = options.tier || StorageTier.HOT;
      const provider = this.getProvider(tier);
      
      // Prepare metadata
      const metadata = {
        id: uuidv4(),
        category,
        timestamp: options.timestamp ? options.timestamp.toISOString() : new Date().toISOString(),
        tags: options.tags || {},
        tier
      };
      
      // Store data with retry
      const id = await withRetry(
        () => provider.store(data, metadata),
        this.options.retryAttempts,
        this.options.retryDelayMs
      );
      
      this.updateMetrics(Date.now() - startTime);
      return id;
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.STORE_FAILED,
        'Failed to store data',
        { category, error }
      );
    }
  }
  
  /**
   * Retrieve data
   * 
   * @param query - Data query
   * @param tier - Storage tier to query (optional)
   * @returns Stored data
   */
  async retrieve(query: DataQuery, tier?: StorageTierType): Promise<StoredData[]> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      if (tier) {
        // Query specific tier
        const provider = this.getProvider(tier);
        const results = await withRetry(
          () => provider.retrieve(query),
          this.options.retryAttempts,
          this.options.retryDelayMs
        );
        
        this.updateMetrics(Date.now() - startTime);
        return results;
      } else {
        // Query all tiers, starting with hot and moving to colder tiers
        const tiers = [StorageTier.HOT, StorageTier.WARM, StorageTier.COLD];
        let results: StoredData[] = [];
        
        for (const t of tiers) {
          if (this.providers[t]) {
            try {
              const tierResults = await this.providers[t]!.retrieve(query);
              results = [...results, ...tierResults];
              
              // If we found results and the query has a limit, check if we've reached it
              if (results.length > 0 && query.limit && results.length >= query.limit) {
                results = results.slice(0, query.limit);
                break;
              }
            } catch (err) {
              console.warn(`Error retrieving from ${t} tier:`, err);
              // Continue to next tier on error
            }
          }
        }
        
        this.updateMetrics(Date.now() - startTime);
        return results;
      }
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.RETRIEVE_FAILED,
        'Failed to retrieve data',
        { query, tier, error }
      );
    }
  }
  
  /**
   * Update stored data
   * 
   * @param data - Data to update
   */
  async update(data: StoredData): Promise<void> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      const tier = data.metadata.tier || StorageTier.HOT;
      const provider = this.getProvider(tier);
      
      await withRetry(
        () => provider.update(data),
        this.options.retryAttempts,
        this.options.retryDelayMs
      );
      
      this.updateMetrics(Date.now() - startTime);
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.UPDATE_FAILED,
        'Failed to update data',
        { id: data.metadata.id, error }
      );
    }
  }
  
  /**
   * Delete data
   * 
   * @param id - Data ID
   */
  async delete(id: string): Promise<void> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // Try to delete from all tiers
      const tiers = [StorageTier.HOT, StorageTier.WARM, StorageTier.COLD];
      let deleted = false;
      
      for (const tier of tiers) {
        if (this.providers[tier]) {
          try {
            await this.providers[tier]!.delete(id);
            deleted = true;
          } catch (err) {
            // Continue to next tier on error
            console.warn(`Error deleting from ${tier} tier:`, err);
          }
        }
      }
      
      if (!deleted) {
        throw new Error('Data not found in any tier');
      }
      
      this.updateMetrics(Date.now() - startTime);
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.DELETE_FAILED,
        'Failed to delete data',
        { id, error }
      );
    }
  }
  
  /**
   * Store multiple data items
   * 
   * @param items - Data items to store
   * @returns Data IDs
   */
  async storeBatch(items: Array<{data: any, category: string, options?: StorageOptions}>): Promise<string[]> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // Group items by tier
      const tierGroups: Record<string, Array<{data: any, metadata: any}>> = {
        'storage_hot': [],
        'storage_warm': [],
        'storage_cold': []
      };
      
      // Prepare items for each tier
      for (const item of items) {
        const tier = item.options?.tier || StorageTier.HOT;
        
        tierGroups[tier].push({
          data: item.data,
          metadata: {
            id: uuidv4(),
            category: item.category,
            timestamp: item.options?.timestamp ? item.options.timestamp.toISOString() : new Date().toISOString(),
            tags: item.options?.tags || {},
            tier
          }
        });
      }
      
      // Store items for each tier
      const results: string[] = [];
      
      for (const [tier, tierItems] of Object.entries(tierGroups)) {
        if (tierItems.length > 0 && this.providers[tier as StorageTierType]) {
          const provider = this.providers[tier as StorageTierType]!;
          const tierResults = await withRetry(
            () => provider.storeBatch(tierItems),
            this.options.retryAttempts,
            this.options.retryDelayMs
          );
          
          results.push(...tierResults);
        }
      }
      
      this.updateMetrics(Date.now() - startTime);
      return results;
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.STORE_FAILED,
        'Failed to store batch data',
        { itemCount: items.length, error }
      );
    }
  }
  
  /**
   * Retrieve multiple data sets
   * 
   * @param queries - Data queries
   * @param tier - Storage tier to query (optional)
   * @returns Stored data sets
   */
  async retrieveBatch(queries: DataQuery[], tier?: StorageTierType): Promise<StoredData[][]> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      const results: StoredData[][] = [];
      
      for (const query of queries) {
        const queryResults = await this.retrieve(query, tier);
        results.push(queryResults);
      }
      
      this.updateMetrics(Date.now() - startTime);
      return results;
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.RETRIEVE_FAILED,
        'Failed to retrieve batch data',
        { queryCount: queries.length, error }
      );
    }
  }
  
  /**
   * Clean up old data
   * 
   * @param category - Data category
   * @param tier - Storage tier
   * @param retentionDays - Retention period in days
   * @returns Cleanup result
   */
  async cleanup(category: string, tier: StorageTierType, retentionDays: number): Promise<CleanupResult> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      const provider = this.getProvider(tier);
      
      const result = await withRetry(
        () => provider.cleanup(category, retentionDays),
        this.options.retryAttempts,
        this.options.retryDelayMs
      );
      
      this.updateMetrics(Date.now() - startTime);
      return result;
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.CLEANUP_FAILED,
        'Failed to clean up data',
        { category, tier, retentionDays, error }
      );
    }
  }
  
  /**
   * Migrate data between tiers
   * 
   * @param id - Data ID
   * @param fromTier - Source tier
   * @param toTier - Destination tier
   */
  async migrate(id: string, fromTier: StorageTierType, toTier: StorageTierType): Promise<void> {
    this.ensureInitialized();
    
    const startTime = Date.now();
    let error: Error | null = null;
    
    try {
      // Get source provider
      const sourceProvider = this.getProvider(fromTier);
      
      // Get destination provider
      const destProvider = this.getProvider(toTier);
      
      // Retrieve data from source
      const data = await sourceProvider.retrieve({ id } as DataQuery);
      
      if (data.length === 0) {
        throw new Error(`Data with ID ${id} not found in ${fromTier} tier`);
      }
      
      // Store data in destination
      const item = data[0];
      
      // Update tier in metadata
      const updatedItem = {
        ...item,
        metadata: {
          ...item.metadata,
          tier: toTier
        }
      };
      
      // Store in destination
      await destProvider.store(updatedItem.data, updatedItem.metadata);
      
      // Delete from source
      await sourceProvider.delete(id);
      
      this.updateMetrics(Date.now() - startTime);
    } catch (err) {
      error = err instanceof Error ? err : new Error(String(err));
      this.updateMetrics(Date.now() - startTime, true);
      
      throw new StorageError(
        StorageErrorCode.MIGRATION_FAILED,
        'Failed to migrate data',
        { id, fromTier, toTier, error }
      );
    }
  }
  
  /**
   * Shutdown the storage manager
   */
  async shutdown(): Promise<void> {
    if (!this.initialized) {
      return;
    }
    
    try {
      // Stop metrics collection
      if (this.metricsInterval) {
        clearInterval(this.metricsInterval);
        this.metricsInterval = undefined;
      }
      
      // Close all providers
      for (const [tier, provider] of Object.entries(this.providers)) {
        try {
          await provider.shutdown();
          console.log(`Closed ${tier} storage provider`);
        } catch (error) {
          console.error(`Error closing ${tier} storage provider:`, error);
        }
      }
      
      this.initialized = false;
      console.log('Storage manager closed');
    } catch (error) {
      console.error('Error closing storage manager:', error);
      throw new StorageError(
        StorageErrorCode.SYSTEM_ERROR,
        'Failed to close storage manager',
        { error }
      );
    }
  }
  
  /**
   * Get storage status
   * 
   * @returns Storage status
   */
  async getStatus(): Promise<StorageStatus> {
    this.ensureInitialized();
    
    const status: StorageStatus = {
      connected: true,
      healthy: true,
      lastOperation: new Date(),
      metrics: {
        operationsPerSecond: this.metrics.operationsPerSecond,
        averageLatency: this.metrics.averageLatency,
        errorRate: this.metrics.errorRate,
        storageUsed: 0,
        storageAvailable: 0
      },
      queryPerformance: {
        hot: {
          averageQueryTime: 0,
          queriesPerSecond: 0
        },
        warm: {
          averageQueryTime: 0,
          queriesPerSecond: 0
        },
        cold: {
          averageQueryTime: 0,
          queriesPerSecond: 0
        },
        pendingItems: 0,
        averageBatchSize: 0,
        totalBatches: 0,
        failedBatches: 0,
        itemsRemoved: 0,
        bytesFreed: 0,
        duration: 0,
        hotStorageUsage: 0,
        warmStorageUsage: 0,
        coldStorageUsage: 0,
        totalStorageUsage: 0,
        hotStorageCapacity: 0,
        warmStorageCapacity: 0,
        coldStorageCapacity: 0,
        totalStorageCapacity: 0,
        operationsPerSecond: this.metrics.operationsPerSecond,
        averageLatency: this.metrics.averageLatency,
        errorRate: this.metrics.errorRate,
        queryPerformance: {
          hot: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          warm: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          cold: {
            averageQueryTime: 0,
            queriesPerSecond: 0
          },
          overall: HealthStatus.HEALTHY,
          providers: {
            hot: HealthStatus.HEALTHY,
            warm: HealthStatus.HEALTHY,
            cold: HealthStatus.HEALTHY
          },
          issues: {
            severity: AlertSeverity.INFO,
            message: 'Storage manager status',
            component: 'storage-manager',
            timestamp: new Date()
          },
          circuitBreaker: {
            state: {},
            failures: {},
            lastFailure: {}
          },
          retry: {
            retryCount: {},
            successCount: {},
            failureCount: {},
            averageDelay: {}
          },
          fallback: {
            fallbackCount: {},
            successCount: {},
            failureCount: {}
          },
          timeout: {
            timeoutCount: {},
            averageDuration: {}
          },
          bulkhead: {
            concurrentOperations: {},
            queueSize: {},
            rejectionCount: {}
          },
          rateLimit: {
            limitedCount: {},
            bypassCount: {},
            waitTimeMs: {},
            avgTokensPerSecond: {},
            currentTokens: {}
          }
        }
      }
    };
    
    // Check provider status
    for (const [tier, provider] of Object.entries(this.providers)) {
      try {
        const providerStatus = await provider.getStatus();
        
        if (!providerStatus.connected || !providerStatus.healthy) {
          status.healthy = false;
          status.error = `${tier} provider is unhealthy: ${providerStatus.error}`;
        }
      } catch (error) {
        status.healthy = false;
        status.error = `Failed to get ${tier} provider status: ${error}`;
      }
    }
    
    return status;
  }
  
  /**
   * Get provider for tier
   * 
   * @param tier - Storage tier
   * @returns Storage provider
   * @throws StorageError if provider not found
   */
  private getProvider(tier: StorageTierType): StorageProvider {
    const provider = this.providers[tier];
    
    if (!provider) {
      throw new StorageError(
        StorageErrorCode.INVALID_CONFIG,
        `No provider configured for ${tier} tier`,
        { tier }
      );
    }
    
    return provider;
  }
  
  /**
   * Ensure storage manager is initialized
   * 
   * @throws StorageError if not initialized
   */
  private ensureInitialized(): void {
    if (!this.initialized) {
      throw new StorageError(
        StorageErrorCode.SYSTEM_ERROR,
        'Storage manager not initialized',
        {}
      );
    }
  }
  
  /**
   * Update metrics
   * 
   * @param latency - Operation latency in milliseconds
   * @param isError - Whether operation resulted in error
   */
  private updateMetrics(latency: number, isError = false): void {
    this.metrics.operationCount++;
    this.metrics.totalLatency += latency;
    
    if (isError) {
      this.metrics.errorCount++;
    }
    
    this.metrics.averageLatency = this.metrics.totalLatency / this.metrics.operationCount;
    this.metrics.errorRate = this.metrics.errorCount / this.metrics.operationCount;
  }
  
  /**
   * Start metrics collection
   */
  private startMetricsCollection(): void {
    this.metricsInterval = setInterval(() => {
      // Calculate operations per second
      if (this.options.metricsInterval) {
        this.metrics.operationsPerSecond = this.metrics.operationCount / (this.options.metricsInterval / 1000);
      }
      
      // Reset counters
      this.metrics.operationCount = 0;
      this.metrics.errorCount = 0;
      this.metrics.totalLatency = 0;
    }, this.options.metricsInterval);
  }
}

/**
 * Create a new storage manager
 * 
 * @param config - Storage configuration
 * @param options - Storage manager options
 * @returns Storage manager
 */
export function createStorageManager(
  config: StorageConfig,
  options?: StorageManagerOptions
): StorageManager {
  // Config validation can be added here if needed
  return new StorageManagerImpl(config, options);
}