@sailboat-computer/data-storage/src/config/StorageConfigManager.ts

Shared data storage library for sailboat computer v3

/**
 * Storage Configuration Manager
 * 
 * This implementation uses the @sailboat-computer/config-client package to interact
 * with the Configuration Service, with fallback to local files when the service is unavailable.
 */

import { createConfigClient, ConfigClient, ConfigChangeEvent } from '@sailboat-computer/config-client';
import { EventBus, EventCategory } from '../event-bus';
import { createLogger } from '../utils/logger';
import { StorageConfig, UnifiedStorageManagerOptions } from '../types';
import { IStorageConfigManager } from './IStorageConfigManager';
import path from 'path';
import fs from 'fs/promises';
import { loadConfig, saveConfig } from '@sailboat-computer/config-loader';

// Create logger
const logger = createLogger('storage-config-manager');

/**
 * Default storage configuration
 */
export const DEFAULT_STORAGE_CONFIG: StorageConfig = {
  providers: {
    hot: {
      type: 'redis',
      config: {
        host: 'localhost',
        port: 6379,
        password: '',
        db: 0,
        tls: false
      }
    },
    batching: {
      timeBased: {
        enabled: true,
        intervalMs: 5000
      },
      sizeBased: {
        enabled: true,
        maxBatchSize: 100
      },
      priorityOverride: true
    }
  }
};

/**
 * Default unified storage manager options
 */
export const DEFAULT_UNIFIED_STORAGE_MANAGER_OPTIONS: UnifiedStorageManagerOptions = {
  localStorage: {
    directory: './data',
    maxSizeBytes: 104857600,
    encrypt: false,
    compressionLevel: 6
  },
  sync: {
    enabled: true,
    intervalMs: 60000,
    batchSize: 100
  },
  resilience: {
    circuitBreaker: {
      failureThreshold: 3,
      resetTimeoutMs: 30000
    },
    retry: {
      maxRetries: 3,
      baseDelayMs: 1000
    },
    timeout: {
      defaultTimeoutMs: 5000,
      operationTimeouts: {
        store: 10000,
        retrieve: 15000,
        update: 10000,
        delete: 5000,
        storeBatch: 30000,
        retrieveBatch: 30000,
        cleanup: 60000,
        migrate: 60000
      }
    },
    bulkhead: {
      maxConcurrentOperations: 10,
      maxQueueSize: 100
    },
    rateLimit: {
      defaultTokensPerSecond: 50,
      defaultBucketSize: 100,
      tokensPerSecond: 50,
      bucketSize: 100,
      waitWhenLimited: true,
      maxWaitTimeMs: 5000,
      bypassOperations: ['retrieve']
    }
  },
  logger: {
    level: 'info',
    console: true
  }
};

/**
 * Storage configuration manager options
 */
export interface StorageConfigManagerOptions {
  /**
   * Configuration service URL
   * If not provided, will use CONFIG_SERVICE_URL environment variable or default to http://localhost:3000
   */
  serviceUrl?: string;
  
  /**
   * Local configuration directory
   * If not provided, will use './config'
   */
  localDir?: string;
  
  /**
   * Environment to use
   * If not provided, will use 'production'
   */
  environment?: string;
  
  /**
   * Service name
   * If not provided, will use 'data-manager'
   */
  serviceName?: string;
  
  /**
   * Configuration ID
   * If not provided, will use 'config'
   */
  configId?: string;
  
  /**
   * Custom logger
   */
  logger?: {
    info: (message: string, meta?: any) => void;
    warn: (message: string, meta?: any) => void;
    error: (message: string, meta?: any) => void;
    debug: (message: string, meta?: any) => void;
  };
}

/**
 * Storage configuration manager
 */
export class StorageConfigManager implements IStorageConfigManager {
  private storageConfig: StorageConfig;
  private unifiedStorageManagerOptions: UnifiedStorageManagerOptions;
  private configClient: ConfigClient;
  private eventBus?: EventBus;
  private stopWatching?: () => void;
  private localDir: string;
  private serviceName: string;
  private configId: string;
  private environment: string;
  
  /**
   * Create a new storage configuration manager
   * @param options Configuration options
   */
  constructor(options: StorageConfigManagerOptions = {}) {
    this.storageConfig = DEFAULT_STORAGE_CONFIG;
    this.unifiedStorageManagerOptions = DEFAULT_UNIFIED_STORAGE_MANAGER_OPTIONS;
    this.localDir = options.localDir || './config';
    this.serviceName = options.serviceName || 'data-manager';
    this.configId = options.configId || 'config';
    this.environment = options.environment || 'production';
    
    const serviceUrl = options.serviceUrl || 
                       process.env.CONFIG_SERVICE_URL || 
                       'http://localhost:3000';
    
    // Create config client
    this.configClient = createConfigClient({
      serviceUrl,
      serviceName: this.serviceName,
      fallbackDir: this.localDir,
      environment: this.environment,
      resilience: {
        timeoutMs: 5000,
        circuitBreaker: {
          failureThreshold: 3,
          resetTimeoutMs: 30000,
        },
        retry: {
          maxRetries: 3,
          baseDelayMs: 1000,
          maxDelayMs: 10000,
        },
      },
      logger: options.logger || logger,
    });
    
    logger.info('Storage configuration manager initialized', {
      serviceUrl,
      localDir: this.localDir,
      environment: this.environment,
      serviceName: this.serviceName,
      configId: this.configId
    });
  }
  
  /**
   * Initialize the configuration manager
   */
  async initialize(): Promise<void> {
    try {
      // Try to load configuration from service first
      try {
        const config = await this.configClient.getConfig<any>(this.configId);
        
        // Extract storage config and unified storage manager options
        this.extractConfig(config);
        
        logger.info('Configuration loaded from service');
        
        // Cache the configuration locally
        await this.cacheConfigLocally(config);
      } catch (serviceError) {
        logger.warn('Failed to load configuration from service, falling back to local file', { 
          error: (serviceError as Error).message 
        });
        
        // Fall back to local file
        try {
          const config = await loadConfig<any>(
            `${this.serviceName}/${this.configId}.json`,
            this.createFullConfig(),
            { baseDir: this.localDir }
          );
          
          // Extract storage config and unified storage manager options
          this.extractConfig(config);
          
          logger.info('Configuration loaded from local file');
        } catch (localError) {
          logger.warn('Failed to load configuration from local file, using default', { 
            error: (localError as Error).message 
          });
          
          // Use default configuration
          this.storageConfig = DEFAULT_STORAGE_CONFIG;
          this.unifiedStorageManagerOptions = DEFAULT_UNIFIED_STORAGE_MANAGER_OPTIONS;
          
          // Cache the default configuration
          await this.cacheConfigLocally(this.createFullConfig());
        }
      }
      
      // Start watching for configuration changes
      this.startWatching();
      
      logger.info('Configuration manager initialized successfully');
    } catch (error) {
      logger.error('Failed to initialize configuration manager', { error: (error as Error).message });
      throw error;
    }
  }
  
  /**
   * Extract storage config and unified storage manager options from full config
   * @param config Full configuration
   */
  private extractConfig(config: any): void {
    // Extract storage config
    this.storageConfig = {
      providers: config.providers || DEFAULT_STORAGE_CONFIG.providers
    };
    
    // Extract unified storage manager options
    this.unifiedStorageManagerOptions = {
      localStorage: config.localStorage || DEFAULT_UNIFIED_STORAGE_MANAGER_OPTIONS.localStorage,
      sync: config.sync || DEFAULT_UNIFIED_STORAGE_MANAGER_OPTIONS.sync,
      resilience: config.resilience || DEFAULT_UNIFIED_STORAGE_MANAGER_OPTIONS.resilience,
      logger: config.logger || DEFAULT_UNIFIED_STORAGE_MANAGER_OPTIONS.logger
    };
  }
  
  /**
   * Create full configuration from storage config and unified storage manager options
   * @returns Full configuration
   */
  private createFullConfig(): any {
    return {
      providers: this.storageConfig.providers,
      localStorage: this.unifiedStorageManagerOptions.localStorage,
      sync: this.unifiedStorageManagerOptions.sync,
      resilience: this.unifiedStorageManagerOptions.resilience,
      logger: this.unifiedStorageManagerOptions.logger,
      features: {
        useConfigService: true,
        useLocalFallback: true,
        enableVersioning: true,
        enableSchemaValidation: true
      }
    };
  }
  
  /**
   * Cache configuration locally
   * @param config Configuration to cache
   */
  private async cacheConfigLocally(config: any): Promise<void> {
    try {
      // Ensure directory exists
      const configDir = path.join(this.localDir, this.serviceName);
      await fs.mkdir(configDir, { recursive: true });
      
      // Save configuration to file
      await saveConfig(
        `${this.serviceName}/${this.configId}.json`,
        config,
        'Cached from configuration service',
        undefined,
        { baseDir: this.localDir }
      );
      
      logger.debug('Configuration cached locally');
    } catch (error) {
      logger.error('Failed to cache configuration locally', { error: (error as Error).message });
      // Don't throw error, as this is a non-critical operation
    }
  }
  
  /**
   * Start watching for configuration changes
   */
  private startWatching(): void {
    // Stop any existing watcher
    if (this.stopWatching) {
      this.stopWatching();
    }
    
    // Start watching for configuration changes
    this.stopWatching = this.configClient.watchConfig<any>(
      this.configId,
      this.handleConfigChange.bind(this)
    );
    
    logger.info('Started watching for configuration changes');
  }
  
  /**
   * Handle configuration change event
   * @param event Configuration change event
   */
  private async handleConfigChange(event: ConfigChangeEvent<any>): Promise<void> {
    try {
      logger.info('Configuration changed', {
        version: event.version,
        timestamp: event.timestamp,
      });
      
      // Extract storage config and unified storage manager options
      this.extractConfig(event.data);
      
      // Cache the updated configuration locally
      await this.cacheConfigLocally(event.data);
      
      // Apply changes
      await this.applyChanges();
    } catch (error) {
      logger.error('Failed to handle configuration change', { error: (error as Error).message });
    }
  }
  
  /**
   * Set the event bus
   * @param eventBus Event bus
   */
  setEventBus(eventBus: EventBus): void {
    this.eventBus = eventBus;
    
    // Subscribe to configuration change events
    this.eventBus.subscribe('system.config_changed.v1', async (event: any) => {
      if (event.data && event.data.service === this.serviceName) {
        await this.reload();
      }
    });
  }
  
  /**
   * Reload configuration
   */
  async reload(): Promise<void> {
    try {
      // Try to load configuration from service first
      try {
        const config = await this.configClient.getConfig<any>(this.configId);
        
        // Extract storage config and unified storage manager options
        this.extractConfig(config);
        
        logger.info('Configuration reloaded from service');
        
        // Cache the configuration locally
        await this.cacheConfigLocally(config);
      } catch (serviceError) {
        logger.warn('Failed to reload configuration from service, falling back to local file', { 
          error: (serviceError as Error).message 
        });
        
        // Fall back to local file
        try {
          const config = await loadConfig<any>(
            `${this.serviceName}/${this.configId}.json`,
            this.createFullConfig(),
            { baseDir: this.localDir }
          );
          
          // Extract storage config and unified storage manager options
          this.extractConfig(config);
          
          logger.info('Configuration reloaded from local file');
        } catch (localError) {
          logger.error('Failed to reload configuration from local file', { 
            error: (localError as Error).message 
          });
          throw localError;
        }
      }
      
      logger.info('Configuration reloaded');
    } catch (error) {
      logger.error('Failed to reload configuration', { error: (error as Error).message });
      throw error;
    }
  }
  
  /**
   * Reset configuration to default
   */
  async resetToDefault(): Promise<void> {
    try {
      const defaultConfig = this.createFullConfig();
      
      // Try to save default configuration to service
      try {
        await this.configClient.saveConfig(
          this.configId,
          defaultConfig,
          'Reset to default configuration'
        );
        logger.info('Default configuration saved to service');
      } catch (serviceError) {
        logger.warn('Failed to save default configuration to service, falling back to local file', { 
          error: (serviceError as Error).message 
        });
      }
      
      // Always save to local file as well
      await saveConfig(
        `${this.serviceName}/${this.configId}.json`,
        defaultConfig,
        'Reset to default configuration',
        undefined,
        { baseDir: this.localDir }
      );
      
      // Reset in-memory configuration
      this.storageConfig = DEFAULT_STORAGE_CONFIG;
      this.unifiedStorageManagerOptions = DEFAULT_UNIFIED_STORAGE_MANAGER_OPTIONS;
      
      // Publish configuration change event
      await this.publishConfigChanged();
      
      logger.info('Configuration reset to default');
    } catch (error) {
      logger.error('Failed to reset configuration to default', { error: (error as Error).message });
      throw error;
    }
  }
  
  /**
   * Apply configuration changes
   */
  async applyChanges(): Promise<void> {
    // Publish configuration change event
    await this.publishConfigChanged();
    
    logger.info('Configuration changes applied');
  }
  
  /**
   * Publish configuration change event
   */
  async publishConfigChanged(): Promise<void> {
    if (!this.eventBus) {
      logger.warn('Event bus not available, cannot publish configuration change event');
      return;
    }
    
    try {
      // Create configuration change event
      const configChangedEvent = {
        service: this.serviceName,
        timestamp: new Date(),
        config: this.createFullConfig()
      };
      
      // Publish event
      await this.eventBus.publish(
        'system.config_changed.v1',
        configChangedEvent,
        {
          category: EventCategory.CONFIGURATION,
          priority: 'high'
        }
      );
      
      logger.info('Published configuration change event');
    } catch (error) {
      logger.error('Failed to publish configuration change event', { 
        error: (error as Error).message
      });
      // Continue operation even if publishing the event fails
    }
  }
  
  /**
   * Get the storage configuration
   */
  getStorageConfig(): StorageConfig {
    return this.storageConfig;
  }
  
  /**
   * Get the unified storage manager options
   */
  getUnifiedStorageManagerOptions(): UnifiedStorageManagerOptions {
    return this.unifiedStorageManagerOptions;
  }
  
  /**
   * Save configuration
   * @param config Configuration to save
   * @param description Optional description
   */
  async saveConfig(config: StorageConfig, description?: string): Promise<void> {
    try {
      // Update storage config
      this.storageConfig = config;
      
      // Create full configuration
      const fullConfig = this.createFullConfig();
      
      // Try to save configuration to service
      try {
        await this.configClient.saveConfig(
          this.configId,
          fullConfig,
          description || 'Updated configuration'
        );
        logger.info('Configuration saved to service');
      } catch (serviceError) {
        logger.warn('Failed to save configuration to service, falling back to local file', { 
          error: (serviceError as Error).message 
        });
      }
      
      // Always save to local file as well
      await saveConfig(
        `${this.serviceName}/${this.configId}.json`,
        fullConfig,
        description || 'Updated configuration',
        undefined,
        { baseDir: this.localDir }
      );
      
      // Publish configuration change event
      await this.publishConfigChanged();
      
      logger.info('Configuration saved');
    } catch (error) {
      logger.error('Failed to save configuration', { error: (error as Error).message });
      throw error;
    }
  }
  
  /**
   * Shutdown the configuration manager
   */
  shutdown(): void {
    // Stop watching for configuration changes
    if (this.stopWatching) {
      this.stopWatching();
      this.stopWatching = undefined;
    }
    
    logger.info('Configuration manager shut down');
  }
}

/**
 * Create a new storage configuration manager
 * @param options Configuration options
 * @returns Storage configuration manager
 */
export function createStorageConfigManager(options: StorageConfigManagerOptions = {}): StorageConfigManager {
  return new StorageConfigManager(options);
}