@sailboat-computer/data-storage/tests/resilience/graceful-degradation.test.ts

Shared data storage library for sailboat computer v3

/**
 * Graceful Degradation Tests
 * 
 * Tests the behavior of the UnifiedStorageManager when some storage tiers are unavailable.
 */

import * as path from 'path';
import * as os from 'os';
import * as fs from 'fs';
import { StorageTier, StorageConfig } from '../../src/types';
import { createUnifiedStorageManager } from '../../src/storage';

describe('Graceful Degradation Tests', () => {
  // Create a temporary directory for local storage
  const tempDir = path.join(os.tmpdir(), 'data-storage-tests', `test-${Date.now()}`);
  
  // Storage configuration
  let config: StorageConfig;
  
  beforeAll(() => {
    // Create temp directory
    fs.mkdirSync(tempDir, { recursive: true });
    
    // Create storage configuration with multiple tiers
    config = {
      providers: {
        hot: {
          type: 'redis' as any,
          config: {
            host: 'localhost',
            port: 6379
          }
        },
        warm: {
          type: 'influxdb' as any,
          config: {
            url: 'http://localhost:8086',
            token: 'token',
            org: 'org',
            bucket: 'bucket'
          }
        },
        cold: {
          type: 'postgresql' as any,
          config: {
            host: 'localhost',
            port: 5432,
            database: 'db',
            user: 'user',
            password: 'password'
          }
        },
        batching: {
          timeBased: {
            enabled: true,
            intervalMs: 1000
          },
          sizeBased: {
            enabled: true,
            maxBatchSize: 10
          },
          priorityOverride: true
        }
      }
    };
  });
  
  afterAll(() => {
    // Clean up temp directory
    fs.rmSync(tempDir, { recursive: true, force: true });
  });
  
  it('should continue operation when hot tier is unavailable', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      }
    });
    
    // Mock the provider factory with hot tier unavailable
    const mockHotStore = jest.fn().mockRejectedValue(new Error('Hot tier unavailable'));
    const mockWarmStore = jest.fn().mockResolvedValue('warm-id');
    const mockColdStore = jest.fn().mockResolvedValue('cold-id');
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: jest.fn().mockImplementation((data, category, options) => {
        if (options?.tier === StorageTier.HOT) {
          return mockHotStore(data, category, options);
        } else if (options?.tier === StorageTier.WARM) {
          return mockWarmStore(data, category, options);
        } else if (options?.tier === StorageTier.COLD) {
          return mockColdStore(data, category, options);
        } else {
          return mockHotStore(data, category, options); // Default to hot tier
        }
      }),
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 0, queriesPerSecond: 0 },
          warm: { averageQueryTime: 100, queriesPerSecond: 5 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'degraded',
          providers: {
            hot: 'unhealthy',
            warm: 'healthy',
            cold: 'healthy'
          },
          issues: {
            severity: 'warning',
            message: 'Hot tier is unavailable',
            component: 'hot-tier',
            timestamp: new Date()
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Act
    await storageManager.initialize();
    
    // Store data in hot tier (should fail and fall back to warm tier)
    const hotId = await storageManager.store({ value: 1 }, 'test', { tier: StorageTier.HOT });
    
    // Store data in warm tier (should succeed)
    const warmId = await storageManager.store({ value: 2 }, 'test', { tier: StorageTier.WARM });
    
    // Store data in cold tier (should succeed)
    const coldId = await storageManager.store({ value: 3 }, 'test', { tier: StorageTier.COLD });
    
    // Get status
    const status = await storageManager.getStatus();
    
    // Assert
    expect(mockHotStore).toHaveBeenCalledTimes(1);
    expect(mockWarmStore).toHaveBeenCalledTimes(2); // Once for warm tier, once for fallback from hot
    expect(mockColdStore).toHaveBeenCalledTimes(1);
    
    expect(hotId).toBe('warm-id'); // Should fall back to warm tier
    expect(warmId).toBe('warm-id');
    expect(coldId).toBe('cold-id');
    
    expect(status.healthy).toBe(false); // System is degraded
    expect(status.queryPerformance.overall).toBe('degraded');
    
    // Clean up
    await storageManager.close();
  });
  
  it('should continue operation when warm tier is unavailable', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      }
    });
    
    // Mock the provider factory with warm tier unavailable
    const mockHotStore = jest.fn().mockResolvedValue('hot-id');
    const mockWarmStore = jest.fn().mockRejectedValue(new Error('Warm tier unavailable'));
    const mockColdStore = jest.fn().mockResolvedValue('cold-id');
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: jest.fn().mockImplementation((data, category, options) => {
        if (options?.tier === StorageTier.HOT) {
          return mockHotStore(data, category, options);
        } else if (options?.tier === StorageTier.WARM) {
          return mockWarmStore(data, category, options);
        } else if (options?.tier === StorageTier.COLD) {
          return mockColdStore(data, category, options);
        } else {
          return mockHotStore(data, category, options); // Default to hot tier
        }
      }),
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 50, queriesPerSecond: 10 },
          warm: { averageQueryTime: 0, queriesPerSecond: 0 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'degraded',
          providers: {
            hot: 'healthy',
            warm: 'unhealthy',
            cold: 'healthy'
          },
          issues: {
            severity: 'warning',
            message: 'Warm tier is unavailable',
            component: 'warm-tier',
            timestamp: new Date()
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Act
    await storageManager.initialize();
    
    // Store data in hot tier (should succeed)
    const hotId = await storageManager.store({ value: 1 }, 'test', { tier: StorageTier.HOT });
    
    // Store data in warm tier (should fail and fall back to hot tier)
    const warmId = await storageManager.store({ value: 2 }, 'test', { tier: StorageTier.WARM });
    
    // Store data in cold tier (should succeed)
    const coldId = await storageManager.store({ value: 3 }, 'test', { tier: StorageTier.COLD });
    
    // Get status
    const status = await storageManager.getStatus();
    
    // Assert
    expect(mockHotStore).toHaveBeenCalledTimes(2); // Once for hot tier, once for fallback from warm
    expect(mockWarmStore).toHaveBeenCalledTimes(1);
    expect(mockColdStore).toHaveBeenCalledTimes(1);
    
    expect(hotId).toBe('hot-id');
    expect(warmId).toBe('hot-id'); // Should fall back to hot tier
    expect(coldId).toBe('cold-id');
    
    expect(status.healthy).toBe(false); // System is degraded
    expect(status.queryPerformance.overall).toBe('degraded');
    
    // Clean up
    await storageManager.close();
  });
  
  it('should continue operation when all remote tiers are unavailable', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      }
    });
    
    // Mock the provider factory with all tiers unavailable
    const mockHotStore = jest.fn().mockRejectedValue(new Error('Hot tier unavailable'));
    const mockWarmStore = jest.fn().mockRejectedValue(new Error('Warm tier unavailable'));
    const mockColdStore = jest.fn().mockRejectedValue(new Error('Cold tier unavailable'));
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: jest.fn().mockImplementation((data, category, options) => {
        if (options?.tier === StorageTier.HOT) {
          return mockHotStore(data, category, options);
        } else if (options?.tier === StorageTier.WARM) {
          return mockWarmStore(data, category, options);
        } else if (options?.tier === StorageTier.COLD) {
          return mockColdStore(data, category, options);
        } else {
          return mockHotStore(data, category, options); // Default to hot tier
        }
      }),
      retrieve: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')),
      update: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')),
      delete: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')),
      storeBatch: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')),
      retrieveBatch: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')),
      cleanup: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')),
      migrate: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')),
      getStatus: jest.fn().mockResolvedValue({
        connected: false,
        healthy: false,
        error: 'All remote tiers are unavailable',
        queryPerformance: {
          hot: { averageQueryTime: 0, queriesPerSecond: 0 },
          warm: { averageQueryTime: 0, queriesPerSecond: 0 },
          cold: { averageQueryTime: 0, queriesPerSecond: 0 },
          overall: 'unhealthy',
          providers: {
            hot: 'unhealthy',
            warm: 'unhealthy',
            cold: 'unhealthy'
          },
          issues: {
            severity: 'error',
            message: 'All remote tiers are unavailable',
            component: 'remote-storage',
            timestamp: new Date()
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Create a spy on the local storage provider
    const localStoreSpy = jest.fn().mockResolvedValue('local-id');
    (storageManager as any).localStorageProvider = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: localStoreSpy,
      retrieve: jest.fn().mockResolvedValue([{ data: { value: 'test' }, metadata: { id: 'local-id', category: 'test' } }]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue(['local-id-1', 'local-id-2']),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 10, queriesPerSecond: 100 },
          warm: { averageQueryTime: 10, queriesPerSecond: 100 },
          cold: { averageQueryTime: 10, queriesPerSecond: 100 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined),
      getLocalStorageStatus: jest.fn().mockResolvedValue({
        available: true,
        healthy: true,
        totalSizeBytes: 1024 * 1024 * 1024,
        usedSizeBytes: 1024 * 1024,
        freeSizeBytes: 1023 * 1024 * 1024,
        itemCount: 10
      })
    };
    
    // Act
    await storageManager.initialize();
    
    // Store data in hot tier (should fail and fall back to local storage)
    const hotId = await storageManager.store({ value: 1 }, 'test', { tier: StorageTier.HOT });
    
    // Store data in warm tier (should fail and fall back to local storage)
    const warmId = await storageManager.store({ value: 2 }, 'test', { tier: StorageTier.WARM });
    
    // Store data in cold tier (should fail and fall back to local storage)
    const coldId = await storageManager.store({ value: 3 }, 'test', { tier: StorageTier.COLD });
    
    // Retrieve data (should use local storage)
    const results = await storageManager.retrieve({ category: 'test' });
    
    // Get status
    const status = await storageManager.getStatus();
    
    // Get local storage status
    const localStatus = await storageManager.getLocalStorageStatus();
    
    // Assert
    expect(mockHotStore).toHaveBeenCalledTimes(1);
    expect(mockWarmStore).toHaveBeenCalledTimes(1);
    expect(mockColdStore).toHaveBeenCalledTimes(1);
    expect(localStoreSpy).toHaveBeenCalledTimes(3);
    
    expect(hotId).toBe('local-id');
    expect(warmId).toBe('local-id');
    expect(coldId).toBe('local-id');
    
    expect(results.length).toBe(1);
    expect(results[0].data.value).toBe('test');
    
    expect(status.connected).toBe(true); // System is connected via local storage
    expect(status.healthy).toBe(false); // System is degraded
    expect(status.queryPerformance.overall).toBe('unhealthy');
    
    expect(localStatus.available).toBe(true);
    expect(localStatus.healthy).toBe(true);
    
    // Verify pending sync contains the stored items
    expect((storageManager as any).pendingSync.size).toBe(3);
    expect((storageManager as any).pendingSync.has(hotId)).toBe(true);
    expect((storageManager as any).pendingSync.has(warmId)).toBe(true);
    expect((storageManager as any).pendingSync.has(coldId)).toBe(true);
    
    // Clean up
    await storageManager.close();
  });
  
  it('should recover when tiers become available again', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      }
    });
    
    // Mock the provider factory with tiers initially unavailable, then available
    const mockHotStore = jest.fn()
      .mockRejectedValueOnce(new Error('Hot tier unavailable'))
      .mockResolvedValue('hot-id');
    
    const mockWarmStore = jest.fn()
      .mockRejectedValueOnce(new Error('Warm tier unavailable'))
      .mockResolvedValue('warm-id');
    
    const mockColdStore = jest.fn()
      .mockRejectedValueOnce(new Error('Cold tier unavailable'))
      .mockResolvedValue('cold-id');
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: jest.fn().mockImplementation((data, category, options) => {
        if (options?.tier === StorageTier.HOT) {
          return mockHotStore(data, category, options);
        } else if (options?.tier === StorageTier.WARM) {
          return mockWarmStore(data, category, options);
        } else if (options?.tier === StorageTier.COLD) {
          return mockColdStore(data, category, options);
        } else {
          return mockHotStore(data, category, options); // Default to hot tier
        }
      }),
      retrieve: jest.fn()
        .mockRejectedValueOnce(new Error('Remote storage unavailable'))
        .mockResolvedValue([{ data: { value: 'remote' }, metadata: { id: 'remote-id', category: 'test' } }]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue(['remote-id-1', 'remote-id-2']),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn()
        .mockResolvedValueOnce({
          connected: false,
          healthy: false,
          error: 'All remote tiers are unavailable',
          queryPerformance: {
            hot: { averageQueryTime: 0, queriesPerSecond: 0 },
            warm: { averageQueryTime: 0, queriesPerSecond: 0 },
            cold: { averageQueryTime: 0, queriesPerSecond: 0 },
            overall: 'unhealthy',
            providers: {
              hot: 'unhealthy',
              warm: 'unhealthy',
              cold: 'unhealthy'
            }
          }
        })
        .mockResolvedValue({
          connected: true,
          healthy: true,
          queryPerformance: {
            hot: { averageQueryTime: 50, queriesPerSecond: 10 },
            warm: { averageQueryTime: 100, queriesPerSecond: 5 },
            cold: { averageQueryTime: 200, queriesPerSecond: 2 },
            overall: 'healthy',
            providers: {
              hot: 'healthy',
              warm: 'healthy',
              cold: 'healthy'
            }
          }
        }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Create a spy on the local storage provider
    const localStoreSpy = jest.fn().mockResolvedValue('local-id');
    (storageManager as any).localStorageProvider = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: localStoreSpy,
      retrieve: jest.fn().mockResolvedValue([{ data: { value: 'local' }, metadata: { id: 'local-id', category: 'test' } }]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue(['local-id-1', 'local-id-2']),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 10, queriesPerSecond: 100 },
          warm: { averageQueryTime: 10, queriesPerSecond: 100 },
          cold: { averageQueryTime: 10, queriesPerSecond: 100 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined),
      getLocalStorageStatus: jest.fn().mockResolvedValue({
        available: true,
        healthy: true,
        totalSizeBytes: 1024 * 1024 * 1024,
        usedSizeBytes: 1024 * 1024,
        freeSizeBytes: 1023 * 1024 * 1024,
        itemCount: 10
      })
    };
    
    // Act
    await storageManager.initialize();
    
    // First phase: tiers are unavailable
    
    // Store data in hot tier (should fail and fall back to local storage)
    const hotId1 = await storageManager.store({ value: 1 }, 'test', { tier: StorageTier.HOT });
    
    // Retrieve data (should use local storage)
    const results1 = await storageManager.retrieve({ category: 'test' });
    
    // Get status (should be degraded)
    const status1 = await storageManager.getStatus();
    
    // Second phase: tiers are available again
    
    // Store data in hot tier (should succeed)
    const hotId2 = await storageManager.store({ value: 2 }, 'test', { tier: StorageTier.HOT });
    
    // Retrieve data (should use remote storage)
    const results2 = await storageManager.retrieve({ category: 'test' });
    
    // Get status (should be healthy)
    const status2 = await storageManager.getStatus();
    
    // Force synchronization (should sync local data to remote)
    const syncResult = await storageManager.forceSynchronization();
    
    // Assert
    expect(hotId1).toBe('local-id');
    expect(hotId2).toBe('hot-id');
    
    expect(results1[0].data.value).toBe('local');
    expect(results2[0].data.value).toBe('remote');
    
    expect(status1.healthy).toBe(false);
    expect(status1.queryPerformance.overall).toBe('unhealthy');
    
    expect(status2.healthy).toBe(true);
    expect(status2.queryPerformance.overall).toBe('healthy');
    
    expect(syncResult.success).toBe(true);
    expect(syncResult.itemCount).toBeGreaterThan(0);
    
    // Clean up
    await storageManager.close();
  });
});