@sailboat-computer/data-storage/tests/resilience/circuit-breaker.test.ts

Shared data storage library for sailboat computer v3

/**
 * Circuit Breaker Tests
 * 
 * Tests the behavior of the circuit breaker pattern in the UnifiedStorageManager.
 */

import * as path from 'path';
import * as os from 'os';
import * as fs from 'fs';
import { StorageTier, StorageConfig } from '../../src/types';
import { createUnifiedStorageManager } from '../../src/storage';
import { CircuitBreaker } from '../../src/utils/errors';

describe('Circuit Breaker Tests', () => {
  // Create a temporary directory for local storage
  const tempDir = path.join(os.tmpdir(), 'data-storage-tests', `test-${Date.now()}`);
  
  // Storage configuration
  let config: StorageConfig;
  
  beforeAll(() => {
    // Create temp directory
    fs.mkdirSync(tempDir, { recursive: true });
    
    // Create storage configuration
    config = {
      providers: {
        hot: {
          type: 'redis' as any,
          config: {
            host: 'localhost',
            port: 6379
          }
        },
        batching: {
          timeBased: {
            enabled: true,
            intervalMs: 1000
          },
          sizeBased: {
            enabled: true,
            maxBatchSize: 10
          },
          priorityOverride: true
        }
      }
    };
  });
  
  afterAll(() => {
    // Clean up temp directory
    fs.rmSync(tempDir, { recursive: true, force: true });
  });
  
  it('should open circuit after multiple failures', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      },
      resilience: {
        circuitBreaker: {
          failureThreshold: 3,
          resetTimeoutMs: 30000 // 30 seconds
        }
      }
    });
    
    // Mock the provider factory with failures
    const mockRemoteStore = jest.fn()
      .mockRejectedValueOnce(new Error('Remote storage error 1'))
      .mockRejectedValueOnce(new Error('Remote storage error 2'))
      .mockRejectedValueOnce(new Error('Remote storage error 3'))
      .mockRejectedValueOnce(new Error('Remote storage error 4'));
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: mockRemoteStore,
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 50, queriesPerSecond: 10 },
          warm: { averageQueryTime: 100, queriesPerSecond: 5 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Create a spy on the circuit breaker
    const circuitBreakerSpy = jest.spyOn(CircuitBreaker.prototype, 'execute');
    
    // Act
    await storageManager.initialize();
    
    // First store attempt (failure 1)
    await storageManager.store({ value: 1 }, 'test');
    
    // Second store attempt (failure 2)
    await storageManager.store({ value: 2 }, 'test');
    
    // Third store attempt (failure 3 - should open circuit)
    await storageManager.store({ value: 3 }, 'test');
    
    // Fourth store attempt (circuit should be open, should not call remote store)
    await storageManager.store({ value: 4 }, 'test');
    
    // Assert
    expect(mockRemoteStore).toHaveBeenCalledTimes(3); // Only 3 calls, not 4
    expect(circuitBreakerSpy).toHaveBeenCalledTimes(4);
    
    // Get metrics to verify circuit breaker state
    const metrics = storageManager.getResilienceMetrics();
    expect(metrics.circuitBreaker.state['remoteStorage']).toBe('open');
    expect(metrics.circuitBreaker.failures['remoteStorage']).toBe(3);
    
    // Clean up
    circuitBreakerSpy.mockRestore();
    await storageManager.close();
  });
  
  it('should transition to half-open state after reset timeout', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      },
      resilience: {
        circuitBreaker: {
          failureThreshold: 3,
          resetTimeoutMs: 100 // Short timeout for testing
        }
      }
    });
    
    // Mock the provider factory with failures then success
    const mockRemoteStore = jest.fn()
      .mockRejectedValueOnce(new Error('Remote storage error 1'))
      .mockRejectedValueOnce(new Error('Remote storage error 2'))
      .mockRejectedValueOnce(new Error('Remote storage error 3'))
      .mockResolvedValueOnce('remote-id-4');
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: mockRemoteStore,
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 50, queriesPerSecond: 10 },
          warm: { averageQueryTime: 100, queriesPerSecond: 5 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Act
    await storageManager.initialize();
    
    // First store attempt (failure 1)
    await storageManager.store({ value: 1 }, 'test');
    
    // Second store attempt (failure 2)
    await storageManager.store({ value: 2 }, 'test');
    
    // Third store attempt (failure 3 - should open circuit)
    await storageManager.store({ value: 3 }, 'test');
    
    // Get metrics to verify circuit breaker is open
    let metrics = storageManager.getResilienceMetrics();
    expect(metrics.circuitBreaker.state['remoteStorage']).toBe('open');
    
    // Wait for reset timeout
    await new Promise(resolve => setTimeout(resolve, 150));
    
    // Fourth store attempt (circuit should be half-open, should try remote store)
    const id = await storageManager.store({ value: 4 }, 'test');
    
    // Assert
    expect(mockRemoteStore).toHaveBeenCalledTimes(4);
    expect(id).toBe('remote-id-4');
    
    // Get metrics to verify circuit breaker state is now closed
    metrics = storageManager.getResilienceMetrics();
    expect(metrics.circuitBreaker.state['remoteStorage']).toBe('closed');
    expect(metrics.circuitBreaker.failures['remoteStorage']).toBe(0);
    
    // Clean up
    await storageManager.close();
  });
  
  it('should keep circuit open if test request fails in half-open state', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      },
      resilience: {
        circuitBreaker: {
          failureThreshold: 3,
          resetTimeoutMs: 100 // Short timeout for testing
        }
      }
    });
    
    // Mock the provider factory with failures
    const mockRemoteStore = jest.fn()
      .mockRejectedValueOnce(new Error('Remote storage error 1'))
      .mockRejectedValueOnce(new Error('Remote storage error 2'))
      .mockRejectedValueOnce(new Error('Remote storage error 3'))
      .mockRejectedValueOnce(new Error('Remote storage error 4'))
      .mockRejectedValueOnce(new Error('Remote storage error 5'));
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: mockRemoteStore,
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 50, queriesPerSecond: 10 },
          warm: { averageQueryTime: 100, queriesPerSecond: 5 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Act
    await storageManager.initialize();
    
    // First store attempt (failure 1)
    await storageManager.store({ value: 1 }, 'test');
    
    // Second store attempt (failure 2)
    await storageManager.store({ value: 2 }, 'test');
    
    // Third store attempt (failure 3 - should open circuit)
    await storageManager.store({ value: 3 }, 'test');
    
    // Get metrics to verify circuit breaker is open
    let metrics = storageManager.getResilienceMetrics();
    expect(metrics.circuitBreaker.state['remoteStorage']).toBe('open');
    
    // Wait for reset timeout
    await new Promise(resolve => setTimeout(resolve, 150));
    
    // Fourth store attempt (circuit should be half-open, should try remote store)
    await storageManager.store({ value: 4 }, 'test');
    
    // Get metrics to verify circuit breaker is still open
    metrics = storageManager.getResilienceMetrics();
    expect(metrics.circuitBreaker.state['remoteStorage']).toBe('open');
    
    // Fifth store attempt (circuit should be open, should not call remote store)
    await storageManager.store({ value: 5 }, 'test');
    
    // Assert
    expect(mockRemoteStore).toHaveBeenCalledTimes(4); // Only 4 calls, not 5
    
    // Clean up
    await storageManager.close();
  });
  
  it('should use separate circuit breakers for different operations', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      },
      resilience: {
        circuitBreaker: {
          failureThreshold: 3,
          resetTimeoutMs: 30000 // 30 seconds
        }
      }
    });
    
    // Mock the provider factory with failures for store but success for retrieve
    const mockRemoteStore = jest.fn()
      .mockRejectedValueOnce(new Error('Remote storage error 1'))
      .mockRejectedValueOnce(new Error('Remote storage error 2'))
      .mockRejectedValueOnce(new Error('Remote storage error 3'))
      .mockRejectedValueOnce(new Error('Remote storage error 4'));
    
    const mockRemoteRetrieve = jest.fn()
      .mockResolvedValue([{ data: { value: 'test' }, metadata: { id: 'remote-id', category: 'test' } }]);
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: mockRemoteStore,
      retrieve: mockRemoteRetrieve,
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 50, queriesPerSecond: 10 },
          warm: { averageQueryTime: 100, queriesPerSecond: 5 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Act
    await storageManager.initialize();
    
    // First store attempt (failure 1)
    await storageManager.store({ value: 1 }, 'test');
    
    // Second store attempt (failure 2)
    await storageManager.store({ value: 2 }, 'test');
    
    // Third store attempt (failure 3 - should open circuit for store)
    await storageManager.store({ value: 3 }, 'test');
    
    // Fourth store attempt (circuit should be open for store)
    await storageManager.store({ value: 4 }, 'test');
    
    // Retrieve attempt (circuit should be closed for retrieve)
    const results = await storageManager.retrieve({ category: 'test' });
    
    // Assert
    expect(mockRemoteStore).toHaveBeenCalledTimes(3); // Only 3 calls, not 4
    expect(mockRemoteRetrieve).toHaveBeenCalledTimes(1); // Retrieve should still work
    expect(results.length).toBe(1);
    expect(results[0].data.value).toBe('test');
    
    // Get metrics to verify circuit breaker states
    const metrics = storageManager.getResilienceMetrics();
    expect(metrics.circuitBreaker.state['remoteStorage']).toBe('open');
    expect(metrics.circuitBreaker.failures['remoteStorage']).toBe(3);
    
    // Clean up
    await storageManager.close();
  });
});