@sailboat-computer/data-storage/tests/resilience/retry-behavior.test.ts

Shared data storage library for sailboat computer v3

/**
 * Retry Behavior Tests
 * 
 * Tests the behavior of the retry mechanism in the UnifiedStorageManager.
 */

import * as path from 'path';
import * as os from 'os';
import * as fs from 'fs';
import { StorageTier, StorageConfig } from '../../src/types';
import { createUnifiedStorageManager } from '../../src/storage';
import { withRetry } from '../../src/utils/errors';

describe('Retry Behavior Tests', () => {
  // Create a temporary directory for local storage
  const tempDir = path.join(os.tmpdir(), 'data-storage-tests', `test-${Date.now()}`);
  
  // Storage configuration
  let config: StorageConfig;
  
  beforeAll(() => {
    // Create temp directory
    fs.mkdirSync(tempDir, { recursive: true });
    
    // Create storage configuration
    config = {
      providers: {
        hot: {
          type: 'redis' as any,
          config: {
            host: 'localhost',
            port: 6379
          }
        },
        batching: {
          timeBased: {
            enabled: true,
            intervalMs: 1000
          },
          sizeBased: {
            enabled: true,
            maxBatchSize: 10
          },
          priorityOverride: true
        }
      }
    };
  });
  
  afterAll(() => {
    // Clean up temp directory
    fs.rmSync(tempDir, { recursive: true, force: true });
  });
  
  it('should retry failed operations', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      },
      resilience: {
        retry: {
          maxRetries: 3,
          baseDelayMs: 10 // Short delay for testing
        }
      }
    });
    
    // Mock the provider factory with failures then success
    const mockRemoteStore = jest.fn()
      .mockRejectedValueOnce(new Error('Remote storage error 1'))
      .mockRejectedValueOnce(new Error('Remote storage error 2'))
      .mockResolvedValueOnce('remote-id-3');
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: mockRemoteStore,
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 50, queriesPerSecond: 10 },
          warm: { averageQueryTime: 100, queriesPerSecond: 5 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Create a spy on the setTimeout function
    const setTimeoutSpy = jest.spyOn(global, 'setTimeout');
    
    // Act
    await storageManager.initialize();
    
    // Store attempt (should fail twice, then succeed on third try)
    const id = await storageManager.store({ value: 1 }, 'test');
    
    // Assert
    expect(mockRemoteStore).toHaveBeenCalledTimes(3);
    expect(id).toBe('remote-id-3');
    
    // Get metrics to verify retry count
    const metrics = storageManager.getResilienceMetrics();
    expect(Object.values(metrics.retry.retryCount).reduce((sum, count) => sum + count, 0)).toBe(2);
    
    // Clean up
    setTimeoutSpy.mockRestore();
    await storageManager.close();
  });
  
  it('should fall back to local storage after max retries', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      },
      resilience: {
        retry: {
          maxRetries: 2,
          baseDelayMs: 10 // Short delay for testing
        }
      }
    });
    
    // Mock the provider factory with persistent failures
    const mockRemoteStore = jest.fn()
      .mockRejectedValueOnce(new Error('Remote storage error 1'))
      .mockRejectedValueOnce(new Error('Remote storage error 2'))
      .mockRejectedValueOnce(new Error('Remote storage error 3'));
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: mockRemoteStore,
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 50, queriesPerSecond: 10 },
          warm: { averageQueryTime: 100, queriesPerSecond: 5 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Create a spy on the local storage provider
    const localStoreSpy = jest.fn().mockResolvedValue('local-id');
    (storageManager as any).localStorageProvider = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: localStoreSpy,
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 10, queriesPerSecond: 100 },
          warm: { averageQueryTime: 10, queriesPerSecond: 100 },
          cold: { averageQueryTime: 10, queriesPerSecond: 100 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined),
      getLocalStorageStatus: jest.fn().mockResolvedValue({
        available: true,
        healthy: true,
        totalSizeBytes: 1024 * 1024 * 1024,
        usedSizeBytes: 1024 * 1024,
        freeSizeBytes: 1023 * 1024 * 1024,
        itemCount: 10
      })
    };
    
    // Act
    await storageManager.initialize();
    
    // Store attempt (should fail after max retries and fall back to local storage)
    const id = await storageManager.store({ value: 1 }, 'test');
    
    // Assert
    expect(mockRemoteStore).toHaveBeenCalledTimes(3); // Initial attempt + 2 retries
    expect(localStoreSpy).toHaveBeenCalledTimes(1);
    expect(id).toBe('local-id');
    
    // Get metrics to verify retry count
    const metrics = storageManager.getResilienceMetrics();
    expect(Object.values(metrics.retry.retryCount).reduce((sum, count) => sum + count, 0)).toBe(2);
    
    // Verify pending sync contains the stored item
    expect((storageManager as any).pendingSync.size).toBe(1);
    expect((storageManager as any).pendingSync.has(id)).toBe(true);
    
    // Clean up
    await storageManager.close();
  });
  
  it('should use exponential backoff with jitter', async () => {
    // Arrange
    const storageManager = createUnifiedStorageManager(config, {
      localStorage: {
        directory: tempDir,
        maxSizeBytes: 10 * 1024 * 1024, // 10MB
        encrypt: false,
        compressionLevel: 0
      },
      resilience: {
        retry: {
          maxRetries: 3,
          baseDelayMs: 10 // Short delay for testing
        }
      }
    });
    
    // Mock the provider factory with failures
    const mockRemoteStore = jest.fn()
      .mockRejectedValueOnce(new Error('Remote storage error 1'))
      .mockRejectedValueOnce(new Error('Remote storage error 2'))
      .mockRejectedValueOnce(new Error('Remote storage error 3'))
      .mockRejectedValueOnce(new Error('Remote storage error 4'));
    
    (storageManager as any).remoteStorageManager = {
      initialize: jest.fn().mockResolvedValue(undefined),
      store: mockRemoteStore,
      retrieve: jest.fn().mockResolvedValue([]),
      update: jest.fn().mockResolvedValue(undefined),
      delete: jest.fn().mockResolvedValue(undefined),
      storeBatch: jest.fn().mockResolvedValue([]),
      retrieveBatch: jest.fn().mockResolvedValue([]),
      cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }),
      migrate: jest.fn().mockResolvedValue(undefined),
      getStatus: jest.fn().mockResolvedValue({
        connected: true,
        healthy: true,
        queryPerformance: {
          hot: { averageQueryTime: 50, queriesPerSecond: 10 },
          warm: { averageQueryTime: 100, queriesPerSecond: 5 },
          cold: { averageQueryTime: 200, queriesPerSecond: 2 },
          overall: 'healthy',
          providers: {
            hot: 'healthy',
            warm: 'healthy',
            cold: 'healthy'
          }
        }
      }),
      close: jest.fn().mockResolvedValue(undefined)
    };
    
    // Create a spy on the setTimeout function
    const setTimeoutSpy = jest.spyOn(global, 'setTimeout');
    
    // Act
    await storageManager.initialize();
    
    try {
      // Store attempt (should fail after max retries)
      await storageManager.store({ value: 1 }, 'test');
    } catch (error) {
      // Expected to fail
    }
    
    // Assert
    expect(mockRemoteStore).toHaveBeenCalledTimes(4); // Initial attempt + 3 retries
    
    // Verify setTimeout was called with increasing delays
    const calls = setTimeoutSpy.mock.calls;
    const retryDelays = calls
      .map(call => call[1]) // Extract delay argument
      .filter((delay): delay is number => typeof delay === 'number' && delay >= 10 && delay <= 100); // Filter retry delays with type guard
    
    expect(retryDelays.length).toBeGreaterThanOrEqual(3);
    
    // Verify delays are increasing (exponential backoff)
    for (let i = 1; i < retryDelays.length; i++) {
      expect(retryDelays[i]).toBeGreaterThan(retryDelays[i - 1]);
    }
    
    // Clean up
    setTimeoutSpy.mockRestore();
    await storageManager.close();
  });
  
  it('should directly use withRetry utility function', async () => {
    // Test the withRetry utility function directly
    const successAfterFailures = jest.fn()
      .mockRejectedValueOnce(new Error('Failure 1'))
      .mockRejectedValueOnce(new Error('Failure 2'))
      .mockResolvedValueOnce('success');
    
    const alwaysFails = jest.fn()
      .mockRejectedValue(new Error('Always fails'));
    
    // Function that succeeds after failures should eventually return
    const result = await withRetry(successAfterFailures, 3, 10);
    expect(result).toBe('success');
    expect(successAfterFailures).toHaveBeenCalledTimes(3);
    
    // Function that always fails should throw after max retries
    await expect(withRetry(alwaysFails, 2, 10)).rejects.toThrow('Always fails');
    expect(alwaysFails).toHaveBeenCalledTimes(3); // Initial attempt + 2 retries
  });
});