@sailboat-computer/data-storage/tests/integration/lifecycle-operations.test.ts

Shared data storage library for sailboat computer v3

/**
 * Integration tests for lifecycle operations
 */

import { createStorageManager } from '../../src/storage/manager';
import { createLifecycleManager } from '../../src/lifecycle/manager';
import { createDownsamplingEngine } from '../../src/downsampling/engine';
import { createTimeBasedDownsampler } from '../../src/downsampling/strategies/time-based';
import { createGeospatialDownsampler } from '../../src/downsampling/strategies/geospatial';
import { createCycleBasedDownsampler } from '../../src/downsampling/strategies/cycle-based';
import { createMaintenanceBasedDownsampler } from '../../src/downsampling/strategies/maintenance-based';
import { createGeotemporalGridDownsampler } from '../../src/downsampling/strategies/geotemporal-grid';
import {
  createWeatherDataGrid,
  createWeatherTimeSeries,
  createStormWeatherDataPoint
} from '../fixtures/weather-data';
import {
  createSeaStateDataGrid,
  createSeaStateTimeSeries,
  createRoughSeaStateDataPoint
} from '../fixtures/sea-state-data';
import { StorageTier, BatchPriority } from '../../src/types';

// Mock storage providers
jest.mock('../../src/storage/providers/redis');
jest.mock('../../src/storage/providers/influxdb');
jest.mock('../../src/storage/providers/postgresql');

describe('Lifecycle Operations Integration', () => {
  // Create components
  const storageManager = createStorageManager({
    providers: {
      hot: {
        type: 'redis',
        config: {
          host: 'localhost',
          port: 6379
        }
      },
      warm: {
        type: 'influxdb',
        config: {
          url: 'http://localhost:8086',
          token: 'test-token',
          org: 'test-org',
          bucket: 'test-bucket'
        }
      }
    },
    batching: {
      enabled: true,
      maxBatchSize: 10,
      timeInterval: 1000,
      priorityThresholds: {
        [BatchPriority.HIGH]: 0,
        [BatchPriority.NORMAL]: 100,
        [BatchPriority.LOW]: 1000
      }
    }
  } as any); // Type assertion to bypass type checking for test

  // Create downsampling engine
  const downsamplingEngine = createDownsamplingEngine();

  // Register all downsamplers
  beforeAll(() => {
    downsamplingEngine.registerDownsampler(createTimeBasedDownsampler());
    downsamplingEngine.registerDownsampler(createGeospatialDownsampler());
    downsamplingEngine.registerDownsampler(createCycleBasedDownsampler());
    downsamplingEngine.registerDownsampler(createMaintenanceBasedDownsampler());
    downsamplingEngine.registerDownsampler(createGeotemporalGridDownsampler());
  });

  // Initialize storage manager
  beforeEach(async () => {
    await storageManager.initialize();
  });

  // Shutdown storage manager after tests
  afterEach(async () => {
    await storageManager.shutdown();
  });

  describe('Lifecycle Manager with GeotemporalGridDownsampler', () => {
    it('should initialize with weather and sea state policies', () => {
      // Create lifecycle manager
      const lifecycleManager = createLifecycleManager(
        storageManager,
        {
          policies: [
            {
              category: 'weather.data',
              hotRetentionDays: 7,
              warmRetentionDays: 90,
              coldRetentionDays: 730,
              migrationThresholdDays: 3,
              downsample: true,
              downsamplingRuleId: 'weather-rule'
            },
            {
              category: 'sea.state',
              hotRetentionDays: 5,
              warmRetentionDays: 60,
              coldRetentionDays: 365,
              migrationThresholdDays: 2,
              downsample: true,
              downsamplingRuleId: 'sea-state-rule'
            }
          ],
          downsamplingRules: [
            {
              id: 'weather-rule',
              name: 'Weather Data Downsampling',
              dataType: 'weather.data',
              strategy: {
                type: 'geotemporal-grid',
                baseGridSize: 1.0,
                adaptiveGridLevels: 3,
                temporalHierarchy: {
                  recent: { maxAge: 7, resolution: 1 },
                  mediumTerm: { maxAge: 30, resolution: 6 },
                  longTerm: { maxAge: 90, resolution: 24 },
                  historical: { maxAge: 365, resolution: 168 },
                  seasonal: { resolution: 720 }
                }
              }
            },
            {
              id: 'sea-state-rule',
              name: 'Sea State Data Downsampling',
              dataType: 'sea.state',
              strategy: {
                type: 'geotemporal-grid',
                baseGridSize: 0.5,
                adaptiveGridLevels: 3,
                temporalHierarchy: {
                  recent: { maxAge: 3, resolution: 1 },
                  mediumTerm: { maxAge: 14, resolution: 3 },
                  longTerm: { maxAge: 60, resolution: 12 },
                  historical: { maxAge: 180, resolution: 72 },
                  seasonal: { resolution: 720 }
                }
              }
            }
          ],
          downsamplingSchedules: [
            { ruleId: 'weather-rule', cronExpression: '0 3 * * *', enabled: true },
            { ruleId: 'sea-state-rule', cronExpression: '0 9 * * *', enabled: true }
          ],
          cleanupInterval: 24 * 60 * 60 * 1000,
          migrationInterval: 12 * 60 * 60 * 1000
        },
        downsamplingEngine
      );

      // Verify initialization
      const policies = lifecycleManager.getPolicies();
      expect(policies.length).toBe(2);
      expect(policies[0].category).toBe('weather.data');
      expect(policies[1].category).toBe('sea.state');
    });

    it('should run downsampling for weather data', async () => {
      // Create lifecycle manager
      const lifecycleManager = createLifecycleManager(
        storageManager,
        {
          policies: [
            {
              category: 'weather.data',
              hotRetentionDays: 7,
              warmRetentionDays: 90,
              coldRetentionDays: 730,
              migrationThresholdDays: 3,
              downsample: true,
              downsamplingRuleId: 'weather-rule'
            }
          ],
          downsamplingRules: [
            {
              id: 'weather-rule',
              name: 'Weather Data Downsampling',
              dataType: 'weather.data',
              strategy: {
                type: 'geotemporal-grid',
                baseGridSize: 1.0,
                adaptiveGridLevels: 3,
                temporalHierarchy: {
                  recent: { maxAge: 7, resolution: 1 },
                  mediumTerm: { maxAge: 30, resolution: 6 },
                  longTerm: { maxAge: 90, resolution: 24 },
                  historical: { maxAge: 365, resolution: 168 },
                  seasonal: { resolution: 720 }
                }
              }
            }
          ],
          downsamplingSchedules: [
            { ruleId: 'weather-rule', cronExpression: '0 3 * * *', enabled: true }
          ],
          cleanupInterval: 24 * 60 * 60 * 1000,
          migrationInterval: 12 * 60 * 60 * 1000
        },
        downsamplingEngine
      );

      // Mock the storage manager's retrieve and storeBatch methods
      const weatherData = createWeatherDataGrid(37.7749, -122.4194, 1.0, 10, new Date());
      
      // @ts-ignore - Mock implementation
      storageManager.retrieve = jest.fn().mockResolvedValue(weatherData);
      
      // @ts-ignore - Mock implementation
      storageManager.storeBatch = jest.fn().mockResolvedValue(
        Array(50).fill(0).map((_, i) => `mock-id-${i}`)
      );

      // Run downsampling
      await lifecycleManager.runDownsampling('weather-rule');

      // Verify retrieve was called with correct parameters
      expect(storageManager.retrieve).toHaveBeenCalledWith(
        expect.objectContaining({
          category: 'weather.data'
        }),
        StorageTier.WARM
      );

      // Verify storeBatch was called
      expect(storageManager.storeBatch).toHaveBeenCalled();
      
      // Get the first argument of the first call
      const storeBatchArgs = (storageManager.storeBatch as jest.Mock).mock.calls[0][0];
      
      // Verify downsampled data was stored
      expect(storeBatchArgs.length).toBeLessThan(weatherData.length);
      
      // Verify downsampling metadata
      const firstItem = storeBatchArgs[0];
      expect(firstItem.data).toBeDefined();
      expect(firstItem.options.tags.downsampled).toBe('true');
      expect(firstItem.options.tags.downsamplingStrategy).toBe('geotemporal-grid');
    });

    it('should run downsampling for sea state data', async () => {
      // Create lifecycle manager
      const lifecycleManager = createLifecycleManager(
        storageManager,
        {
          policies: [
            {
              category: 'sea.state',
              hotRetentionDays: 5,
              warmRetentionDays: 60,
              coldRetentionDays: 365,
              migrationThresholdDays: 2,
              downsample: true,
              downsamplingRuleId: 'sea-state-rule'
            }
          ],
          downsamplingRules: [
            {
              id: 'sea-state-rule',
              name: 'Sea State Data Downsampling',
              dataType: 'sea.state',
              strategy: {
                type: 'geotemporal-grid',
                baseGridSize: 0.5,
                adaptiveGridLevels: 3,
                temporalHierarchy: {
                  recent: { maxAge: 3, resolution: 1 },
                  mediumTerm: { maxAge: 14, resolution: 3 },
                  longTerm: { maxAge: 60, resolution: 12 },
                  historical: { maxAge: 180, resolution: 72 },
                  seasonal: { resolution: 720 }
                }
              }
            }
          ],
          downsamplingSchedules: [
            { ruleId: 'sea-state-rule', cronExpression: '0 9 * * *', enabled: true }
          ],
          cleanupInterval: 24 * 60 * 60 * 1000,
          migrationInterval: 12 * 60 * 60 * 1000
        },
        downsamplingEngine
      );

      // Mock the storage manager's retrieve and storeBatch methods
      const seaStateData = createSeaStateDataGrid(37.8, -122.5, 0.5, 10, new Date());
      
      // @ts-ignore - Mock implementation
      storageManager.retrieve = jest.fn().mockResolvedValue(seaStateData);
      
      // @ts-ignore - Mock implementation
      storageManager.storeBatch = jest.fn().mockResolvedValue(
        Array(50).fill(0).map((_, i) => `mock-id-${i}`)
      );

      // Run downsampling
      await lifecycleManager.runDownsampling('sea-state-rule');

      // Verify retrieve was called with correct parameters
      expect(storageManager.retrieve).toHaveBeenCalledWith(
        expect.objectContaining({
          category: 'sea.state'
        }),
        StorageTier.WARM
      );

      // Verify storeBatch was called
      expect(storageManager.storeBatch).toHaveBeenCalled();
      
      // Get the first argument of the first call
      const storeBatchArgs = (storageManager.storeBatch as jest.Mock).mock.calls[0][0];
      
      // Verify downsampled data was stored
      expect(storeBatchArgs.length).toBeLessThan(seaStateData.length);
      
      // Verify downsampling metadata
      const firstItem = storeBatchArgs[0];
      expect(firstItem.data).toBeDefined();
      expect(firstItem.options.tags.downsampled).toBe('true');
      expect(firstItem.options.tags.downsamplingStrategy).toBe('geotemporal-grid');
    });
  });
});