@sailboat-computer/data-storage/src/batching/manager.ts

Shared data storage library for sailboat computer v3

/**
 * Batch manager implementation
 */

import { BatchManager, BatchConfig, BatchPriority, BatchPriorityType, BatchStatus, DataMetadata, StorageProvider } from '../types';
import { StorageError, StorageErrorCode } from '../utils/errors';

/**
 * Default batch configuration
 */
const DEFAULT_CONFIG: BatchConfig = {
  timeBased: {
    enabled: true,
    intervalMs: 30000 // 30 seconds
  },
  sizeBased: {
    enabled: true,
    maxBatchSize: 100
  },
  priorityOverride: true,
  maxRetries: 3,
  retryDelayMs: 1000
};

/**
 * Batch manager implementation
 */
export class BatchManagerImpl implements BatchManager {
  private config: BatchConfig;
  private batches: Map<string, Array<{data: any, metadata: DataMetadata, priority: BatchPriorityType}>> = new Map();
  private flushTimer?: NodeJS.Timeout;
  private initialized = false;
  private stopped = false;
  private stats = {
    pendingItems: 0,
    lastFlush: undefined as Date | undefined,
    averageBatchSize: 0,
    totalBatches: 0,
    failedBatches: 0
  };
  
  /**
   * Create a new batch manager
   * 
   * @param provider - Storage provider
   */
  constructor(private provider: StorageProvider) {
    this.config = DEFAULT_CONFIG;
  }
  
  /**
   * Initialize the batch manager
   * 
   * @param config - Batch configuration
   */
  initialize(config: BatchConfig): void {
    if (this.initialized) {
      return;
    }
    
    this.config = {
      ...DEFAULT_CONFIG,
      ...config
    };
    
    // Start time-based flushing if enabled
    if (this.config.timeBased.enabled) {
      this.startTimedFlush();
    }
    
    this.initialized = true;
    console.log('Batch manager initialized');
  }
  
  /**
   * Add data to batch
   * 
   * @param data - Data to add
   * @param metadata - Data metadata
   * @param priority - Batch priority
   */
  addToBatch(data: any, metadata: DataMetadata, priority: BatchPriorityType = BatchPriority.NORMAL): void {
    this.ensureInitialized();
    
    // Get category key
    const categoryKey = metadata.category;
    
    // Get or create batch for category
    if (!this.batches.has(categoryKey)) {
      this.batches.set(categoryKey, []);
    }
    
    const batch = this.batches.get(categoryKey)!;
    
    // Add item to batch
    batch.push({ data, metadata, priority });
    this.stats.pendingItems++;
    
    // Check if we should flush due to priority
    if (this.config.priorityOverride && priority === BatchPriority.CRITICAL) {
      this.flushCategory(categoryKey);
      return;
    }
    
    // Check if we should flush due to size
    if (this.config.sizeBased.enabled && batch.length >= this.config.sizeBased.maxBatchSize) {
      this.flushCategory(categoryKey);
    }
  }
  
  /**
   * Flush all batches
   */
  async flush(): Promise<void> {
    this.ensureInitialized();
    
    try {
      // Get all category keys
      const categories = Array.from(this.batches.keys());
      
      // Flush each category
      await Promise.all(categories.map(category => this.flushCategory(category)));
      
      // Update stats
      this.stats.lastFlush = new Date();
    } catch (error) {
      console.error('Failed to flush batches:', error);
      throw new StorageError(
        StorageErrorCode.STORE_FAILED,
        'Failed to flush batches',
        { error }
      );
    }
  }
  
  /**
   * Get batch status
   * 
   * @returns Batch status
   */
  getBatchStatus(): BatchStatus {
    return {
      pendingItems: this.stats.pendingItems,
      lastFlush: this.stats.lastFlush,
      averageBatchSize: this.stats.averageBatchSize,
      totalBatches: this.stats.totalBatches,
      failedBatches: this.stats.failedBatches
    };
  }
  
  /**
   * Stop the batch manager
   */
  stop(): void {
    if (this.stopped) {
      return;
    }
    
    // Stop timed flush
    if (this.flushTimer) {
      clearInterval(this.flushTimer);
      this.flushTimer = undefined;
    }
    
    this.stopped = true;
    console.log('Batch manager stopped');
  }
  
  /**
   * Start timed flush
   */
  private startTimedFlush(): void {
    this.flushTimer = setInterval(() => {
      this.flush().catch(error => {
        console.error('Error during timed flush:', error);
      });
    }, this.config.timeBased.intervalMs);
  }
  
  /**
   * Flush a specific category
   * 
   * @param category - Category to flush
   */
  private async flushCategory(category: string): Promise<void> {
    const batch = this.batches.get(category);
    
    if (!batch || batch.length === 0) {
      return;
    }
    
    try {
      // Sort batch by priority
      batch.sort((a, b) => {
        const priorityOrder: Record<string, number> = {
          'batch_critical': 0,
          'batch_high': 1,
          'batch_normal': 2,
          'batch_low': 3
        };
        
        return priorityOrder[a.priority] - priorityOrder[b.priority];
      });
      
      // Prepare items for storage
      const items = batch.map(item => ({
        data: item.data,
        metadata: item.metadata
      }));
      
      // Store batch
      await this.provider.storeBatch(items);
      
      // Update stats
      this.stats.totalBatches++;
      this.stats.pendingItems -= batch.length;
      this.stats.averageBatchSize = (this.stats.averageBatchSize * (this.stats.totalBatches - 1) + batch.length) / this.stats.totalBatches;
      
      // Clear batch
      this.batches.set(category, []);
    } catch (error) {
      console.error(`Failed to flush batch for category ${category}:`, error);
      this.stats.failedBatches++;
      throw new StorageError(
        StorageErrorCode.STORE_FAILED,
        `Failed to flush batch for category ${category}`,
        { category, error }
      );
    }
  }
  
  /**
   * Ensure batch manager is initialized
   * 
   * @throws StorageError if not initialized
   */
  private ensureInitialized(): void {
    if (!this.initialized) {
      throw new StorageError(
        StorageErrorCode.SYSTEM_ERROR,
        'Batch manager not initialized',
        {}
      );
    }
    
    if (this.stopped) {
      throw new StorageError(
        StorageErrorCode.SYSTEM_ERROR,
        'Batch manager has been stopped',
        {}
      );
    }
  }
}

/**
 * Create a new batch manager
 * 
 * @param provider - Storage provider
 * @returns Batch manager
 */
export function createBatchManager(provider: StorageProvider): BatchManager {
  return new BatchManagerImpl(provider);
}