@sailboat-computer/data-storage/src/downsampling/strategies/geotemporal-grid.ts

Shared data storage library for sailboat computer v3

/**
 * Geotemporal grid downsampling strategy implementation
 * 
 * Specialized for weather and sea state data using a grid-based approach
 * that combines spatial and temporal dimensions.
 */

import { Downsampler, StoredData, GeotemporalGridStrategy } from '../../types';
// TODO: Import from generated types once available
// import { GeotemporalGridStrategy } from '@sailboat-computer/types';
import { StorageError, StorageErrorCode } from '../../utils/errors';

/**
 * Grid cell interface
 */
interface GridCell {
  latitude: number;
  longitude: number;
  level: number;
  temporalBuckets: Map<string, TemporalBucket>;
}

/**
 * Temporal bucket interface
 */
interface TemporalBucket {
  startTime: Date;
  endTime: Date;
  resolution: number; // in hours
  points: StoredData[];
  aggregatedData?: any;
}

/**
 * Geotemporal point interface
 */
interface GeotemporalPoint {
  latitude: number;
  longitude: number;
  timestamp: Date;
  data: StoredData;
  isCriticalFeature?: boolean;
}

/**
 * Geotemporal grid downsampler implementation
 */
export class GeotemporalGridDownsampler implements Downsampler {
  readonly type = 'geotemporal-grid';
  
  /**
   * Downsample data using geotemporal grid strategy
   * 
   * @param data - Data to downsample
   * @param strategy - Geotemporal grid strategy
   * @returns Downsampled data
   */
  async downsample(data: StoredData[], strategy: GeotemporalGridStrategy): Promise<StoredData[]> {
    if (data.length <= 2) {
      // No downsampling needed for 0, 1, or 2 points
      return data;
    }
    
    try {
      // 1. Extract location and time from each data point
      const geotemporalPoints = this.extractGeotemporalPoints(data);
      
      if (geotemporalPoints.length <= 2) {
        // Not enough valid points for downsampling
        return data;
      }
      
      // 2. Create grid structure
      const grid = this.createAdaptiveGrid(
        geotemporalPoints, 
        strategy.baseGridSize,
        strategy.adaptiveGridLevels,
        strategy.vesselPosition,
        strategy.plannedRoute,
        strategy.coastalResolutionBoost
      );
      
      // 3. Assign data points to grid cells
      this.assignPointsToGridCells(geotemporalPoints, grid);
      
      // 4. For each cell, create temporal buckets
      this.createTemporalBuckets(grid, strategy.temporalHierarchy);
      
      // 5. Identify critical weather features
      if (strategy.criticalFeatureThresholds) {
        this.identifyCriticalFeatures(geotemporalPoints, strategy.criticalFeatureThresholds);
      }
      
      // 6. Aggregate data within each cell-time bucket
      this.aggregateDataInBuckets(grid);
      
      // 7. Generate representative data points from aggregated buckets
      const result = this.generateRepresentativePoints(grid);
      
      // 8. Add downsampling metadata
      for (const item of result) {
        item.metadata.tags = {
          ...item.metadata.tags,
          downsampled: 'true',
          downsampledFrom: data.length.toString(),
          downsamplingStrategy: 'geotemporal-grid'
        };
      }
      
      return result;
    } catch (error) {
      console.error('Failed to apply geotemporal grid downsampling:', error);
      // Return original data on error
      return data;
    }
  }
  
  /**
   * Extract geotemporal points from data
   * 
   * @param data - Data to extract points from
   * @returns Extracted points
   */
  private extractGeotemporalPoints(data: StoredData[]): GeotemporalPoint[] {
    const points: GeotemporalPoint[] = [];
    
    for (const item of data) {
      // Try to extract latitude and longitude from data
      const lat = this.extractCoordinate(item, 'latitude', 'lat');
      const lon = this.extractCoordinate(item, 'longitude', 'lon', 'lng');
      
      if (lat !== null && lon !== null) {
        points.push({
          latitude: lat,
          longitude: lon,
          timestamp: new Date(item.metadata.timestamp),
          data: item
        });
      }
    }
    
    return points;
  }
  
  /**
   * Extract coordinate from data
   * 
   * @param item - Data item
   * @param primaryKey - Primary key to look for
   * @param alternateKeys - Alternate keys to look for
   * @returns Coordinate value or null if not found
   */
  private extractCoordinate(item: StoredData, primaryKey: string, ...alternateKeys: string[]): number | null {
    // Check primary key
    if (typeof item.data[primaryKey] === 'number') {
      return item.data[primaryKey];
    }
    
    // Check alternate keys
    for (const key of alternateKeys) {
      if (typeof item.data[key] === 'number') {
        return item.data[key];
      }
    }
    
    // Check if coordinates are in a nested object
    for (const key of ['position', 'location', 'coordinates', 'coord', 'gps']) {
      if (item.data[key] && typeof item.data[key] === 'object') {
        const obj = item.data[key];
        
        // Check primary key in nested object
        if (typeof obj[primaryKey] === 'number') {
          return obj[primaryKey];
        }
        
        // Check alternate keys in nested object
        for (const altKey of alternateKeys) {
          if (typeof obj[altKey] === 'number') {
            return obj[altKey];
          }
        }
      }
    }
    
    return null;
  }
  
  /**
   * Create adaptive grid based on data distribution
   * 
   * @param points - Points to create grid for
   * @param baseGridSize - Base grid size in degrees
   * @param adaptiveGridLevels - Number of resolution levels
   * @param vesselPosition - Current vessel position
   * @param plannedRoute - Planned route
   * @param coastalResolutionBoost - Factor to increase resolution near coasts
   * @returns Grid structure
   */
  private createAdaptiveGrid(
    points: GeotemporalPoint[],
    baseGridSize: number,
    adaptiveGridLevels: number,
    vesselPosition?: { latitude: number, longitude: number },
    plannedRoute?: Array<{ latitude: number, longitude: number }>,
    coastalResolutionBoost?: number
  ): Map<string, GridCell> {
    const grid = new Map<string, GridCell>();
    
    // Calculate bounds
    let minLat = 90;
    let maxLat = -90;
    let minLon = 180;
    let maxLon = -180;
    
    for (const point of points) {
      minLat = Math.min(minLat, point.latitude);
      maxLat = Math.max(maxLat, point.latitude);
      minLon = Math.min(minLon, point.longitude);
      maxLon = Math.max(maxLon, point.longitude);
    }
    
    // Add some padding
    minLat -= baseGridSize;
    maxLat += baseGridSize;
    minLon -= baseGridSize;
    maxLon += baseGridSize;
    
    // Create base grid
    for (let lat = minLat; lat <= maxLat; lat += baseGridSize) {
      for (let lon = minLon; lon <= maxLon; lon += baseGridSize) {
        const cellKey = this.getCellKey(lat, lon, 0);
        grid.set(cellKey, {
          latitude: lat,
          longitude: lon,
          level: 0,
          temporalBuckets: new Map<string, TemporalBucket>()
        });
      }
    }
    
    // Create adaptive grid levels if needed
    if (adaptiveGridLevels > 0 && vesselPosition) {
      this.createAdaptiveGridLevels(
        grid,
        baseGridSize,
        adaptiveGridLevels,
        vesselPosition,
        plannedRoute,
        coastalResolutionBoost
      );
    }
    
    return grid;
  }
  
  /**
   * Create adaptive grid levels
   * 
   * @param grid - Base grid
   * @param baseGridSize - Base grid size in degrees
   * @param adaptiveGridLevels - Number of resolution levels
   * @param vesselPosition - Current vessel position
   * @param plannedRoute - Planned route
   * @param coastalResolutionBoost - Factor to increase resolution near coasts
   */
  private createAdaptiveGridLevels(
    grid: Map<string, GridCell>,
    baseGridSize: number,
    adaptiveGridLevels: number,
    vesselPosition: { latitude: number, longitude: number },
    plannedRoute?: Array<{ latitude: number, longitude: number }>,
    coastalResolutionBoost?: number
  ): void {
    // Calculate vessel cell
    const vesselCellLat = Math.floor(vesselPosition.latitude / baseGridSize) * baseGridSize;
    const vesselCellLon = Math.floor(vesselPosition.longitude / baseGridSize) * baseGridSize;
    
    // Create higher resolution cells around vessel
    for (let level = 1; level <= adaptiveGridLevels; level++) {
      const levelGridSize = baseGridSize / Math.pow(2, level);
      const radius = Math.max(1, adaptiveGridLevels - level + 1); // Radius decreases with level
      
      // Create cells in a radius around vessel
      for (let latOffset = -radius; latOffset <= radius; latOffset++) {
        for (let lonOffset = -radius; lonOffset <= radius; lonOffset++) {
          const centerLat = vesselCellLat + latOffset * baseGridSize;
          const centerLon = vesselCellLon + lonOffset * baseGridSize;
          
          // Create higher resolution cells
          for (let lat = centerLat; lat < centerLat + baseGridSize; lat += levelGridSize) {
            for (let lon = centerLon; lon < centerLon + baseGridSize; lon += levelGridSize) {
              const cellKey = this.getCellKey(lat, lon, level);
              grid.set(cellKey, {
                latitude: lat,
                longitude: lon,
                level: level,
                temporalBuckets: new Map<string, TemporalBucket>()
              });
            }
          }
        }
      }
    }
    
    // Add higher resolution cells along planned route if provided
    if (plannedRoute && plannedRoute.length > 0) {
      for (const waypoint of plannedRoute) {
        const waypointCellLat = Math.floor(waypoint.latitude / baseGridSize) * baseGridSize;
        const waypointCellLon = Math.floor(waypoint.longitude / baseGridSize) * baseGridSize;
        
        // Create higher resolution cells around waypoint
        for (let level = 1; level <= adaptiveGridLevels - 1; level++) { // One level less than vessel
          const levelGridSize = baseGridSize / Math.pow(2, level);
          
          // Create cells in a smaller radius around waypoint
          for (let lat = waypointCellLat; lat < waypointCellLat + baseGridSize; lat += levelGridSize) {
            for (let lon = waypointCellLon; lon < waypointCellLon + baseGridSize; lon += levelGridSize) {
              const cellKey = this.getCellKey(lat, lon, level);
              if (!grid.has(cellKey)) {
                grid.set(cellKey, {
                  latitude: lat,
                  longitude: lon,
                  level: level,
                  temporalBuckets: new Map<string, TemporalBucket>()
                });
              }
            }
          }
        }
      }
    }
    
    // Apply coastal resolution boost if provided
    // This would require a coastline database, which is beyond the scope of this example
    // In a real implementation, we would check if a cell is near a coast and increase its resolution
  }
  
  /**
   * Get cell key for grid
   * 
   * @param lat - Latitude
   * @param lon - Longitude
   * @param level - Grid level
   * @returns Cell key
   */
  private getCellKey(lat: number, lon: number, level: number): string {
    return `${lat.toFixed(6)}_${lon.toFixed(6)}_${level}`;
  }
  
  /**
   * Assign points to grid cells
   * 
   * @param points - Points to assign
   * @param grid - Grid structure
   */
  private assignPointsToGridCells(
    points: GeotemporalPoint[],
    grid: Map<string, GridCell>
  ): void {
    // Sort points by timestamp
    points.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
    
    // Find the highest resolution cell for each point
    for (const point of points) {
      let highestLevel = -1;
      let bestCell: GridCell | null = null;
      
      // Check all cells to find the highest resolution cell that contains the point
      for (const [_, cell] of grid) {
        const latDiff = point.latitude - cell.latitude;
        const lonDiff = point.longitude - cell.longitude;
        
        // Calculate cell size based on level
        const cellSize = 1.0 / Math.pow(2, cell.level);
        
        // Check if point is in cell
        if (latDiff >= 0 && latDiff < cellSize && lonDiff >= 0 && lonDiff < cellSize) {
          // If this cell has a higher level (resolution) than the current best, use it
          if (cell.level > highestLevel) {
            highestLevel = cell.level;
            bestCell = cell;
          }
        }
      }
      
      // If we found a cell, add the point to it
      if (bestCell) {
        // We'll add it to temporal buckets later
        if (!bestCell.temporalBuckets.has('all')) {
          bestCell.temporalBuckets.set('all', {
            startTime: new Date(0),
            endTime: new Date(8640000000000000), // Max date
            resolution: 0,
            points: []
          });
        }
        
        bestCell.temporalBuckets.get('all')!.points.push(point.data);
      }
    }
  }
  
  /**
   * Create temporal buckets within each grid cell
   * 
   * @param grid - Grid structure
   * @param temporalHierarchy - Temporal hierarchy configuration
   */
  private createTemporalBuckets(
    grid: Map<string, GridCell>,
    temporalHierarchy: GeotemporalGridStrategy['temporalHierarchy']
  ): void {
    const now = new Date();
    
    // Process each cell
    for (const [_, cell] of grid) {
      // Skip cells with no points
      if (!cell.temporalBuckets.has('all') || cell.temporalBuckets.get('all')!.points.length === 0) {
        continue;
      }
      
      const points = cell.temporalBuckets.get('all')!.points;
      cell.temporalBuckets.delete('all'); // Remove temporary bucket
      
      // Create buckets for each temporal level
      this.createTemporalLevel(cell, points, 'recent', temporalHierarchy.recent, now);
      this.createTemporalLevel(cell, points, 'mediumTerm', temporalHierarchy.mediumTerm, now);
      this.createTemporalLevel(cell, points, 'longTerm', temporalHierarchy.longTerm, now);
      this.createTemporalLevel(cell, points, 'historical', temporalHierarchy.historical, now);
      this.createTemporalLevel(cell, points, 'seasonal', temporalHierarchy.seasonal, now);
    }
  }
  
  /**
   * Create temporal level buckets
   * 
   * @param cell - Grid cell
   * @param points - Points to assign
   * @param levelName - Level name
   * @param levelConfig - Level configuration
   * @param now - Current time
   */
  private createTemporalLevel(
    cell: GridCell,
    points: StoredData[],
    levelName: string,
    levelConfig: { maxAge?: number, resolution: number },
    now: Date
  ): void {
    // Calculate time range for this level
    const startTime = levelConfig.maxAge 
      ? new Date(now.getTime() - levelConfig.maxAge * 24 * 60 * 60 * 1000)
      : new Date(0);
    
    const endTime = levelName === 'recent' 
      ? now 
      : levelConfig.maxAge 
        ? new Date(now.getTime() - (levelConfig.maxAge - (levelName === 'mediumTerm' ? temporalHierarchy.recent.maxAge : 
                                                         levelName === 'longTerm' ? temporalHierarchy.mediumTerm.maxAge : 
                                                         temporalHierarchy.longTerm.maxAge)) * 24 * 60 * 60 * 1000)
        : new Date(8640000000000000); // Max date for seasonal
    
    // Filter points for this time range
    const levelPoints = points.filter(point => {
      const timestamp = new Date(point.metadata.timestamp);
      return timestamp >= startTime && timestamp <= endTime;
    });
    
    if (levelPoints.length === 0) {
      return; // No points in this time range
    }
    
    // Create buckets based on resolution
    const resolutionMs = levelConfig.resolution * 60 * 60 * 1000; // Convert hours to ms
    
    // Find min and max timestamps
    let minTime = new Date(8640000000000000);
    let maxTime = new Date(0);
    
    for (const point of levelPoints) {
      const timestamp = new Date(point.metadata.timestamp);
      if (timestamp < minTime) minTime = timestamp;
      if (timestamp > maxTime) maxTime = timestamp;
    }
    
    // Create buckets
    for (let time = minTime.getTime(); time <= maxTime.getTime(); time += resolutionMs) {
      const bucketStart = new Date(time);
      const bucketEnd = new Date(time + resolutionMs);
      const bucketKey = `${levelName}_${bucketStart.toISOString()}`;
      
      // Filter points for this bucket
      const bucketPoints = levelPoints.filter(point => {
        const timestamp = new Date(point.metadata.timestamp);
        return timestamp >= bucketStart && timestamp < bucketEnd;
      });
      
      if (bucketPoints.length > 0) {
        cell.temporalBuckets.set(bucketKey, {
          startTime: bucketStart,
          endTime: bucketEnd,
          resolution: levelConfig.resolution,
          points: bucketPoints
        });
      }
    }
  }
  
  /**
   * Identify critical weather features
   * 
   * @param points - Points to check
   * @param thresholds - Critical feature thresholds
   */
  private identifyCriticalFeatures(
    points: GeotemporalPoint[],
    thresholds: GeotemporalGridStrategy['criticalFeatureThresholds'] | undefined
  ): void {
    // If no thresholds are defined, return early
    if (!thresholds) {
      return;
    }
    
    // Check each point against thresholds
    for (const point of points) {
      const data = point.data.data;
      
      // Check pressure change rate
      if (thresholds.pressureChangeRate !== undefined && data.pressureChangeRate && 
          Math.abs(data.pressureChangeRate) >= thresholds.pressureChangeRate) {
        point.isCriticalFeature = true;
        continue;
      }
      
      // Check wind speed
      if (thresholds.windSpeed !== undefined && 
          (data.windSpeed || data.wind?.speed) >= thresholds.windSpeed) {
        point.isCriticalFeature = true;
        continue;
      }
      
      // Check wave height
      if (thresholds.waveHeight !== undefined && 
          (data.waveHeight || data.significantWaveHeight || data.waves?.height) >= thresholds.waveHeight) {
        point.isCriticalFeature = true;
        continue;
      }
      
      // Check temperature gradient (would require comparing with nearby points)
      // This is a simplification - in a real implementation we would calculate actual gradients
      if (thresholds.temperatureGradient !== undefined && data.temperatureGradient && 
          Math.abs(data.temperatureGradient) >= thresholds.temperatureGradient) {
        point.isCriticalFeature = true;
      }
    }
  }
  
  /**
   * Aggregate data within each cell-time bucket
   * 
   * @param grid - Grid structure
   */
  private aggregateDataInBuckets(grid: Map<string, GridCell>): void {
    // Process each cell
    for (const [_, cell] of grid) {
      // Process each temporal bucket
      for (const [bucketKey, bucket] of cell.temporalBuckets) {
        // Skip buckets with no points
        if (bucket.points.length === 0) {
          continue;
        }
        
        // Create aggregated data object
        const aggregatedData: Record<string, any> = {
          count: bucket.points.length,
          timestamp: new Date((bucket.startTime.getTime() + bucket.endTime.getTime()) / 2).toISOString(),
          location: {
            latitude: cell.latitude + (1.0 / Math.pow(2, cell.level)) / 2,
            longitude: cell.longitude + (1.0 / Math.pow(2, cell.level)) / 2
          }
        };
        
        // Find all numeric fields to aggregate
        const numericFields = new Set<string>();
        const criticalPoints = bucket.points.filter(p => 
          (p as any).isCriticalFeature || 
          p.metadata.tags?.critical === 'true'
        );
        
        // Collect all numeric fields
        for (const point of bucket.points) {
          this.collectNumericFields(point.data, '', numericFields);
        }
        
        // Calculate aggregates for each numeric field
        for (const field of numericFields) {
          const values = this.extractValues(bucket.points, field);
          
          if (values.length > 0) {
            // Calculate basic statistics
            aggregatedData[`${field}_min`] = Math.min(...values);
            aggregatedData[`${field}_max`] = Math.max(...values);
            aggregatedData[`${field}_avg`] = values.reduce((a, b) => a + b, 0) / values.length;
            
            // Add more advanced statistics if needed
            if (values.length > 1) {
              // Calculate standard deviation
              const mean = aggregatedData[`${field}_avg`];
              const variance = values.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / values.length;
              aggregatedData[`${field}_std`] = Math.sqrt(variance);
            }
          }
        }
        
        // Store critical points separately if any
        if (criticalPoints.length > 0) {
          aggregatedData.criticalPoints = criticalPoints.map(p => p.data);
        }
        
        // Store aggregated data
        bucket.aggregatedData = aggregatedData;
      }
    }
  }
  
  /**
   * Collect numeric fields from data object
   * 
   * @param data - Data object
   * @param prefix - Field prefix
   * @param fields - Set to collect fields in
   */
  private collectNumericFields(data: any, prefix: string, fields: Set<string>): void {
    if (!data || typeof data !== 'object') {
      return;
    }
    
    for (const [key, value] of Object.entries(data)) {
      const fieldName = prefix ? `${prefix}.${key}` : key;
      
      if (typeof value === 'number') {
        fields.add(fieldName);
      } else if (typeof value === 'object' && value !== null) {
        this.collectNumericFields(value, fieldName, fields);
      }
    }
  }
  
  /**
   * Extract values for a field from data points
   * 
   * @param points - Data points
   * @param field - Field to extract
   * @returns Extracted values
   */
  private extractValues(points: StoredData[], field: string): number[] {
    const values: number[] = [];
    
    for (const point of points) {
      const value = this.getNestedValue(point.data, field);
      if (typeof value === 'number') {
        values.push(value);
      }
    }
    
    return values;
  }
  
  /**
   * Get nested value from object
   * 
   * @param obj - Object to get value from
   * @param path - Path to value
   * @returns Value or undefined
   */
  private getNestedValue(obj: any, path: string): any {
    const parts = path.split('.');
    let current = obj;
    
    for (const part of parts) {
      if (current === null || current === undefined) {
        return undefined;
      }
      
      current = current[part];
    }
    
    return current;
  }
  
  /**
   * Generate representative points from aggregated buckets
   * 
   * @param grid - Grid structure
   * @returns Representative points
   */
  private generateRepresentativePoints(grid: Map<string, GridCell>): StoredData[] {
    const result: StoredData[] = [];
    
    // Process each cell
    for (const [_, cell] of grid) {
      // Process each temporal bucket
      for (const [bucketKey, bucket] of cell.temporalBuckets) {
        // Skip buckets with no aggregated data
        if (!bucket.aggregatedData) {
          continue;
        }
        
        // Create representative point
        const representativePoint: StoredData = {
          data: bucket.aggregatedData,
          metadata: {
            category: bucket.points[0].metadata.category,
            timestamp: bucket.aggregatedData.timestamp,
            tags: {
              ...bucket.points[0].metadata.tags,
              gridCell: `${cell.latitude.toFixed(6)},${cell.longitude.toFixed(6)}`,
              gridLevel: cell.level.toString(),
              bucketResolution: bucket.resolution.toString(),
              pointCount: bucket.points.length.toString()
            }
          }
        };
        
        // Add to result
        result.push(representativePoint);
        
        // Add critical points as separate points if any
        if (bucket.aggregatedData.criticalPoints) {
          for (const criticalPoint of bucket.aggregatedData.criticalPoints) {
            const originalPoint = bucket.points.find(p => p.data === criticalPoint);
            if (originalPoint) {
              result.push({
                data: criticalPoint,
                metadata: {
                  ...originalPoint.metadata,
                  tags: {
                    ...originalPoint.metadata.tags,
                    critical: 'true'
                  }
                }
              });
            }
          }
          
          // Remove critical points from aggregated data to avoid duplication
          delete bucket.aggregatedData.criticalPoints;
        }
      }
    }
    
    return result;
  }
}

// For TypeScript to recognize the temporalHierarchy variable
const temporalHierarchy = {
  recent: { maxAge: 7, resolution: 1 },
  mediumTerm: { maxAge: 30, resolution: 6 },
  longTerm: { maxAge: 90, resolution: 24 },
  historical: { maxAge: 365, resolution: 168 }
};

/**
 * Create a new geotemporal grid downsampler
 * 
 * @returns Geotemporal grid downsampler
 */
export function createGeotemporalGridDownsampler(): Downsampler {
  return new GeotemporalGridDownsampler();
}