@sailboat-computer/data-storage
Version:
Shared data storage library for sailboat computer v3
608 lines • 26.3 kB
JavaScript
;
/**
* Geotemporal grid downsampling strategy implementation
*
* Specialized for weather and sea state data using a grid-based approach
* that combines spatial and temporal dimensions.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.createGeotemporalGridDownsampler = exports.GeotemporalGridDownsampler = void 0;
/**
* Geotemporal grid downsampler implementation
*/
class GeotemporalGridDownsampler {
constructor() {
this.type = 'geotemporal-grid';
}
/**
* Downsample data using geotemporal grid strategy
*
* @param data - Data to downsample
* @param strategy - Geotemporal grid strategy
* @returns Downsampled data
*/
async downsample(data, strategy) {
if (data.length <= 2) {
// No downsampling needed for 0, 1, or 2 points
return data;
}
try {
// 1. Extract location and time from each data point
const geotemporalPoints = this.extractGeotemporalPoints(data);
if (geotemporalPoints.length <= 2) {
// Not enough valid points for downsampling
return data;
}
// 2. Create grid structure
const grid = this.createAdaptiveGrid(geotemporalPoints, strategy.baseGridSize, strategy.adaptiveGridLevels, strategy.vesselPosition, strategy.plannedRoute, strategy.coastalResolutionBoost);
// 3. Assign data points to grid cells
this.assignPointsToGridCells(geotemporalPoints, grid);
// 4. For each cell, create temporal buckets
this.createTemporalBuckets(grid, strategy.temporalHierarchy);
// 5. Identify critical weather features
if (strategy.criticalFeatureThresholds) {
this.identifyCriticalFeatures(geotemporalPoints, strategy.criticalFeatureThresholds);
}
// 6. Aggregate data within each cell-time bucket
this.aggregateDataInBuckets(grid);
// 7. Generate representative data points from aggregated buckets
const result = this.generateRepresentativePoints(grid);
// 8. Add downsampling metadata
for (const item of result) {
item.metadata.tags = {
...item.metadata.tags,
downsampled: 'true',
downsampledFrom: data.length.toString(),
downsamplingStrategy: 'geotemporal-grid'
};
}
return result;
}
catch (error) {
console.error('Failed to apply geotemporal grid downsampling:', error);
// Return original data on error
return data;
}
}
/**
* Extract geotemporal points from data
*
* @param data - Data to extract points from
* @returns Extracted points
*/
extractGeotemporalPoints(data) {
const points = [];
for (const item of data) {
// Try to extract latitude and longitude from data
const lat = this.extractCoordinate(item, 'latitude', 'lat');
const lon = this.extractCoordinate(item, 'longitude', 'lon', 'lng');
if (lat !== null && lon !== null) {
points.push({
latitude: lat,
longitude: lon,
timestamp: new Date(item.metadata.timestamp),
data: item
});
}
}
return points;
}
/**
* Extract coordinate from data
*
* @param item - Data item
* @param primaryKey - Primary key to look for
* @param alternateKeys - Alternate keys to look for
* @returns Coordinate value or null if not found
*/
extractCoordinate(item, primaryKey, ...alternateKeys) {
// Check primary key
if (typeof item.data[primaryKey] === 'number') {
return item.data[primaryKey];
}
// Check alternate keys
for (const key of alternateKeys) {
if (typeof item.data[key] === 'number') {
return item.data[key];
}
}
// Check if coordinates are in a nested object
for (const key of ['position', 'location', 'coordinates', 'coord', 'gps']) {
if (item.data[key] && typeof item.data[key] === 'object') {
const obj = item.data[key];
// Check primary key in nested object
if (typeof obj[primaryKey] === 'number') {
return obj[primaryKey];
}
// Check alternate keys in nested object
for (const altKey of alternateKeys) {
if (typeof obj[altKey] === 'number') {
return obj[altKey];
}
}
}
}
return null;
}
/**
* Create adaptive grid based on data distribution
*
* @param points - Points to create grid for
* @param baseGridSize - Base grid size in degrees
* @param adaptiveGridLevels - Number of resolution levels
* @param vesselPosition - Current vessel position
* @param plannedRoute - Planned route
* @param coastalResolutionBoost - Factor to increase resolution near coasts
* @returns Grid structure
*/
createAdaptiveGrid(points, baseGridSize, adaptiveGridLevels, vesselPosition, plannedRoute, coastalResolutionBoost) {
const grid = new Map();
// Calculate bounds
let minLat = 90;
let maxLat = -90;
let minLon = 180;
let maxLon = -180;
for (const point of points) {
minLat = Math.min(minLat, point.latitude);
maxLat = Math.max(maxLat, point.latitude);
minLon = Math.min(minLon, point.longitude);
maxLon = Math.max(maxLon, point.longitude);
}
// Add some padding
minLat -= baseGridSize;
maxLat += baseGridSize;
minLon -= baseGridSize;
maxLon += baseGridSize;
// Create base grid
for (let lat = minLat; lat <= maxLat; lat += baseGridSize) {
for (let lon = minLon; lon <= maxLon; lon += baseGridSize) {
const cellKey = this.getCellKey(lat, lon, 0);
grid.set(cellKey, {
latitude: lat,
longitude: lon,
level: 0,
temporalBuckets: new Map()
});
}
}
// Create adaptive grid levels if needed
if (adaptiveGridLevels > 0 && vesselPosition) {
this.createAdaptiveGridLevels(grid, baseGridSize, adaptiveGridLevels, vesselPosition, plannedRoute, coastalResolutionBoost);
}
return grid;
}
/**
* Create adaptive grid levels
*
* @param grid - Base grid
* @param baseGridSize - Base grid size in degrees
* @param adaptiveGridLevels - Number of resolution levels
* @param vesselPosition - Current vessel position
* @param plannedRoute - Planned route
* @param coastalResolutionBoost - Factor to increase resolution near coasts
*/
createAdaptiveGridLevels(grid, baseGridSize, adaptiveGridLevels, vesselPosition, plannedRoute, coastalResolutionBoost) {
// Calculate vessel cell
const vesselCellLat = Math.floor(vesselPosition.latitude / baseGridSize) * baseGridSize;
const vesselCellLon = Math.floor(vesselPosition.longitude / baseGridSize) * baseGridSize;
// Create higher resolution cells around vessel
for (let level = 1; level <= adaptiveGridLevels; level++) {
const levelGridSize = baseGridSize / Math.pow(2, level);
const radius = Math.max(1, adaptiveGridLevels - level + 1); // Radius decreases with level
// Create cells in a radius around vessel
for (let latOffset = -radius; latOffset <= radius; latOffset++) {
for (let lonOffset = -radius; lonOffset <= radius; lonOffset++) {
const centerLat = vesselCellLat + latOffset * baseGridSize;
const centerLon = vesselCellLon + lonOffset * baseGridSize;
// Create higher resolution cells
for (let lat = centerLat; lat < centerLat + baseGridSize; lat += levelGridSize) {
for (let lon = centerLon; lon < centerLon + baseGridSize; lon += levelGridSize) {
const cellKey = this.getCellKey(lat, lon, level);
grid.set(cellKey, {
latitude: lat,
longitude: lon,
level: level,
temporalBuckets: new Map()
});
}
}
}
}
}
// Add higher resolution cells along planned route if provided
if (plannedRoute && plannedRoute.length > 0) {
for (const waypoint of plannedRoute) {
const waypointCellLat = Math.floor(waypoint.latitude / baseGridSize) * baseGridSize;
const waypointCellLon = Math.floor(waypoint.longitude / baseGridSize) * baseGridSize;
// Create higher resolution cells around waypoint
for (let level = 1; level <= adaptiveGridLevels - 1; level++) { // One level less than vessel
const levelGridSize = baseGridSize / Math.pow(2, level);
// Create cells in a smaller radius around waypoint
for (let lat = waypointCellLat; lat < waypointCellLat + baseGridSize; lat += levelGridSize) {
for (let lon = waypointCellLon; lon < waypointCellLon + baseGridSize; lon += levelGridSize) {
const cellKey = this.getCellKey(lat, lon, level);
if (!grid.has(cellKey)) {
grid.set(cellKey, {
latitude: lat,
longitude: lon,
level: level,
temporalBuckets: new Map()
});
}
}
}
}
}
}
// Apply coastal resolution boost if provided
// This would require a coastline database, which is beyond the scope of this example
// In a real implementation, we would check if a cell is near a coast and increase its resolution
}
/**
* Get cell key for grid
*
* @param lat - Latitude
* @param lon - Longitude
* @param level - Grid level
* @returns Cell key
*/
getCellKey(lat, lon, level) {
return `${lat.toFixed(6)}_${lon.toFixed(6)}_${level}`;
}
/**
* Assign points to grid cells
*
* @param points - Points to assign
* @param grid - Grid structure
*/
assignPointsToGridCells(points, grid) {
// Sort points by timestamp
points.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
// Find the highest resolution cell for each point
for (const point of points) {
let highestLevel = -1;
let bestCell = null;
// Check all cells to find the highest resolution cell that contains the point
for (const [_, cell] of grid) {
const latDiff = point.latitude - cell.latitude;
const lonDiff = point.longitude - cell.longitude;
// Calculate cell size based on level
const cellSize = 1.0 / Math.pow(2, cell.level);
// Check if point is in cell
if (latDiff >= 0 && latDiff < cellSize && lonDiff >= 0 && lonDiff < cellSize) {
// If this cell has a higher level (resolution) than the current best, use it
if (cell.level > highestLevel) {
highestLevel = cell.level;
bestCell = cell;
}
}
}
// If we found a cell, add the point to it
if (bestCell) {
// We'll add it to temporal buckets later
if (!bestCell.temporalBuckets.has('all')) {
bestCell.temporalBuckets.set('all', {
startTime: new Date(0),
endTime: new Date(8640000000000000), // Max date
resolution: 0,
points: []
});
}
bestCell.temporalBuckets.get('all').points.push(point.data);
}
}
}
/**
* Create temporal buckets within each grid cell
*
* @param grid - Grid structure
* @param temporalHierarchy - Temporal hierarchy configuration
*/
createTemporalBuckets(grid, temporalHierarchy) {
const now = new Date();
// Process each cell
for (const [_, cell] of grid) {
// Skip cells with no points
if (!cell.temporalBuckets.has('all') || cell.temporalBuckets.get('all').points.length === 0) {
continue;
}
const points = cell.temporalBuckets.get('all').points;
cell.temporalBuckets.delete('all'); // Remove temporary bucket
// Create buckets for each temporal level
this.createTemporalLevel(cell, points, 'recent', temporalHierarchy.recent, now);
this.createTemporalLevel(cell, points, 'mediumTerm', temporalHierarchy.mediumTerm, now);
this.createTemporalLevel(cell, points, 'longTerm', temporalHierarchy.longTerm, now);
this.createTemporalLevel(cell, points, 'historical', temporalHierarchy.historical, now);
this.createTemporalLevel(cell, points, 'seasonal', temporalHierarchy.seasonal, now);
}
}
/**
* Create temporal level buckets
*
* @param cell - Grid cell
* @param points - Points to assign
* @param levelName - Level name
* @param levelConfig - Level configuration
* @param now - Current time
*/
createTemporalLevel(cell, points, levelName, levelConfig, now) {
// Calculate time range for this level
const startTime = levelConfig.maxAge
? new Date(now.getTime() - levelConfig.maxAge * 24 * 60 * 60 * 1000)
: new Date(0);
const endTime = levelName === 'recent'
? now
: levelConfig.maxAge
? new Date(now.getTime() - (levelConfig.maxAge - (levelName === 'mediumTerm' ? temporalHierarchy.recent.maxAge :
levelName === 'longTerm' ? temporalHierarchy.mediumTerm.maxAge :
temporalHierarchy.longTerm.maxAge)) * 24 * 60 * 60 * 1000)
: new Date(8640000000000000); // Max date for seasonal
// Filter points for this time range
const levelPoints = points.filter(point => {
const timestamp = new Date(point.metadata.timestamp);
return timestamp >= startTime && timestamp <= endTime;
});
if (levelPoints.length === 0) {
return; // No points in this time range
}
// Create buckets based on resolution
const resolutionMs = levelConfig.resolution * 60 * 60 * 1000; // Convert hours to ms
// Find min and max timestamps
let minTime = new Date(8640000000000000);
let maxTime = new Date(0);
for (const point of levelPoints) {
const timestamp = new Date(point.metadata.timestamp);
if (timestamp < minTime)
minTime = timestamp;
if (timestamp > maxTime)
maxTime = timestamp;
}
// Create buckets
for (let time = minTime.getTime(); time <= maxTime.getTime(); time += resolutionMs) {
const bucketStart = new Date(time);
const bucketEnd = new Date(time + resolutionMs);
const bucketKey = `${levelName}_${bucketStart.toISOString()}`;
// Filter points for this bucket
const bucketPoints = levelPoints.filter(point => {
const timestamp = new Date(point.metadata.timestamp);
return timestamp >= bucketStart && timestamp < bucketEnd;
});
if (bucketPoints.length > 0) {
cell.temporalBuckets.set(bucketKey, {
startTime: bucketStart,
endTime: bucketEnd,
resolution: levelConfig.resolution,
points: bucketPoints
});
}
}
}
/**
* Identify critical weather features
*
* @param points - Points to check
* @param thresholds - Critical feature thresholds
*/
identifyCriticalFeatures(points, thresholds) {
// If no thresholds are defined, return early
if (!thresholds) {
return;
}
// Check each point against thresholds
for (const point of points) {
const data = point.data.data;
// Check pressure change rate
if (thresholds.pressureChangeRate !== undefined && data.pressureChangeRate &&
Math.abs(data.pressureChangeRate) >= thresholds.pressureChangeRate) {
point.isCriticalFeature = true;
continue;
}
// Check wind speed
if (thresholds.windSpeed !== undefined &&
(data.windSpeed || data.wind?.speed) >= thresholds.windSpeed) {
point.isCriticalFeature = true;
continue;
}
// Check wave height
if (thresholds.waveHeight !== undefined &&
(data.waveHeight || data.significantWaveHeight || data.waves?.height) >= thresholds.waveHeight) {
point.isCriticalFeature = true;
continue;
}
// Check temperature gradient (would require comparing with nearby points)
// This is a simplification - in a real implementation we would calculate actual gradients
if (thresholds.temperatureGradient !== undefined && data.temperatureGradient &&
Math.abs(data.temperatureGradient) >= thresholds.temperatureGradient) {
point.isCriticalFeature = true;
}
}
}
/**
* Aggregate data within each cell-time bucket
*
* @param grid - Grid structure
*/
aggregateDataInBuckets(grid) {
// Process each cell
for (const [_, cell] of grid) {
// Process each temporal bucket
for (const [bucketKey, bucket] of cell.temporalBuckets) {
// Skip buckets with no points
if (bucket.points.length === 0) {
continue;
}
// Create aggregated data object
const aggregatedData = {
count: bucket.points.length,
timestamp: new Date((bucket.startTime.getTime() + bucket.endTime.getTime()) / 2).toISOString(),
location: {
latitude: cell.latitude + (1.0 / Math.pow(2, cell.level)) / 2,
longitude: cell.longitude + (1.0 / Math.pow(2, cell.level)) / 2
}
};
// Find all numeric fields to aggregate
const numericFields = new Set();
const criticalPoints = bucket.points.filter(p => p.isCriticalFeature ||
p.metadata.tags?.critical === 'true');
// Collect all numeric fields
for (const point of bucket.points) {
this.collectNumericFields(point.data, '', numericFields);
}
// Calculate aggregates for each numeric field
for (const field of numericFields) {
const values = this.extractValues(bucket.points, field);
if (values.length > 0) {
// Calculate basic statistics
aggregatedData[`${field}_min`] = Math.min(...values);
aggregatedData[`${field}_max`] = Math.max(...values);
aggregatedData[`${field}_avg`] = values.reduce((a, b) => a + b, 0) / values.length;
// Add more advanced statistics if needed
if (values.length > 1) {
// Calculate standard deviation
const mean = aggregatedData[`${field}_avg`];
const variance = values.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / values.length;
aggregatedData[`${field}_std`] = Math.sqrt(variance);
}
}
}
// Store critical points separately if any
if (criticalPoints.length > 0) {
aggregatedData.criticalPoints = criticalPoints.map(p => p.data);
}
// Store aggregated data
bucket.aggregatedData = aggregatedData;
}
}
}
/**
* Collect numeric fields from data object
*
* @param data - Data object
* @param prefix - Field prefix
* @param fields - Set to collect fields in
*/
collectNumericFields(data, prefix, fields) {
if (!data || typeof data !== 'object') {
return;
}
for (const [key, value] of Object.entries(data)) {
const fieldName = prefix ? `${prefix}.${key}` : key;
if (typeof value === 'number') {
fields.add(fieldName);
}
else if (typeof value === 'object' && value !== null) {
this.collectNumericFields(value, fieldName, fields);
}
}
}
/**
* Extract values for a field from data points
*
* @param points - Data points
* @param field - Field to extract
* @returns Extracted values
*/
extractValues(points, field) {
const values = [];
for (const point of points) {
const value = this.getNestedValue(point.data, field);
if (typeof value === 'number') {
values.push(value);
}
}
return values;
}
/**
* Get nested value from object
*
* @param obj - Object to get value from
* @param path - Path to value
* @returns Value or undefined
*/
getNestedValue(obj, path) {
const parts = path.split('.');
let current = obj;
for (const part of parts) {
if (current === null || current === undefined) {
return undefined;
}
current = current[part];
}
return current;
}
/**
* Generate representative points from aggregated buckets
*
* @param grid - Grid structure
* @returns Representative points
*/
generateRepresentativePoints(grid) {
const result = [];
// Process each cell
for (const [_, cell] of grid) {
// Process each temporal bucket
for (const [bucketKey, bucket] of cell.temporalBuckets) {
// Skip buckets with no aggregated data
if (!bucket.aggregatedData) {
continue;
}
// Create representative point
const representativePoint = {
data: bucket.aggregatedData,
metadata: {
category: bucket.points[0].metadata.category,
timestamp: bucket.aggregatedData.timestamp,
tags: {
...bucket.points[0].metadata.tags,
gridCell: `${cell.latitude.toFixed(6)},${cell.longitude.toFixed(6)}`,
gridLevel: cell.level.toString(),
bucketResolution: bucket.resolution.toString(),
pointCount: bucket.points.length.toString()
}
}
};
// Add to result
result.push(representativePoint);
// Add critical points as separate points if any
if (bucket.aggregatedData.criticalPoints) {
for (const criticalPoint of bucket.aggregatedData.criticalPoints) {
const originalPoint = bucket.points.find(p => p.data === criticalPoint);
if (originalPoint) {
result.push({
data: criticalPoint,
metadata: {
...originalPoint.metadata,
tags: {
...originalPoint.metadata.tags,
critical: 'true'
}
}
});
}
}
// Remove critical points from aggregated data to avoid duplication
delete bucket.aggregatedData.criticalPoints;
}
}
}
return result;
}
}
exports.GeotemporalGridDownsampler = GeotemporalGridDownsampler;
// For TypeScript to recognize the temporalHierarchy variable
const temporalHierarchy = {
recent: { maxAge: 7, resolution: 1 },
mediumTerm: { maxAge: 30, resolution: 6 },
longTerm: { maxAge: 90, resolution: 24 },
historical: { maxAge: 365, resolution: 168 }
};
/**
* Create a new geotemporal grid downsampler
*
* @returns Geotemporal grid downsampler
*/
function createGeotemporalGridDownsampler() {
return new GeotemporalGridDownsampler();
}
exports.createGeotemporalGridDownsampler = createGeotemporalGridDownsampler;
//# sourceMappingURL=geotemporal-grid.js.map