signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
799 lines • 34 kB
JavaScript
;
/**
* Aggregation Service
*
* Handles multi-tier data aggregation from raw data to progressively
* downsampled tiers for efficient long-term storage and querying.
*
* Tiers:
* - raw: Original data (~1s resolution)
* - 5s: 5-second aggregates
* - 60s: 1-minute aggregates
* - 1h: Hourly aggregates
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.AggregationService = exports.TIER_RETENTION_MULTIPLIER = void 0;
exports.buildPerTierRetention = buildPerTierRetention;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const glob_1 = require("glob");
const duckdb_pool_1 = require("../utils/duckdb-pool");
const hive_path_builder_1 = require("../utils/hive-path-builder");
const angular_paths_1 = require("../utils/angular-paths");
const constants_1 = require("../constants");
const retention_rules_1 = require("../utils/retention-rules");
// Tier multipliers used when scaling a raw-tier retention to upper
// tiers. The same convention applies to the global default and to
// per-path overrides; exported so callers in index.ts and the route
// layer build identical AggregationConfig.retentionDays without
// drifting from this file.
exports.TIER_RETENTION_MULTIPLIER = {
raw: 1,
'5s': 2,
'60s': 4,
'1h': 12,
};
/**
* Build the per-tier retention block from a raw-tier value. 0 stays
* 0 in every tier (= keep forever).
*/
function buildPerTierRetention(rawDays) {
return {
raw: rawDays * exports.TIER_RETENTION_MULTIPLIER.raw,
'5s': rawDays * exports.TIER_RETENTION_MULTIPLIER['5s'],
'60s': rawDays * exports.TIER_RETENTION_MULTIPLIER['60s'],
'1h': rawDays * exports.TIER_RETENTION_MULTIPLIER['1h'],
};
}
const bulkAggregationJobs = new Map();
const BULK_JOB_TTL_MS = 60 * 60 * 1000; // 1 hour
function scheduleBulkJobCleanup(jobId) {
setTimeout(() => {
const job = bulkAggregationJobs.get(jobId);
if (job && job.status !== 'running' && job.status !== 'scanning') {
bulkAggregationJobs.delete(jobId);
}
}, BULK_JOB_TTL_MS);
}
const TIER_INTERVALS = {
raw: 1,
'5s': 5,
'60s': 60,
'1h': 3600,
};
const TIER_HIERARCHY = ['raw', '5s', '60s', '1h'];
class AggregationService {
constructor(config, app) {
this.currentJob = null;
this.cancelRequested = false;
this.config = config;
this.app = app;
this.hivePathBuilder = new hive_path_builder_1.HivePathBuilder();
this.retentionRules = new retention_rules_1.RetentionRuleSet(config.pathRetentionOverrides || [], (rule, err) => {
// A pattern that fails to compile is dropped; log so the
// operator sees the offending rule rather than silently losing
// it. Doesn't block plugin start.
this.app.error(`[Retention] Dropping rule with invalid pattern '${rule.pattern}': ${err.message}`);
});
}
/**
* Run aggregation for a specific date.
* Optional pathFilter restricts which signalk paths are aggregated (used by
* targeted migrations like position re-aggregation).
*/
async aggregateDate(date, pathFilter) {
const results = [];
// Aggregate through the hierarchy: raw -> 5s -> 60s -> 1h
for (let i = 0; i < TIER_HIERARCHY.length - 1; i++) {
const sourceTier = TIER_HIERARCHY[i];
const targetTier = TIER_HIERARCHY[i + 1];
try {
const result = await this.aggregateTier(sourceTier, targetTier, date, pathFilter);
results.push(result);
}
catch (error) {
this.app.error(`Aggregation ${sourceTier} -> ${targetTier} failed: ${error.message}`);
results.push({
sourceTier,
targetTier,
filesProcessed: 0,
recordsAggregated: 0,
filesCreated: 0,
duration: 0,
errors: [error.message],
});
}
}
return results;
}
/**
* Aggregate from one tier to the next
*/
async aggregateTier(sourceTier, targetTier, date, pathFilter) {
const startTime = Date.now();
const errors = [];
let filesProcessed = 0;
let recordsAggregated = 0;
let filesCreated = 0;
const year = date.getUTCFullYear();
const dayOfYear = this.hivePathBuilder.getDayOfYear(date);
// Find all source files for this date
const sourcePattern = path.join(this.config.outputDirectory, `tier=${sourceTier}`, 'context=*', 'path=*', `year=${year}`, `day=${String(dayOfYear).padStart(3, '0')}`, '*.parquet');
const allSourceFiles = await (0, glob_1.glob)(sourcePattern);
// Exclude files in processed, quarantine, failed, repaired directories
const sourceFiles = allSourceFiles.filter(f => !f.includes('/processed/') &&
!f.includes('/quarantine/') &&
!f.includes('/failed/') &&
!f.includes('/repaired/'));
if (sourceFiles.length === 0) {
this.app.debug(`No source files found for ${sourceTier} on ${date.toISOString().slice(0, 10)}`);
return {
sourceTier,
targetTier,
filesProcessed: 0,
recordsAggregated: 0,
filesCreated: 0,
duration: Date.now() - startTime,
errors: [],
};
}
// Group files by context and path
const fileGroups = this.groupFilesByContextPath(sourceFiles);
// Process each group
for (const [key, files] of fileGroups) {
if (this.cancelRequested)
break;
try {
const { context, signalkPath } = this.parseGroupKey(key);
if (pathFilter && !pathFilter(signalkPath))
continue;
// Honour per-path skipAggregation. These paths live only in
// tier=raw and are cleaned up directly via cleanupOldData.
if (this.retentionRules.shouldSkipAggregation(signalkPath)) {
this.app.debug(`Skipping aggregation for ${signalkPath}: matched a retention rule with skipAggregation=true`);
continue;
}
const result = await this.aggregateGroup(files, context, signalkPath, sourceTier, targetTier, date);
filesProcessed += files.length;
recordsAggregated += result.recordsAggregated;
if (result.outputFile) {
filesCreated++;
}
}
catch (error) {
const errorMsg = `Failed to aggregate ${key}: ${error.message}`;
this.app.error(errorMsg);
errors.push(errorMsg);
}
}
return {
sourceTier,
targetTier,
filesProcessed,
recordsAggregated,
filesCreated,
duration: Date.now() - startTime,
errors,
};
}
/**
* Aggregate a group of files for a specific context/path
*/
async aggregateGroup(files, context, signalkPath, sourceTier, targetTier, date) {
const intervalSeconds = TIER_INTERVALS[targetTier];
const isSourceRaw = sourceTier === 'raw';
const fileListStr = files.map(f => `'${f}'`).join(', ');
// Detect schema: scalar ('value' / 'value_avg'), position ('value_latitude'/'value_longitude'),
// or unsupported object-type (skip).
const connection = await duckdb_pool_1.DuckDBPool.getConnection();
let isPosition = false;
try {
const schemaQuery = `SELECT column_name FROM (DESCRIBE SELECT * FROM read_parquet([${fileListStr}], union_by_name=true))`;
const schemaResult = await connection.runAndReadAll(schemaQuery);
const columns = schemaResult
.getRowObjects()
.map((r) => r.column_name);
const hasLatLon = columns.includes('value_latitude') &&
columns.includes('value_longitude');
const requiredColumn = isSourceRaw ? 'value' : 'bucket_time';
const hasScalarColumn = columns.includes(requiredColumn);
if (hasLatLon) {
isPosition = true;
}
else if (!hasScalarColumn) {
this.app.debug(`Skipping ${signalkPath}: no '${requiredColumn}' column (object-type data stays in raw tier)`);
connection.disconnectSync();
return { recordsAggregated: 0, outputFile: null };
}
}
catch (error) {
connection.disconnectSync();
throw error;
}
// Build output path
const outputDir = this.hivePathBuilder.buildPath(this.config.outputDirectory, targetTier, context, signalkPath, date);
await fs.ensureDir(outputDir);
const outputFile = path.join(outputDir, `${this.config.filenamePrefix}_${date.toISOString().slice(0, 10)}_aggregated.parquet`);
// Use different query depending on source tier schema and whether the path is angular/position
// Raw tier has: received_timestamp, value (or value_latitude/value_longitude for position)
// Aggregated tiers have: bucket_time, value_avg/value_latitude, value_min, value_max, sample_count, first_timestamp, last_timestamp
const angular = !isPosition && (0, angular_paths_1.isAngularPath)(signalkPath, this.app, context);
const query = isPosition
? this.buildPositionAggregationQuery(fileListStr, intervalSeconds, isSourceRaw, outputFile)
: this.buildAggregationQuery(fileListStr, intervalSeconds, isSourceRaw, angular, outputFile);
try {
await connection.runAndReadAll(query);
// Get record count from output
const countQuery = `SELECT COUNT(*) as cnt FROM read_parquet('${outputFile}')`;
const countResult = await connection.runAndReadAll(countQuery);
const rows = countResult.getRowObjects();
const recordCount = rows[0]?.cnt || 0;
return {
recordsAggregated: Number(recordCount),
outputFile,
};
}
finally {
connection.disconnectSync();
}
}
/**
* Build the aggregation SQL query, branching on angular vs scalar paths
*/
buildAggregationQuery(fileListStr, intervalSeconds, isSourceRaw, isAngular, outputFile) {
if (isAngular) {
return this.buildAngularAggregationQuery(fileListStr, intervalSeconds, isSourceRaw, outputFile);
}
// Standard scalar aggregation
if (isSourceRaw) {
return `
COPY (
SELECT
time_bucket(INTERVAL '${intervalSeconds} seconds', received_timestamp::TIMESTAMP) as bucket_time,
context,
path,
AVG(CASE WHEN value IS NOT NULL AND TRY_CAST(value AS DOUBLE) IS NOT NULL THEN CAST(value AS DOUBLE) END) as value_avg,
MIN(CASE WHEN value IS NOT NULL AND TRY_CAST(value AS DOUBLE) IS NOT NULL THEN CAST(value AS DOUBLE) END) as value_min,
MAX(CASE WHEN value IS NOT NULL AND TRY_CAST(value AS DOUBLE) IS NOT NULL THEN CAST(value AS DOUBLE) END) as value_max,
COUNT(*) as sample_count,
MIN(received_timestamp) as first_timestamp,
MAX(received_timestamp) as last_timestamp
FROM read_parquet([${fileListStr}], union_by_name=true)
GROUP BY bucket_time, context, path
ORDER BY bucket_time
) TO '${outputFile}' (FORMAT PARQUET, COMPRESSION 'SNAPPY');
`;
}
return `
COPY (
SELECT
time_bucket(INTERVAL '${intervalSeconds} seconds', src_bucket_time::TIMESTAMP) as bucket_time,
context,
path,
SUM(value_avg * sample_count) / SUM(sample_count) as value_avg,
MIN(value_min) as value_min,
MAX(value_max) as value_max,
SUM(sample_count)::BIGINT as sample_count,
MIN(first_timestamp) as first_timestamp,
MAX(last_timestamp) as last_timestamp
FROM (
SELECT bucket_time as src_bucket_time, context, path, value_avg, value_min, value_max, sample_count, first_timestamp, last_timestamp
FROM read_parquet([${fileListStr}], union_by_name=true)
) src
GROUP BY time_bucket(INTERVAL '${intervalSeconds} seconds', src_bucket_time::TIMESTAMP), context, path
ORDER BY 1
) TO '${outputFile}' (FORMAT PARQUET, COMPRESSION 'SNAPPY');
`;
}
/**
* Build angular-specific aggregation query using vector decomposition:
* ATAN2(AVG(SIN(value)), AVG(COS(value)))
*/
buildAngularAggregationQuery(fileListStr, intervalSeconds, isSourceRaw, outputFile) {
if (isSourceRaw) {
// Vector average from raw radian values
return `
COPY (
SELECT
time_bucket(INTERVAL '${intervalSeconds} seconds', received_timestamp::TIMESTAMP) as bucket_time,
context,
path,
ATAN2(
AVG(SIN(CAST(value AS DOUBLE))),
AVG(COS(CAST(value AS DOUBLE)))
) as value_avg,
NULL::DOUBLE as value_min,
NULL::DOUBLE as value_max,
COUNT(*) as sample_count,
AVG(SIN(CAST(value AS DOUBLE))) as value_sin_avg,
AVG(COS(CAST(value AS DOUBLE))) as value_cos_avg,
MIN(received_timestamp) as first_timestamp,
MAX(received_timestamp) as last_timestamp
FROM read_parquet([${fileListStr}], union_by_name=true)
WHERE value IS NOT NULL AND TRY_CAST(value AS DOUBLE) IS NOT NULL
GROUP BY bucket_time, context, path
ORDER BY bucket_time
) TO '${outputFile}' (FORMAT PARQUET, COMPRESSION 'SNAPPY');
`;
}
// Re-aggregate from already-aggregated data using stored sin/cos averages
// weighted by sample count for correct vector re-aggregation
return `
COPY (
SELECT
time_bucket(INTERVAL '${intervalSeconds} seconds', src_bucket_time::TIMESTAMP) as bucket_time,
context,
path,
ATAN2(
SUM(value_sin_avg * sample_count) / SUM(sample_count),
SUM(value_cos_avg * sample_count) / SUM(sample_count)
) as value_avg,
NULL::DOUBLE as value_min,
NULL::DOUBLE as value_max,
SUM(sample_count)::BIGINT as sample_count,
SUM(value_sin_avg * sample_count) / SUM(sample_count) as value_sin_avg,
SUM(value_cos_avg * sample_count) / SUM(sample_count) as value_cos_avg,
MIN(first_timestamp) as first_timestamp,
MAX(last_timestamp) as last_timestamp
FROM (
SELECT bucket_time as src_bucket_time, context, path, value_sin_avg, value_cos_avg, sample_count, first_timestamp, last_timestamp
FROM read_parquet([${fileListStr}], union_by_name=true)
) src
GROUP BY time_bucket(INTERVAL '${intervalSeconds} seconds', src_bucket_time::TIMESTAMP), context, path
ORDER BY 1
) TO '${outputFile}' (FORMAT PARQUET, COMPRESSION 'SNAPPY');
`;
}
/**
* Build position aggregation query.
*
* Selects one representative point per bucket by ranking candidates:
* 1. Fewest glitchy neighbors — a neighbor is glitchy when the implied
* speed between it and the candidate exceeds POSITION_MAX_SPEED_MPS.
* Both neighbors clean beats one; one beats none.
* 2. Timestamp at or after the bucket midpoint.
* 3. Closest to the bucket midpoint.
*
* A bucket always produces a row if any candidate exists; buckets made
* entirely of glitches still emit their least-bad candidate.
*/
buildPositionAggregationQuery(fileListStr, intervalSeconds, isSourceRaw, outputFile) {
// Haversine distance in meters between two lat/lon pairs.
const haversine = (lat1, lon1, lat2, lon2) => `
(2 * 6371000 * ASIN(SQRT(
POWER(SIN(RADIANS(${lat2} - ${lat1}) / 2), 2) +
COS(RADIANS(${lat1})) * COS(RADIANS(${lat2})) *
POWER(SIN(RADIANS(${lon2} - ${lon1}) / 2), 2)
)))
`;
// Source columns differ between raw and aggregated tiers.
// Raw: received_timestamp, value_latitude, value_longitude
// Aggregated: bucket_time (as timestamp), value_latitude, value_longitude, sample_count, first_timestamp, last_timestamp
const tsCol = isSourceRaw ? 'received_timestamp' : 'bucket_time';
const srcSampleCount = isSourceRaw ? '1::BIGINT' : 'sample_count';
const srcFirstTs = isSourceRaw ? 'received_timestamp' : 'first_timestamp';
const srcLastTs = isSourceRaw ? 'received_timestamp' : 'last_timestamp';
return `
COPY (
WITH src AS (
SELECT
${tsCol}::TIMESTAMP AS ts,
context,
path,
TRY_CAST(value_latitude AS DOUBLE) AS lat,
TRY_CAST(value_longitude AS DOUBLE) AS lon,
${srcSampleCount} AS src_sample_count,
${srcFirstTs}::TIMESTAMP AS src_first_ts,
${srcLastTs}::TIMESTAMP AS src_last_ts
FROM read_parquet([${fileListStr}], union_by_name=true)
WHERE TRY_CAST(value_latitude AS DOUBLE) BETWEEN -90 AND 90
AND TRY_CAST(value_longitude AS DOUBLE) BETWEEN -180 AND 180
),
bucketed AS (
SELECT
time_bucket(INTERVAL '${intervalSeconds} seconds', ts) AS bucket_time,
*
FROM src
),
with_neighbors AS (
SELECT
*,
LAG(lat) OVER w AS prev_lat,
LAG(lon) OVER w AS prev_lon,
LAG(ts) OVER w AS prev_ts,
LEAD(lat) OVER w AS next_lat,
LEAD(lon) OVER w AS next_lon,
LEAD(ts) OVER w AS next_ts
FROM bucketed
WINDOW w AS (PARTITION BY context, path, bucket_time ORDER BY ts)
),
scored AS (
SELECT
*,
(bucket_time + INTERVAL '${intervalSeconds * 500} milliseconds') AS bucket_mid,
CASE
WHEN prev_ts IS NULL THEN 1
WHEN ${haversine('prev_lat', 'prev_lon', 'lat', 'lon')} /
GREATEST(EXTRACT(EPOCH FROM (ts - prev_ts)), 0.001) <= ${constants_1.POSITION_MAX_SPEED_MPS}
THEN 1
ELSE 0
END AS prev_ok,
CASE
WHEN next_ts IS NULL THEN 1
WHEN ${haversine('lat', 'lon', 'next_lat', 'next_lon')} /
GREATEST(EXTRACT(EPOCH FROM (next_ts - ts)), 0.001) <= ${constants_1.POSITION_MAX_SPEED_MPS}
THEN 1
ELSE 0
END AS next_ok
FROM with_neighbors
),
ranked AS (
SELECT
*,
ROW_NUMBER() OVER (
PARTITION BY context, path, bucket_time
ORDER BY
(prev_ok + next_ok) DESC,
CASE WHEN ts >= bucket_mid THEN 0 ELSE 1 END,
ABS(EXTRACT(EPOCH FROM (ts - bucket_mid)))
) AS rn
FROM scored
),
bucket_stats AS (
SELECT
bucket_time,
context,
path,
SUM(src_sample_count)::BIGINT AS sample_count,
MIN(src_first_ts) AS first_timestamp,
MAX(src_last_ts) AS last_timestamp
FROM bucketed
GROUP BY bucket_time, context, path
)
SELECT
r.bucket_time,
r.context,
r.path,
r.lat AS value_latitude,
r.lon AS value_longitude,
NULL::DOUBLE AS value_min,
NULL::DOUBLE AS value_max,
s.sample_count,
s.first_timestamp,
s.last_timestamp
FROM ranked r
JOIN bucket_stats s
ON r.bucket_time = s.bucket_time
AND r.context = s.context
AND r.path = s.path
WHERE r.rn = 1
ORDER BY r.bucket_time
) TO '${outputFile}' (FORMAT PARQUET, COMPRESSION 'SNAPPY');
`;
}
/**
* Group files by context and path
*/
groupFilesByContextPath(files) {
const groups = new Map();
for (const file of files) {
const parsed = this.hivePathBuilder.detectPathStyle(file);
if (parsed.isHive && parsed.context && parsed.signalkPath) {
const key = `${parsed.context}:${parsed.signalkPath}`;
if (!groups.has(key)) {
groups.set(key, []);
}
groups.get(key).push(file);
}
}
return groups;
}
/**
* Parse a group key back to context and path
*/
parseGroupKey(key) {
const lastColon = key.lastIndexOf(':');
if (lastColon > 0) {
return {
context: key.substring(0, lastColon),
signalkPath: key.substring(lastColon + 1),
};
}
return { context: 'vessels.self', signalkPath: key };
}
/**
* Clean up old data based on retention settings.
*
* Per-tier retention defaults come from `config.retentionDays`. A
* value of 0 at any tier means "keep forever" — that tier is skipped
* entirely. Per-path overrides take precedence: when a file's path
* matches a retention rule, the rule's `days` (scaled by the tier
* multiplier) is used instead of the global tier default.
*
* Files outside the Hive layout (legacy flat storage) are not
* touched: there is no path/year/day to anchor a retention decision
* on.
*/
async cleanupOldData() {
let deletedFiles = 0;
let failedFiles = 0;
let freedBytes = 0;
const now = new Date();
const hasPathRules = !this.retentionRules.isEmpty();
let processedSinceYield = 0;
for (const tier of TIER_HIERARCHY) {
if (this.cancelRequested)
break;
const tierDefaultDays = this.config.retentionDays[tier];
// No tier default and no per-path overrides → nothing to do for
// this tier. With overrides present we still walk the tier so
// path-specific rules can act even when the global is infinite.
if (tierDefaultDays <= 0 && !hasPathRules)
continue;
const tierMultiplier = exports.TIER_RETENTION_MULTIPLIER[tier];
const pattern = path.join(this.config.outputDirectory, `tier=${tier}`, '**', '*.parquet');
const files = await (0, glob_1.glob)(pattern);
for (const file of files) {
if (this.cancelRequested)
break;
if (++processedSinceYield >= constants_1.CLEANUP_YIELD_INTERVAL) {
processedSinceYield = 0;
await new Promise(resolve => setImmediate(resolve));
}
try {
const parsed = this.hivePathBuilder.detectPathStyle(file);
if (!parsed.isHive || !parsed.year || !parsed.dayOfYear)
continue;
const effectiveDays = this.resolveEffectiveRetentionDays(parsed.signalkPath, tierDefaultDays, tierMultiplier);
// null means keep forever for this (path, tier) pair.
if (effectiveDays === null)
continue;
// Compare at day granularity (midnight UTC). The partition's
// fileDate is midnight UTC; if cutoffDate kept the current
// time-of-day, a `days: 1` cleanup running mid-afternoon
// would already see "yesterday" as older than cutoff and
// delete it ~16 hours before its day was actually complete.
const cutoffDate = new Date(now);
cutoffDate.setUTCHours(0, 0, 0, 0);
cutoffDate.setUTCDate(cutoffDate.getUTCDate() - effectiveDays);
const fileDate = this.hivePathBuilder.dateFromDayOfYear(parsed.year, parsed.dayOfYear);
if (fileDate < cutoffDate) {
const stats = await fs.stat(file);
await fs.remove(file);
freedBytes += stats.size;
deletedFiles++;
}
}
catch (error) {
this.app.error(`Failed to check/delete ${file}: ${error.message}`);
failedFiles++;
}
}
}
this.app.debug(`Cleanup: deleted ${deletedFiles} files, ${failedFiles} failed, freed ${(freedBytes / 1024 / 1024).toFixed(2)} MB`);
return { deletedFiles, failedFiles, freedBytes };
}
/**
* Pick the retention to apply for one (path, tier) pair.
*
* - If a path-rule matches and `skipAggregation` is set: use rule.days
* for every tier (no multiplier). The path lives only in raw, so
* any stale rows in upper tiers should sweep at the same horizon
* as raw rather than linger 2/4/12x longer.
* - Else if a path-rule matches: use rule.days × tierMultiplier
* (rule.days is interpreted as raw-tier retention; upper tiers
* scale by the same convention as the global default).
* - Else: use the global tier default.
* - 0 (from either source) means infinite — return null so the caller
* skips deletion.
*/
resolveEffectiveRetentionDays(signalkPath, tierDefaultDays, tierMultiplier) {
if (signalkPath) {
const matched = this.retentionRules.match(signalkPath);
if (matched) {
if (matched.days <= 0)
return null;
return matched.skipAggregation
? matched.days
: matched.days * tierMultiplier;
}
}
return tierDefaultDays > 0 ? tierDefaultDays : null;
}
/**
* Get current job progress
*/
getProgress() {
return this.currentJob;
}
/**
* Cancel current job
*/
cancel() {
this.cancelRequested = true;
}
/**
* Schedule daily aggregation (call from consolidation timer)
*/
async runDailyAggregation() {
const yesterday = new Date();
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
this.app.debug(`Running daily aggregation for ${yesterday.toISOString().slice(0, 10)}`);
const results = await this.aggregateDate(yesterday);
return results;
}
/**
* Discover all unique dates in tier=raw across all contexts and paths
*/
async discoverRawDates(startDate, endDate) {
const rawDir = path.join(this.config.outputDirectory, 'tier=raw');
if (!(await fs.pathExists(rawDir)))
return [];
const dayDirs = await (0, glob_1.glob)(path.join(rawDir, 'context=*', 'path=*', 'year=*', 'day=*'));
const dateSet = new Set();
for (const dir of dayDirs) {
const yearMatch = dir.match(/year=(\d{4})/);
const dayMatch = dir.match(/day=(\d{1,3})/);
if (yearMatch && dayMatch) {
const year = parseInt(yearMatch[1]);
const day = parseInt(dayMatch[1]);
const date = this.hivePathBuilder.dateFromDayOfYear(year, day);
const dateStr = date.toISOString().slice(0, 10);
if (startDate && date < startDate)
continue;
if (endDate && date > endDate)
continue;
dateSet.add(dateStr);
}
}
return Array.from(dateSet)
.sort()
.map(d => new Date(d + 'T00:00:00Z'));
}
/**
* Start bulk aggregation as a background job
*/
startBulkAggregation(startDate, endDate) {
const jobId = `bulk_agg_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`;
const progress = {
jobId,
status: 'scanning',
phase: 'scan',
datesProcessed: 0,
datesTotal: 0,
percent: 0,
filesCreated: 0,
recordsAggregated: 0,
startTime: new Date(),
errors: [],
};
bulkAggregationJobs.set(jobId, progress);
this.cancelRequested = false;
this.runBulkAggregation(jobId, startDate, endDate).catch(error => {
const job = bulkAggregationJobs.get(jobId);
if (job) {
job.status = 'error';
job.error = error.message;
job.completedAt = new Date();
}
});
return jobId;
}
/**
* Run bulk aggregation across all dates in tier=raw
*/
async runBulkAggregation(jobId, startDate, endDate) {
const progress = bulkAggregationJobs.get(jobId);
if (!progress)
return;
try {
// Phase 1: Discover dates
progress.phase = 'scan';
progress.status = 'scanning';
const dates = await this.discoverRawDates(startDate, endDate);
progress.datesTotal = dates.length;
if (dates.length === 0) {
progress.status = 'completed';
progress.completedAt = new Date();
scheduleBulkJobCleanup(jobId);
return;
}
this.app.debug(`[BulkAggregation] Found ${dates.length} dates to aggregate (${dates[0].toISOString().slice(0, 10)} to ${dates[dates.length - 1].toISOString().slice(0, 10)})`);
// Phase 2: Aggregate each date
progress.phase = 'aggregation';
progress.status = 'running';
for (let i = 0; i < dates.length; i++) {
if (this.cancelRequested) {
progress.status = 'cancelled';
progress.completedAt = new Date();
scheduleBulkJobCleanup(jobId);
return;
}
const date = dates[i];
const dateStr = date.toISOString().slice(0, 10);
progress.currentDate = dateStr;
progress.datesProcessed = i;
progress.percent = Math.round((i / dates.length) * 100);
try {
const results = await this.aggregateDate(date);
for (const r of results) {
progress.filesCreated += r.filesCreated;
progress.recordsAggregated += r.recordsAggregated;
if (r.errors.length > 0) {
progress.errors.push(...r.errors.map(e => `[${dateStr}] ${e}`));
}
}
}
catch (error) {
const errorMsg = `[${dateStr}] ${error.message}`;
this.app.error(`[BulkAggregation] ${errorMsg}`);
progress.errors.push(errorMsg);
}
}
progress.datesProcessed = dates.length;
progress.percent = 100;
progress.status = 'completed';
progress.completedAt = new Date();
scheduleBulkJobCleanup(jobId);
this.app.debug(`[BulkAggregation] Complete: ${dates.length} dates, ${progress.filesCreated} files created, ${progress.recordsAggregated} records aggregated`);
}
catch (error) {
progress.status = 'error';
progress.error = error.message;
progress.completedAt = new Date();
scheduleBulkJobCleanup(jobId);
}
}
/**
* Get bulk aggregation job progress
*/
getBulkProgress(jobId) {
return bulkAggregationJobs.get(jobId) || null;
}
/**
* Cancel a bulk aggregation job
*/
cancelBulk(jobId) {
const job = bulkAggregationJobs.get(jobId);
if (job && (job.status === 'running' || job.status === 'scanning')) {
this.cancelRequested = true;
return true;
}
return false;
}
}
exports.AggregationService = AggregationService;
//# sourceMappingURL=aggregation-service.js.map