signalk-parquet/dist/services/aggregation-service.d.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

/**
 * Aggregation Service
 *
 * Handles multi-tier data aggregation from raw data to progressively
 * downsampled tiers for efficient long-term storage and querying.
 *
 * Tiers:
 * - raw: Original data (~1s resolution)
 * - 5s: 5-second aggregates
 * - 60s: 1-minute aggregates
 * - 1h: Hourly aggregates
 */
import { ServerAPI } from '@signalk/server-api';
import { AggregationTier } from '../utils/hive-path-builder';
import { PathRetentionRule } from '../utils/retention-rules';
export interface AggregationConfig {
    outputDirectory: string;
    filenamePrefix: string;
    retentionDays: {
        raw: number;
        '5s': number;
        '60s': number;
        '1h': number;
    };
    pathRetentionOverrides?: PathRetentionRule[];
}
export declare const TIER_RETENTION_MULTIPLIER: Record<AggregationTier, number>;
/**
 * Build the per-tier retention block from a raw-tier value. 0 stays
 * 0 in every tier (= keep forever).
 */
export declare function buildPerTierRetention(rawDays: number): AggregationConfig['retentionDays'];
export interface AggregationProgress {
    jobId: string;
    status: 'running' | 'completed' | 'cancelled' | 'error';
    tier: AggregationTier;
    processed: number;
    total: number;
    currentFile?: string;
    startTime: Date;
    completedAt?: Date;
    error?: string;
}
export interface BulkAggregationProgress {
    jobId: string;
    status: 'scanning' | 'running' | 'completed' | 'cancelled' | 'error';
    phase: 'scan' | 'aggregation';
    currentDate?: string;
    datesProcessed: number;
    datesTotal: number;
    percent: number;
    filesCreated: number;
    recordsAggregated: number;
    startTime: Date;
    completedAt?: Date;
    error?: string;
    errors: string[];
}
export interface AggregationResult {
    sourceTier: AggregationTier;
    targetTier: AggregationTier;
    filesProcessed: number;
    recordsAggregated: number;
    filesCreated: number;
    duration: number;
    errors: string[];
}
export declare class AggregationService {
    private readonly config;
    private readonly app;
    private readonly hivePathBuilder;
    private readonly retentionRules;
    private currentJob;
    private cancelRequested;
    constructor(config: AggregationConfig, app: ServerAPI);
    /**
     * Run aggregation for a specific date.
     * Optional pathFilter restricts which signalk paths are aggregated (used by
     * targeted migrations like position re-aggregation).
     */
    aggregateDate(date: Date, pathFilter?: (signalkPath: string) => boolean): Promise<AggregationResult[]>;
    /**
     * Aggregate from one tier to the next
     */
    aggregateTier(sourceTier: AggregationTier, targetTier: AggregationTier, date: Date, pathFilter?: (signalkPath: string) => boolean): Promise<AggregationResult>;
    /**
     * Aggregate a group of files for a specific context/path
     */
    private aggregateGroup;
    /**
     * Build the aggregation SQL query, branching on angular vs scalar paths
     */
    private buildAggregationQuery;
    /**
     * Build angular-specific aggregation query using vector decomposition:
     * ATAN2(AVG(SIN(value)), AVG(COS(value)))
     */
    private buildAngularAggregationQuery;
    /**
     * Build position aggregation query.
     *
     * Selects one representative point per bucket by ranking candidates:
     *   1. Fewest glitchy neighbors — a neighbor is glitchy when the implied
     *      speed between it and the candidate exceeds POSITION_MAX_SPEED_MPS.
     *      Both neighbors clean beats one; one beats none.
     *   2. Timestamp at or after the bucket midpoint.
     *   3. Closest to the bucket midpoint.
     *
     * A bucket always produces a row if any candidate exists; buckets made
     * entirely of glitches still emit their least-bad candidate.
     */
    private buildPositionAggregationQuery;
    /**
     * Group files by context and path
     */
    private groupFilesByContextPath;
    /**
     * Parse a group key back to context and path
     */
    private parseGroupKey;
    /**
     * Clean up old data based on retention settings.
     *
     * Per-tier retention defaults come from `config.retentionDays`. A
     * value of 0 at any tier means "keep forever" — that tier is skipped
     * entirely. Per-path overrides take precedence: when a file's path
     * matches a retention rule, the rule's `days` (scaled by the tier
     * multiplier) is used instead of the global tier default.
     *
     * Files outside the Hive layout (legacy flat storage) are not
     * touched: there is no path/year/day to anchor a retention decision
     * on.
     */
    cleanupOldData(): Promise<{
        deletedFiles: number;
        failedFiles: number;
        freedBytes: number;
    }>;
    /**
     * Pick the retention to apply for one (path, tier) pair.
     *
     * - If a path-rule matches and `skipAggregation` is set: use rule.days
     *   for every tier (no multiplier). The path lives only in raw, so
     *   any stale rows in upper tiers should sweep at the same horizon
     *   as raw rather than linger 2/4/12x longer.
     * - Else if a path-rule matches: use rule.days × tierMultiplier
     *   (rule.days is interpreted as raw-tier retention; upper tiers
     *   scale by the same convention as the global default).
     * - Else: use the global tier default.
     * - 0 (from either source) means infinite — return null so the caller
     *   skips deletion.
     */
    private resolveEffectiveRetentionDays;
    /**
     * Get current job progress
     */
    getProgress(): AggregationProgress | null;
    /**
     * Cancel current job
     */
    cancel(): void;
    /**
     * Schedule daily aggregation (call from consolidation timer)
     */
    runDailyAggregation(): Promise<AggregationResult[]>;
    /**
     * Discover all unique dates in tier=raw across all contexts and paths
     */
    discoverRawDates(startDate?: Date, endDate?: Date): Promise<Date[]>;
    /**
     * Start bulk aggregation as a background job
     */
    startBulkAggregation(startDate?: Date, endDate?: Date): string;
    /**
     * Run bulk aggregation across all dates in tier=raw
     */
    private runBulkAggregation;
    /**
     * Get bulk aggregation job progress
     */
    getBulkProgress(jobId: string): BulkAggregationProgress | null;
    /**
     * Cancel a bulk aggregation job
     */
    cancelBulk(jobId: string): boolean;
}
//# sourceMappingURL=aggregation-service.d.ts.map