signalk-parquet/dist/services/compaction-service.d.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

/**
 * Compaction Service
 *
 * Merges per-day parquet files within a (tier, context, path, year) group
 * into a single per-year parquet file, then deletes the source day-files.
 *
 * Why: the live and import write paths emit one parquet per (path, UTC day).
 * Over multiple years a typical vessel ends up with many thousands of small
 * files in tier=raw, each carrying parquet's per-file overhead (schema,
 * page index, footer). DuckDB queries spanning years pay that overhead per
 * file, and the filesystem suffers metadata bloat.
 *
 * Scope: this is purely a layout transformation. Schema is preserved
 * exactly (`SELECT *` with `union_by_name=true`), records are sorted by
 * `signalk_timestamp`, and writes go through a temp-file + atomic-rename
 * pattern so a partial run never corrupts the partition.
 *
 * After compaction the year directory contains one file like
 *
 *   tier=raw/context=.../path=.../year=2024/year_compact_2024_<TS>.parquet
 *
 * and the `day=DDD/` subdirectories under it are removed. DuckDB's
 * hive-partitioning glob still finds the file; `day` becomes NULL for
 * compacted years, which matters only if a downstream consumer filters by
 * `day=…` directly. The History API filters by timestamp range, so
 * partition pruning at the year level still works and per-file min/max
 * stats handle the rest.
 *
 * Mirrors the same job-tracking / cancellable-job pattern as
 * MigrationService, AggregationService and GpxImportService.
 */
import { ServerAPI } from '@signalk/server-api';
import { AggregationTier } from '../utils/hive-path-builder';
export interface CompactionConfig {
    baseDirectory: string;
    tier: AggregationTier;
    beforeYear: number;
    pathFilter?: string;
}
export interface CompactionPlanGroup {
    tier: string;
    context: string;
    path: string;
    year: number;
    yearDir: string;
    sourcePaths: string[];
    sourceFiles: number;
    sourceBytes: number;
}
export interface CompactionPlan {
    totalGroups: number;
    totalSourceFiles: number;
    totalSourceBytes: number;
    groups: CompactionPlanGroup[];
}
export interface CompactionProgress {
    jobId: string;
    status: 'scanning' | 'running' | 'completed' | 'cancelled' | 'error';
    phase: 'scan' | 'compact';
    processed: number;
    total: number;
    percent: number;
    currentGroup?: string;
    startTime: Date;
    completedAt?: Date;
    error?: string;
    groupsCompacted: number;
    groupsSkipped: number;
    filesRemoved: number;
    bytesBefore: number;
    bytesAfter: number;
    errors: string[];
}
export declare class CompactionConflictError extends Error {
    constructor(message: string);
}
/**
 * Mark every running/scanning compaction job for cancellation. Called
 * from plugin.stop() so a SignalK shutdown does not leave a job spinning
 * past the next group boundary. Does not await: a single in-flight
 * DuckDB COPY is uninterruptible, but the per-group loop will exit on
 * the next iteration check.
 */
export declare function signalShutdownAllCompactionJobs(): number;
/**
 * Signal cancellation to every active job and wait for them to reach a
 * terminal state, bounded by `timeoutMs`. Use this from plugin.stop()
 * so DuckDBPool.shutdown() doesn't run while a COPY is still in flight
 * — that race turns a clean stop into a failed compaction at best, or
 * a partial temp file at worst.
 *
 * Returns counts so the caller can log what happened. `remaining > 0`
 * means a job was still running at timeout; the plugin proceeds with
 * shutdown anyway, since blocking forever is worse than leaking a job.
 */
export declare function quiesceAllCompactionJobs(timeoutMs?: number): Promise<{
    signalled: number;
    quiesced: number;
    remaining: number;
}>;
/**
 * Remove stranded `*.tmp` files left behind by a SignalK crash mid-COPY.
 * Safe to call at every plugin start: only files matching the compaction
 * temp pattern under the data directory are removed.
 */
export declare function cleanupStrandedCompactionTempFiles(app: ServerAPI, baseDirectory: string): Promise<{
    removed: number;
}>;
/**
 * Recover from stale `.compaction-trash-*` directories left by a crash
 * between move-to-trash and rename-to-published. Two cases:
 *
 *   - Yearly file present in the parent: the publish completed before
 *     the crash; trash holds duplicate sources that the post-publish
 *     cleanup never got to remove. Delete the trash dir.
 *   - Yearly file absent: the crash happened mid-move (or after move
 *     but before publish). Restore the trashed files back to their
 *     mirrored locations under the year-dir, then delete the empty
 *     trash dir.
 *
 * This runs early in plugin start, before any new data writes can land
 * in day-dirs, so a restore can't clobber anything.
 */
export declare function recoverStrandedCompactionTrash(app: ServerAPI, baseDirectory: string): Promise<{
    restored: number;
    cleaned: number;
    failed: number;
}>;
export declare class CompactionService {
    private readonly app;
    private readonly hivePathBuilder;
    constructor(app: ServerAPI);
    /**
     * Walk the Hive layout for a given tier and return what would be
     * compacted. Groups with only one file are not included (already
     * compact). Non-destructive — safe to call repeatedly.
     */
    scan(config: CompactionConfig): Promise<CompactionPlan>;
    compact(config: CompactionConfig): Promise<string>;
    private run;
    /**
     * Walk the Hive layout tier=<T>/context=.../path=.../year=... and
     * return one group per year-directory that has more than one parquet
     * file under it.
     *
     * Scale note: result size is O(#contexts × #paths × #years). On a
     * typical SignalK install (one vessel, ~hundreds of paths, single-digit
     * years) this fits comfortably in memory.
     */
    private findCompactableGroups;
    /**
     * Parse the trailing four segments of a Hive year directory into the
     * tier/context/path/year tuple. Returns null on shape mismatch.
     *
     * Example input:  "/data/tier=raw/context=vessels__urn-mrn-…/path=navigation__position/year=2024"
     * Example output: { context: "vessels.urn:mrn:…", path: "navigation.position", year: 2024 }
     */
    private parseYearDir;
    /**
     * Merge all parquet files under one (tier, context, path, year) group
     * into a single output file, then remove the sources.
     *
     * Publish sequence (atomic from a reader's perspective):
     *   1. DuckDB COPY into `<output>.tmp`, size-checked.
     *   2. Source files are *moved* (rename within yearDir) into a trash
     *      directory: `<yearDir>/.compaction-trash-<jobId>/`. Mirrors the
     *      original day=DDD/ structure so rollback is just a reverse
     *      rename.
     *   3. The temp file is renamed to its final `year_compact_*.parquet`
     *      name. This is the publish point.
     *   4. The trash directory is recursively removed.
     *
     * Why moves first, then publish: between steps 3 and 4 in a
     * "publish-then-delete" scheme, queries see both the new yearly file
     * and every still-present source — duplicate rows on the wire. The
     * trash-first scheme makes the year-dir's queryable contents flip
     * atomically: before the publish rename a query sees the originals
     * (less any briefly-renamed files mid-step-2), after it the query
     * sees only the yearly file.
     *
     * Failure handling:
     *   - Step 1/2 failure (pre-publish): reverse any moves that did
     *     succeed, delete temp, delete trash dir. Sources end back where
     *     they started; no data lost. The group surfaces an error.
     *   - Step 3 failure: same rollback as above.
     *   - Step 4 failure (post-publish): data is committed; the yearly
     *     file holds everything. Failure to clean trash is logged and
     *     surfaced via `residualSources` (now meaning "files left in
     *     trash that the operator may want to remove"). Startup sweep
     *     handles these on next plugin start.
     */
    private compactGroup;
    /**
     * Rename with one retry on EBUSY. On Windows a reader (DuckDB query,
     * antivirus, file explorer preview) can briefly hold the destination
     * path; a single backoff-and-retry covers the common case without
     * making us wait indefinitely.
     */
    private renameWithRetry;
    /**
     * Remove a file or directory with exponential backoff on transient
     * errors. Windows readers (antivirus, search indexer, explorer
     * preview) can hold a brief handle; one delete attempt isn't enough.
     * After all attempts the last error is rethrown so the caller can
     * record it (e.g. as a residual). Permanent errors (ENOENT, etc.)
     * are not retried.
     */
    private removeWithRetry;
    private removeEmptyDayDirs;
    getProgress(jobId: string): CompactionProgress | null;
    cancel(jobId: string): boolean;
    getJobIds(): string[];
}
//# sourceMappingURL=compaction-service.d.ts.map