signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
157 lines • 6.44 kB
TypeScript
/**
* GPX Import Service
*
* Imports historical GPS tracks from GPX files into the Hive-partitioned
* parquet store. Each <trkpt> with a valid <time> is expanded into SignalK
* delta-style records for the configured paths (position, SOG, COG,
* altitude) and written directly as parquet files, bypassing the SQLite
* buffer (bulk historical load).
*
* Follows the same progress-tracking / cancellable-job pattern as
* MigrationService.
*
* When adding a new SignalK path:
* 1. Extend the GpxImportPath union below
* 2. Append to DEFAULT_IMPORT_PATHS
* 3. Add a case in pointToValue() that maps the <trkpt> to the value
* in SignalK units (m/s, radians, etc.)
* 4. Extend the GpxPoint interface in gpx-parser.ts if a new tag must
* be parsed out of the GPX
*/
import { ServerAPI } from '@signalk/server-api';
import { ParquetWriter } from '../types';
import { AggregationService } from './aggregation-service';
export type GpxImportPath = 'navigation.position' | 'navigation.speedOverGround' | 'navigation.courseOverGroundTrue' | 'navigation.gnss.antennaAltitude';
export declare const DEFAULT_IMPORT_PATHS: GpxImportPath[];
export interface GpxImportConfig {
sourceDirectory?: string;
sourceFiles?: string[];
targetDirectory: string;
context: string;
paths: GpxImportPath[];
filenamePrefix: string;
deleteSourceAfterImport: boolean;
sourceLabel: string;
}
export interface GpxImportProgress {
jobId: string;
status: 'scanning' | 'running' | 'completed' | 'cancelled' | 'error';
phase: 'scan' | 'parse' | 'write' | 'aggregate';
processed: number;
total: number;
percent: number;
currentFile?: string;
startTime: Date;
completedAt?: Date;
error?: string;
bytesProcessed: number;
pointsParsed: number;
pointsWritten: number;
recordsWritten: number;
filesImported: number;
filesSkipped: number;
filesCreated: string[];
errors: string[];
aggregationDatesTotal?: number;
aggregationDatesProcessed?: number;
aggregationCurrentDate?: string;
}
export interface GpxScanResult {
totalFiles: number;
totalSize: number;
files: Array<{
path: string;
size: number;
}>;
}
export declare class GpxImportService {
private readonly app;
private readonly parquetWriter;
private readonly hivePathBuilder;
private readonly aggregationService?;
private readonly cancelledJobs;
constructor(app: ServerAPI, parquetWriter: ParquetWriter, aggregationService?: AggregationService);
/**
* Scan a directory for .gpx files (non-destructive dry run).
*/
scan(sourceDirectory: string): Promise<GpxScanResult>;
/**
* Start an import job. Runs asynchronously; poll progress via getProgress.
*
* Validates the requested SK paths against DEFAULT_IMPORT_PATHS even
* though the route layer already does — defense in depth so a future
* non-route caller can't slip an unsupported path through to
* pointToValue (which has no default branch).
*/
import(config: GpxImportConfig): Promise<string>;
/**
* Main orchestration: resolve file list, parse each, group records by
* (path, day), write one parquet per group into the Hive layout.
*/
private runImport;
/**
* Look up SignalK metadata once per job for each requested path. Mirrors
* what data-handler.ts does on every live delta — for imports we just
* cache it once. Failures are silent (metadata is best-effort and the
* record stays valid without it).
*/
private buildMetadataCache;
/**
* Parse a single GPX file and write its points to the parquet store.
* Returns true if at least one parquet file was produced.
*
* `touchedDays` is mutated to record each (year, day) partition this
* file wrote into; the caller drives a post-import aggregation phase
* over that set.
*/
private importFile;
/**
* Re-aggregate every (year, day) partition this import wrote into.
* `aggregateDate()` cascades raw -> 5s -> 60s -> 1h, and is idempotent
* (DuckDB COPY ... TO overwrites the per-day output file), so running
* it for already-aggregated dates is safe; it just refolds the new
* raw rows in alongside the existing ones.
*
* Errors per-date are recorded but don't fail the import; partial
* tier coverage is better than rejecting the whole job.
*/
private runAggregationPhase;
private buildHiveFilePath;
/**
* Convert a GPX point into the SignalK value for a given path.
* Returns undefined if the point lacks the needed field.
*
* Units:
* - navigation.position: object {latitude, longitude} in decimal degrees
* - navigation.speedOverGround: number in m/s (GPX <speed> is m/s)
* - navigation.courseOverGroundTrue: number in radians (GPX <course> is degrees → convert)
* - navigation.gnss.antennaAltitude: number in meters
*
* When adding a new GpxImportPath, add a matching case here in the same
* unit as the SignalK path spec defines.
*/
private pointToValue;
/**
* Build a DataRecord matching the shape produced by the live streambundle
* handler in data-handler.ts: scalar values go in `value`, object values
* go in `value_json` with their scalar properties flattened into
* `value_<key>` columns so downstream queries can read them directly.
* `meta` carries the SK metadata (units, displayUnits, etc.) so
* downstream consumers see the same units as live-captured rows.
*
* Kept inline (rather than shared with data-handler.ts) because the live
* path also populates source.$source, source.pgn etc. from the delta
* frame — fields we don't have here. A future refactor could extract
* the shared object-flattening helper; it isn't big enough to pay yet.
*
* `source.type: 'file'` is a plugin-local convention rather than one of
* SK's canonical source types (NMEA0183 / NMEA2000 / signalk). Anything
* filtering on canonical types won't match imported rows; if that
* matters in the future, we could expose it as a config option.
*/
private buildRecord;
getProgress(jobId: string): GpxImportProgress | null;
cancel(jobId: string): boolean;
getJobIds(): string[];
}
//# sourceMappingURL=gpx-import-service.d.ts.map