signalk-parquet/dist/services/parquet-export-service.d.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

/**
 * Parquet Export Service
 *
 * Handles periodic export of data from SQLite buffer to Parquet files.
 * Provides crash recovery by checking for pending records on startup.
 */
import { SQLiteBuffer } from '../utils/sqlite-buffer';
import { ParquetWriter } from '../types';
import { ServerAPI } from '@signalk/server-api';
export interface ExportServiceConfig {
    outputDirectory: string;
    filenamePrefix: string;
    useHivePartitioning: boolean;
    dailyExportHour: number;
    s3Upload?: {
        enabled: boolean;
    };
}
export interface ExportResult {
    batchId: string;
    recordsExported: number;
    filesCreated: string[];
    duration: number;
    errors: string[];
}
export declare class ParquetExportService {
    private readonly sqliteBuffer;
    private readonly parquetWriter;
    private readonly config;
    private readonly app;
    private readonly hivePathBuilder;
    private exportInterval;
    private isExporting;
    private lastExportTime;
    private totalExported;
    private lastBatchExported;
    private lastExportTrigger;
    constructor(sqliteBuffer: SQLiteBuffer, parquetWriter: ParquetWriter, config: ExportServiceConfig, app: ServerAPI);
    /**
     * Start the export service
     *
     * No startup export — catchup for completed days is handled by
     * exportAllUnexported() called from index.ts after a 10s delay.
     * Today's data stays in SQLite for the History API to read live.
     */
    start(): void;
    /**
     * Stop the export service
     */
    stop(): void;
    /**
     * Force an immediate export of completed days (excludes today)
     */
    forceExport(): Promise<ExportResult>;
    /**
     * Build a flat-structure file path (legacy compatibility)
     */
    private buildFlatFilePath;
    /**
     * Generate a unique batch ID
     */
    private generateBatchId;
    /**
     * Get service status
     */
    getStatus(): {
        isRunning: boolean;
        isExporting: boolean;
        lastExportTime: Date | null;
        lastBatchExported: number;
        totalExported: number;
        pendingRecords: number;
        dailyExportHour: number;
        lastExportTrigger: string | null;
        mode: 'daily';
    };
    /**
     * Get health check information
     */
    getHealth(): {
        healthy: boolean;
        lastExportTime: Date | null;
        pendingRecords: number;
        bufferStats: ReturnType<SQLiteBuffer['getStats']>;
    };
    /**
     * Export all unexported data from SQLite to Parquet (complete days only)
     * Used at startup to catch up on any missed exports.
     * Excludes today's data to avoid creating partial day files that would
     * conflict with the daily export.
     */
    exportAllUnexported(): Promise<ExportResult>;
    /**
     * Export a full day's data to Parquet files (one file per context/path)
     * This creates consolidated daily files directly, without needing a separate
     * consolidation step.
     *
     * @param targetDate The date to export (UTC). Typically yesterday.
     * @returns Export result with details about files created
     */
    exportDayToParquet(targetDate: Date): Promise<ExportResult>;
    /**
     * Export a group of records as a timestamped file
     * Uses consistent timestamped naming (same as startup exports) for simplicity
     */
    private exportDailyGroup;
    /**
     * Export records in batches to avoid loading all rows into memory at once.
     * firstBatch is used for schema detection; nextBatch callback pulls subsequent chunks.
     */
    private exportDailyGroupBatched;
}
//# sourceMappingURL=parquet-export-service.d.ts.map