signalk-parquet/dist/parquet-writer.d.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

import { DataRecord, ParquetWriterOptions } from './types';
import { ServerAPI } from '@signalk/server-api';
import { SchemaService } from './schema-service';
export declare class ParquetWriter {
    private format;
    private app?;
    private schemaService?;
    private directoryScanner;
    constructor(options?: ParquetWriterOptions);
    getSchemaService(): SchemaService | undefined;
    writeRecords(filepath: string, records: DataRecord[]): Promise<string>;
    /**
     * Find the base data directory from a nested path
     * This helps invalidate the right cache entry
     */
    private findBaseDataDir;
    writeJSON(filepath: string, records: DataRecord[]): Promise<string>;
    writeCSV(filepath: string, records: DataRecord[]): Promise<string>;
    writeParquet(filepath: string, records: DataRecord[]): Promise<string>;
    /**
     * Write Parquet file in batches to avoid loading all records into memory.
     * Uses firstBatch for schema detection, then pulls subsequent batches via callback.
     */
    writeParquetBatched(filepath: string, firstBatch: DataRecord[], nextBatch: () => DataRecord[], currentPath?: string): Promise<string>;
    createParquetSchema(records: DataRecord[], currentPath?: string): Promise<any>;
    private getTypeForEmptyColumn;
    private getTypeForExplodedField;
    private getTypeFromOtherFiles;
    private inferTypeFromFieldName;
    private extractOutputDirectory;
    prepareRecordForParquet(record: DataRecord, schema: any): {
        [key: string]: any;
    };
    private validateParquetFile;
    private logQuarantine;
}
/**
 * Move undersized (`< 100B`) parquet files left behind by a crash between
 * ParquetWriter.openFile() — which creates a 0-byte stub on disk — and the
 * first appendRow()/close() that would populate it. The per-write catch
 * block can't run if the process is killed in that window, so we sweep
 * on plugin start. Files are moved to a sibling `quarantine/` dir, matching
 * the in-flight quarantine layout; the History API already excludes those
 * paths from queries.
 *
 * The sweep is incremental: a stub is a newly *created* file, which bumps its
 * parent directory's mtime, so directories unchanged since the last sweep are
 * skipped. The last-sweep time is persisted in `<baseDirectory>/.last-empty-
 * sweep`; the first run (no watermark) scans everything once to catch any
 * pre-existing stub, then later runs cost O(directories changed since the
 * previous start) and still catch a stub of any age after long downtime.
 */
export declare function quarantineEmptyParquetFiles(app: ServerAPI, baseDirectory: string): Promise<{
    quarantined: number;
    failed: number;
}>;
//# sourceMappingURL=parquet-writer.d.ts.map