signalk-parquet/dist/utils/sqlite-buffer.d.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

/**
 * SQLite Write-Ahead Buffer for crash-safe data ingestion
 *
 * Per-path table architecture: each SignalK path gets its own table in buffer.db.
 * Scalar paths have a `value` column; object paths have `value_json` + flattened `value_*` columns.
 * This eliminates column pollution from ALTER TABLE ADD COLUMN on a shared table.
 */
import { DataRecord } from '../types';
export interface BufferRecord {
    id: number;
    context: string;
    received_timestamp: string;
    signalk_timestamp: string;
    value: number | string | boolean | null;
    value_json: string | null;
    source: string | null;
    source_label: string | null;
    source_type: string | null;
    source_pgn: number | null;
    source_src: string | null;
    meta: string | null;
    exported: number;
    export_batch_id: string | null;
    created_at: string;
    [key: string]: unknown;
}
export interface BufferStats {
    totalRecords: number;
    pendingRecords: number;
    exportedRecords: number;
    oldestPendingTimestamp: string | null;
    newestRecordTimestamp: string | null;
    dbSizeBytes: number;
    walSizeBytes: number;
}
export interface SQLiteBufferConfig {
    dbPath: string;
    maxBatchSize?: number;
    retentionHours?: number;
}
/**
 * Convert a SignalK path to a SQLite table name.
 * Dots become underscores, any non-alphanumeric/underscore chars are stripped,
 * prefixed with `buffer_`.
 */
export declare function pathToTableName(signalkPath: string): string;
export declare class SQLiteBuffer {
    private db;
    private _open;
    private readonly dbPath;
    private readonly retentionHours;
    private tableMap;
    constructor(config: SQLiteBufferConfig);
    private createMetadataSchema;
    /**
     * Migrate from the legacy single buffer_records table to per-path tables.
     * Runs once automatically if buffer_records exists but buffer_tables is empty.
     */
    private migrateFromLegacy;
    /**
     * Load existing per-path tables from buffer_tables metadata and prepare INSERT statements.
     */
    private loadExistingTables;
    /**
     * Build an INSERT statement for a per-path table.
     */
    private buildInsertStmt;
    /**
     * Ensure a per-path table exists. Creates it on first insert for a new path.
     */
    private ensureTable;
    /**
     * Ensure a dynamic value_* column exists on a per-path table.
     * Only affects the single table for that path.
     */
    private ensureColumn;
    /**
     * Check if the database connection is open
     */
    isOpen(): boolean;
    /**
     * Insert a single record into the buffer
     */
    insert(record: DataRecord): void;
    /**
     * Insert multiple records in a single transaction (much faster)
     */
    insertBatch(records: DataRecord[]): void;
    private prepareRecord;
    /**
     * Convert a BufferRecord back to a DataRecord.
     * Path must be passed in since per-path tables have no path column.
     */
    private bufferRecordToDataRecord;
    /**
     * Clean up old exported records from all per-path tables
     */
    cleanup(): number;
    /**
     * Get buffer statistics aggregated across all per-path tables
     */
    getStats(): BufferStats;
    /**
     * Get count of pending records (faster than full stats)
     */
    getPendingCount(): number;
    /**
     * Checkpoint WAL file (useful before backup or to reduce WAL size)
     */
    checkpoint(): void;
    /**
     * Get distinct dates that have unexported records, excluding today.
     * Scans all per-path tables.
     */
    getDatesWithUnexportedRecords(excludeToday?: boolean): string[];
    /**
     * Get distinct context/path combinations for a specific date (UTC).
     * Scans all per-path tables.
     */
    getPathsForDate(date: Date): Array<{
        context: string;
        path: string;
    }>;
    /**
     * Get all records for a specific context, path, and date (UTC).
     * Returns just DataRecord[] — no IDs needed since markDateExported() handles marking.
     */
    getRecordsForPathAndDate(context: string, signalkPath: string, date: Date): DataRecord[];
    /**
     * Count unexported records for a specific context, path, and date (UTC).
     */
    getRecordCountForPathAndDate(context: string, signalkPath: string, date: Date): number;
    /**
     * Get a batch of unexported records for a specific context, path, and date (UTC).
     * Uses LIMIT/OFFSET to avoid materializing all rows at once.
     */
    getRecordsForPathAndDateBatched(context: string, signalkPath: string, date: Date, limit: number, offset: number): DataRecord[];
    /**
     * Mark records for a specific date as exported.
     * Queries the specific path's table.
     */
    markDateExported(context: string, signalkPath: string, date: Date, batchId: string): void;
    /**
     * Get the set of known SignalK paths that have buffer tables.
     * Used by SQL builders to check if a buffer table exists for federation.
     */
    getKnownPaths(): Set<string>;
    /**
     * Check if a buffer table exists for a given SignalK path.
     */
    hasTable(signalkPath: string): boolean;
    /**
     * Get the set of columns for a given path's buffer table.
     * Returns undefined if no table exists for this path.
     */
    getTableColumns(signalkPath: string): Set<string> | undefined;
    /**
     * Get the column schema (name + declared SQLite type) for a path's buffer table.
     * Returns undefined if no table exists for this path.
     */
    getTableSchema(signalkPath: string): Array<{
        name: string;
        type: string;
    }> | undefined;
    /**
     * Read a batch of unexported rows for federated history queries, keyset-paginated
     * by id so callers can stream large windows without materializing them all.
     * Rows are raw table rows (all columns), matching the table schema.
     */
    getRowsForFederation(signalkPath: string, context: string, fromIso: string, toIso: string, afterId: number, limit: number): Array<Record<string, unknown>>;
    /**
     * Get the database path
     */
    getDbPath(): string;
    /**
     * Close the database connection
     */
    close(): void;
}
//# sourceMappingURL=sqlite-buffer.d.ts.map