signalk-parquet/dist/types.d.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

import { Context, Path, ServerAPI, NormalizedDelta, SourceRef } from '@signalk/server-api';
export { NormalizedDelta, SourceRef };
import { Request, Response, Router } from 'express';
import { LRUCache } from './utils/lru-cache';
export interface SchemaService {
    detectOptimalSchema(records: DataRecord[], currentPath?: string): Promise<any>;
    validateFileSchema(filePath: string): Promise<any>;
    repairFileSchema(filePath: string, filenamePrefix?: string): Promise<any>;
}
export interface SignalKPlugin {
    id: string;
    name: string;
    description: string;
    schema: any;
    start: (options: Partial<PluginConfig>) => void;
    stop: () => void;
    registerWithRouter?: (router: Router) => void;
}
export interface AutoDiscoveryConfig {
    enabled: boolean;
    excludePatterns?: string[];
    includePatterns?: string[];
    maxAutoConfiguredPaths?: number;
    requireLiveData: boolean;
}
export interface PluginConfig {
    bufferSize: number;
    saveIntervalSeconds: number;
    outputDirectory: string;
    filenamePrefix: string;
    retentionDays: number;
    pathRetentionOverrides?: import('./utils/retention-rules').PathRetentionRule[];
    configSchemaVersion?: number;
    fileFormat: 'json' | 'csv' | 'parquet';
    vesselMMSI: string;
    cloudUpload: CloudUploadConfig;
    enableStreaming?: boolean;
    claudeIntegration?: ClaudeIntegrationConfig;
    homePortLatitude?: number;
    homePortLongitude?: number;
    setCurrentLocationAction?: {
        setCurrentLocation: boolean;
    };
    useSqliteBuffer?: boolean;
    exportBatchSize?: number;
    bufferRetentionHours?: number;
    useHivePartitioning?: boolean;
    dailyExportHour?: number;
    autoDiscovery?: AutoDiscoveryConfig;
    enableRawSql?: boolean;
}
import type { ClaudeModel } from './claude-models';
export interface ClaudeIntegrationConfig {
    enabled: boolean;
    apiKey?: string;
    model?: ClaudeModel;
    maxTokens?: number;
    temperature?: number;
    autoAnalysis?: {
        daily: boolean;
        anomaly: boolean;
        threshold: number;
    };
    cacheEnabled?: boolean;
    templates?: string[];
}
export interface VesselContext {
    vesselInfo: VesselInfo;
    customContext: string;
    lastUpdated: string;
    autoExtracted: boolean;
}
export interface VesselInfo {
    name?: string;
    callsign?: string;
    mmsi?: string;
    length?: number;
    beam?: number;
    draft?: number;
    height?: number;
    displacement?: number;
    vesselType?: string;
    classification?: string;
    flag?: string;
    grossTonnage?: number;
    netTonnage?: number;
    deadWeight?: number;
    builder?: string;
    buildYear?: number;
    hullNumber?: string;
    ownerName?: string;
    port?: string;
    notes?: string;
}
export interface VesselContextExtraction {
    path: string;
    signalkPath: string;
    displayName: string;
    unit?: string;
    category: 'identification' | 'physical' | 'classification' | 'technical' | 'build' | 'contact';
}
export interface PathConfig {
    path: Path;
    name?: string;
    enabled?: boolean;
    regimen?: string;
    source?: string;
    context?: Context;
    excludeMMSI?: string[];
    autoDiscovered?: boolean;
}
/**
 * Threshold operator types based on data type
 */
export type ThresholdOperator = 'gt' | 'lt' | 'eq' | 'ne' | 'range' | 'contains' | 'startsWith' | 'endsWith' | 'stringEquals' | 'true' | 'false' | 'withinRadius' | 'outsideRadius' | 'inBoundingBox' | 'outsideBoundingBox';
/**
 * Bounding box for geographic area thresholds
 */
export interface BoundingBox {
    north: number;
    south: number;
    east: number;
    west: number;
}
/**
 * Type-aware threshold configuration
 */
export interface ThresholdConfig {
    enabled: boolean;
    watchPath: string;
    operator: ThresholdOperator;
    value?: number | boolean | string;
    valueMin?: number;
    valueMax?: number;
    latitude?: number;
    longitude?: number;
    radius?: number;
    boundingBox?: BoundingBox;
    useHomePort?: boolean;
    boxSize?: number;
    boxAnchor?: string;
    boxBuffer?: number;
    activateOnMatch: boolean;
    hysteresis?: number;
}
export interface CommandConfig {
    command: string;
    path: string;
    registered: string;
    description?: string;
    keywords?: string[];
    active?: boolean;
    lastExecuted?: string;
    defaultState?: boolean;
    thresholds?: ThresholdConfig[];
    manualOverride?: boolean;
    manualOverrideUntil?: string;
}
export interface CommandRegistrationState {
    registeredCommands: Map<string, CommandConfig>;
    putHandlers: Map<string, CommandPutHandler>;
}
export interface CommandExecutionRequest {
    command: string;
    value: boolean;
    timestamp?: string;
}
export interface CommandRegistrationRequest {
    command: string;
    description?: string;
    keywords?: string[];
    defaultState?: boolean;
    thresholds?: ThresholdConfig[];
}
export interface WebAppPathConfig {
    paths: PathConfig[];
    commands: CommandConfig[];
}
export interface CloudUploadConfig {
    provider: 'none' | 's3' | 'r2';
    bucket?: string;
    region?: string;
    endpoint?: string;
    forcePathStyle?: boolean;
    accountId?: string;
    keyPrefix?: string;
    accessKeyId?: string;
    secretAccessKey?: string;
    deleteAfterUpload?: boolean;
}
export interface SignalKSubscription {
    context: string;
    subscribe: Array<{
        path: string;
        period: number;
    }>;
}
export interface DataRecord {
    received_timestamp: string;
    signalk_timestamp: string;
    context: string;
    path: string;
    value: any;
    value_json?: string | object;
    source?: string | object;
    source_label?: string;
    source_type?: string;
    source_pgn?: number;
    source_src?: string;
    meta?: string | object;
    [key: string]: any;
}
export interface ParquetWriterOptions {
    format: 'json' | 'csv' | 'parquet';
    app?: ServerAPI;
}
export interface FileInfo {
    name: string;
    path: string;
    size: number;
    modified: string;
}
export interface PathInfo {
    path: string;
    directory: string;
    fileCount: number;
}
export interface ApiResponse<T = any> {
    success: boolean;
    data?: T;
    error?: string;
    message?: string;
}
export interface CommandApiResponse extends ApiResponse {
    commands?: CommandConfig[];
    command?: CommandConfig;
    count?: number;
}
export interface CommandExecutionResponse extends ApiResponse {
    command?: string;
    value?: boolean;
    executed?: boolean;
    timestamp?: string;
}
export interface PathsApiResponse extends ApiResponse {
    dataDirectory?: string;
    paths?: PathInfo[];
}
export interface FilesApiResponse extends ApiResponse {
    path?: string;
    directory?: string;
    files?: FileInfo[];
}
export interface QueryApiResponse extends ApiResponse {
    query?: string;
    rowCount?: number;
    data?: any[];
}
export interface SampleApiResponse extends ApiResponse {
    path?: string;
    file?: string;
    columns?: string[];
    rowCount?: number;
    data?: any[];
}
export interface ConfigApiResponse extends ApiResponse {
    paths?: PathConfig[];
}
export interface HealthApiResponse extends ApiResponse {
    status?: string;
    timestamp?: string;
    duckdb?: string;
}
export interface CloudTestApiResponse extends ApiResponse {
    provider?: string;
    bucket?: string;
    region?: string;
    accountId?: string;
    keyPrefix?: string;
}
export type S3TestApiResponse = CloudTestApiResponse;
export interface ValidationViolation {
    file: string;
    vessel?: string;
    issues: string[];
}
export interface ValidationApiResponse extends ApiResponse {
    totalFiles?: number;
    totalVessels?: number;
    correctSchemas?: number;
    violations?: number;
    violationDetails?: string[];
    violationFiles?: ValidationViolation[];
    debugMessages?: string[];
    processedFiles?: number;
    processedVessels?: number;
    progress?: string;
    jobId?: string;
    cancelled?: boolean;
}
export interface ProcessStatusApiResponse extends ApiResponse {
    isRunning: boolean;
    processType?: ProcessType;
    startTime?: string;
    totalFiles?: number;
    processedFiles?: number;
    currentFile?: string;
    progress?: number;
}
export interface ProcessCancelApiResponse extends ApiResponse {
    message: string;
}
export interface AnalysisApiResponse extends ApiResponse {
    analysis?: AnalysisResult;
    history?: AnalysisResult[];
    templates?: AnalysisTemplateInfo[];
    usage?: {
        input_tokens: number;
        output_tokens: number;
    };
}
export interface AnalysisResult {
    id: string;
    analysis: string;
    insights: string[];
    recommendations?: string[];
    anomalies?: AnomalyInfo[];
    confidence: number;
    dataQuality: string;
    timestamp: string;
    metadata: AnalysisMetadata;
}
export interface AnomalyInfo {
    timestamp: string;
    value: any;
    expectedRange: {
        min: number;
        max: number;
    };
    severity: 'low' | 'medium' | 'high';
    description: string;
    confidence: number;
}
export interface AnalysisMetadata {
    dataPath: string;
    analysisType: string;
    recordCount: number;
    timeRange?: {
        start: Date;
        end: Date;
    };
    templateUsed?: string;
}
export interface AnalysisTemplateInfo {
    id: string;
    name: string;
    description: string;
    category: string;
    icon: string;
    complexity: string;
    estimatedTime: string;
    requiredPaths: string[];
}
export interface ClaudeConnectionTestResponse extends ApiResponse {
    model?: string;
    responseTime?: number;
    tokenUsage?: number;
}
export interface TypedRequest<T = any> extends Request {
    body: T;
    params: {
        [key: string]: string;
    };
    query: {
        [key: string]: string;
    };
}
export interface TypedResponse<T = any> extends Response {
    json: (body: T) => this;
    status: (code: number) => this;
}
export type ProcessType = 'validation' | 'repair' | 'consolidation';
export interface ProcessState {
    type: ProcessType;
    isRunning: boolean;
    startTime: Date;
    totalFiles?: number;
    processedFiles?: number;
    currentFile?: string;
    cancelRequested?: boolean;
    abortController?: AbortController;
}
export interface SQLiteBufferInterface {
    isOpen(): boolean;
    insert(record: DataRecord): void;
    insertBatch(records: DataRecord[]): void;
    cleanup(): number;
    getStats(): {
        totalRecords: number;
        pendingRecords: number;
        exportedRecords: number;
        oldestPendingTimestamp: string | null;
        newestRecordTimestamp: string | null;
        dbSizeBytes: number;
        walSizeBytes: number;
    };
    getPendingCount(): number;
    getKnownPaths(): Set<string>;
    getTableColumns(signalkPath: string): Set<string> | undefined;
    getTableSchema(signalkPath: string): Array<{
        name: string;
        type: string;
    }> | undefined;
    getRowsForFederation(signalkPath: string, context: string, fromIso: string, toIso: string, afterId: number, limit: number): Array<Record<string, unknown>>;
    hasTable(signalkPath: string): boolean;
    getDbPath(): string;
    close(): void;
    checkpoint(): void;
    getDatesWithUnexportedRecords(excludeToday?: boolean): string[];
    getPathsForDate(date: Date): Array<{
        context: string;
        path: string;
    }>;
    getRecordsForPathAndDate(context: string, signalkPath: string, date: Date): DataRecord[];
    markDateExported(context: string, signalkPath: string, date: Date, batchId: string): void;
}
export interface ParquetExportServiceInterface {
    start(): void;
    stop(): void;
    forceExport(): Promise<{
        batchId: string;
        recordsExported: number;
        filesCreated: string[];
        duration: number;
        errors: string[];
    }>;
    exportDayToParquet(targetDate: Date): Promise<{
        batchId: string;
        recordsExported: number;
        filesCreated: string[];
        duration: number;
        errors: string[];
    }>;
    exportAllUnexported(): Promise<{
        batchId: string;
        recordsExported: number;
        filesCreated: string[];
        duration: number;
        errors: string[];
    }>;
    getStatus(): {
        isRunning: boolean;
        isExporting: boolean;
        lastExportTime: Date | null;
        lastBatchExported: number;
        totalExported: number;
        pendingRecords: number;
        dailyExportHour: number;
        mode: 'daily';
    };
    getHealth(): {
        healthy: boolean;
        lastExportTime: Date | null;
        pendingRecords: number;
        bufferStats: ReturnType<SQLiteBufferInterface['getStats']>;
    };
}
export interface PluginState {
    unsubscribes: Array<() => void>;
    streamSubscriptions?: any[];
    historicalStreamingService?: any;
    streamingService?: any;
    streamingEnabled?: boolean;
    restoredSubscriptions?: Map<string, any>;
    dataBuffers: LRUCache<string, DataRecord[]>;
    activeRegimens: Set<string>;
    subscribedPaths: Set<string>;
    saveInterval?: NodeJS.Timeout;
    consolidationInterval?: NodeJS.Timeout;
    parquetWriter?: ParquetWriter;
    cloudClient?: any;
    currentConfig?: PluginConfig;
    getDataDirPath: () => string;
    commandState: CommandRegistrationState;
    currentProcess?: ProcessState;
    sqliteBuffer?: SQLiteBufferInterface;
    sqliteBufferError?: string;
    exportService?: ParquetExportServiceInterface;
    autoDiscoveryService?: any;
}
export interface ParquetWriter {
    writeRecords(filepath: string, records: DataRecord[]): Promise<string>;
    writeJSON(filepath: string, records: DataRecord[]): Promise<string>;
    writeCSV(filepath: string, records: DataRecord[]): Promise<string>;
    writeParquet(filepath: string, records: DataRecord[]): Promise<string>;
    writeParquetBatched(filepath: string, firstBatch: DataRecord[], nextBatch: () => DataRecord[], currentPath?: string): Promise<string>;
    getSchemaService(): SchemaService | undefined;
}
export interface S3Config {
    region: string;
    credentials?: {
        accessKeyId: string;
        secretAccessKey: string;
    };
}
export interface QueryRequest {
    query: string;
}
export interface PathConfigRequest {
    path: Path;
    name?: string;
    enabled?: boolean;
    regimen?: string;
    source?: string;
    context?: Context;
}
export type CommandPutHandler = (context: string, path: string, value: any, callback?: (result: CommandExecutionResult) => void) => CommandExecutionResult;
export interface CommandExecutionResult {
    success: boolean;
    state: 'COMPLETED' | 'PENDING' | 'FAILED';
    statusCode?: number;
    message?: string;
    timestamp: string;
}
export interface CommandHistoryEntry {
    command: string;
    action: 'EXECUTE' | 'STOP' | 'REGISTER' | 'UNREGISTER' | 'UPDATE';
    value?: boolean;
    timestamp: string;
    success: boolean;
    error?: string;
}
export declare enum CommandStatus {
    ACTIVE = "ACTIVE",
    INACTIVE = "INACTIVE",
    PENDING = "PENDING",
    ERROR = "ERROR"
}
export type FileFormat = 'json' | 'csv' | 'parquet';
export type UploadTiming = 'realtime' | 'consolidation';
export type BufferKey = string;
export interface PluginError extends Error {
    code?: string;
    details?: any;
}
export interface ConsolidationOptions {
    outputDirectory: string;
    date: Date;
    filenamePrefix: string;
}
export interface ConsolidationResult {
    processedPaths: number;
    consolidatedFiles: string[];
    errors: string[];
}
export interface ParquetField {
    type: string;
    optional?: boolean;
    repeated?: boolean;
}
export interface ParquetSchema {
    [fieldName: string]: ParquetField;
}
export interface DataSummary {
    rowCount: number;
    timeRange: {
        start: Date;
        end: Date;
    };
    columns: ColumnInfo[];
    statisticalSummary: Record<string, Statistics>;
    dataQuality: DataQualityMetrics;
}
export interface ColumnInfo {
    name: string;
    type: string;
    nullCount: number;
    uniqueCount: number;
    sampleValues: any[];
}
export interface Statistics {
    count: number;
    mean?: number;
    median?: number;
    min?: any;
    max?: any;
    stdDev?: number;
}
export interface DataQualityMetrics {
    completeness: number;
    consistency: number;
    timeliness: number;
    accuracy: number;
}
export interface ProcessingStats {
    totalBuffers: number;
    buffersWithData: number;
    totalRecords: number;
    processedPaths: string[];
}
//# sourceMappingURL=types.d.ts.map