signalk-parquet/src/types.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

import {
  Context,
  Path,
  ServerAPI,
  NormalizedDelta,
  SourceRef,
} from '@signalk/server-api';

// Re-export SignalK types for convenience
export { NormalizedDelta, SourceRef };
import { Request, Response, Router } from 'express';
import { LRUCache } from './utils/lru-cache';

// Forward declaration to avoid circular dependency
export interface SchemaService {
  detectOptimalSchema(
    records: DataRecord[],
    currentPath?: string
  ): Promise<any>;
  validateFileSchema(filePath: string): Promise<any>;
  repairFileSchema(filePath: string, filenamePrefix?: string): Promise<any>;
}

// SignalK Plugin Interface
export interface SignalKPlugin {
  id: string;
  name: string;
  description: string;
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  schema: any;
  start: (options: Partial<PluginConfig>) => void;
  stop: () => void;
  registerWithRouter?: (router: Router) => void;
}

// Plugin Configuration
// Auto-discovery configuration for automatic path recording
export interface AutoDiscoveryConfig {
  enabled: boolean; // Master switch (default: false)
  excludePatterns?: string[]; // Paths to never auto-configure (glob patterns)
  includePatterns?: string[]; // Restrict to matching paths only (glob patterns)
  maxAutoConfiguredPaths?: number; // Limit (default: 100)
  requireLiveData: boolean; // Only configure if path has live SignalK data
}

export interface PluginConfig {
  bufferSize: number;
  saveIntervalSeconds: number;
  outputDirectory: string;
  filenamePrefix: string;
  // Global retention period in days for tier=raw. Tiers above raw use a
  // multiplier (5s=2x, 60s=4x, 1h=12x). 0 means "keep forever" — the
  // historical default behaviour, since the cleanup job was previously
  // never scheduled.
  retentionDays: number;
  // Optional per-path overrides. Match by glob (`*` matches any chars
  // including dots); most-specific pattern wins, ties broken by
  // declaration order. See utils/retention-rules.ts.
  pathRetentionOverrides?: import('./utils/retention-rules').PathRetentionRule[];
  // Bumped each time we run a non-trivial migration over saved options.
  // Absent on installs that have never started 0.7.40+. Used to make
  // migrations one-shot — once we've seen and stamped a config, we
  // trust the values in it literally (so an operator who explicitly
  // re-enters the legacy default in the UI keeps it on next start).
  configSchemaVersion?: number;
  fileFormat: 'json' | 'csv' | 'parquet';
  vesselMMSI: string;
  cloudUpload: CloudUploadConfig;
  enableStreaming?: boolean; // Enable WebSocket streaming functionality
  claudeIntegration?: ClaudeIntegrationConfig;
  homePortLatitude?: number;
  homePortLongitude?: number;
  setCurrentLocationAction?: {
    setCurrentLocation: boolean;
  };
  // SQLite buffer and Hive partitioning options
  useSqliteBuffer?: boolean; // Use SQLite WAL buffer instead of in-memory LRU
  exportBatchSize?: number; // Max records to export per cycle (default 10000)
  bufferRetentionHours?: number; // How long to keep exported records in SQLite (default 48)
  useHivePartitioning?: boolean; // Use Hive-style partitioning for Parquet files
  dailyExportHour?: number; // Hour (0-23 UTC) to run daily export (default 4 = 4 AM UTC)
  autoDiscovery?: AutoDiscoveryConfig; // Auto-discovery configuration
  enableRawSql?: boolean; // Enable raw SQL queries via /api/query endpoint
}

import type { ClaudeModel } from './claude-models';

export interface ClaudeIntegrationConfig {
  enabled: boolean;
  apiKey?: string;
  model?: ClaudeModel;
  maxTokens?: number;
  temperature?: number;
  autoAnalysis?: {
    daily: boolean;
    anomaly: boolean;
    threshold: number;
  };
  cacheEnabled?: boolean;
  templates?: string[];
}

// Vessel Context Document for Claude AI Analysis
export interface VesselContext {
  vesselInfo: VesselInfo;
  customContext: string;
  lastUpdated: string;
  autoExtracted: boolean;
}

export interface VesselInfo {
  // Basic vessel identification
  name?: string;
  callsign?: string;
  mmsi?: string;

  // Physical characteristics
  length?: number; // Length Overall (LOA) in meters
  beam?: number; // Beam in meters
  draft?: number; // Draft in meters
  height?: number; // Height/air draft in meters
  displacement?: number; // Weight/displacement in tons

  // Vessel classification
  vesselType?: string; // Type of vessel (sailboat, motorboat, cargo, etc.)
  classification?: string; // Classification society info
  flag?: string; // Flag state

  // Technical specifications
  grossTonnage?: number;
  netTonnage?: number;
  deadWeight?: number;

  // Build information
  builder?: string;
  buildYear?: number;
  hullNumber?: string;

  // Contact information
  ownerName?: string;
  port?: string; // Port of registry

  // Additional context
  notes?: string;
}

export interface VesselContextExtraction {
  path: string;
  signalkPath: string;
  displayName: string;
  unit?: string;
  category:
    | 'identification'
    | 'physical'
    | 'classification'
    | 'technical'
    | 'build'
    | 'contact';
}

export interface PathConfig {
  path: Path;
  name?: string;
  enabled?: boolean;
  regimen?: string;
  source?: string;
  context?: Context;
  excludeMMSI?: string[]; // Array of MMSI numbers to exclude when using vessels.*
  autoDiscovered?: boolean; // Track which paths were auto-discovered
}

// Command Registration Types
/**
 * Threshold operator types based on data type
 */
export type ThresholdOperator =
  // Numeric/Angular operators
  | 'gt' // Greater than
  | 'lt' // Less than
  | 'eq' // Equal to
  | 'ne' // Not equal to
  | 'range' // Within range (min/max)
  // String operators
  | 'contains' // String contains substring
  | 'startsWith' // String starts with
  | 'endsWith' // String ends with
  | 'stringEquals' // String equals (case-sensitive)
  // Boolean operators
  | 'true' // Is true
  | 'false' // Is false
  // Position operators
  | 'withinRadius' // Within radius of point
  | 'outsideRadius' // Outside radius of point
  | 'inBoundingBox' // Inside bounding box
  | 'outsideBoundingBox'; // Outside bounding box

/**
 * Bounding box for geographic area thresholds
 */
export interface BoundingBox {
  north: number; // Northern latitude boundary
  south: number; // Southern latitude boundary
  east: number; // Eastern longitude boundary
  west: number; // Western longitude boundary
}

/**
 * Type-aware threshold configuration
 */
export interface ThresholdConfig {
  enabled: boolean;
  watchPath: string; // SignalK path to monitor
  operator: ThresholdOperator; // Threshold operator

  // Simple value (for most operators)
  value?: number | boolean | string;

  // Range operator values
  valueMin?: number; // Minimum value for range operator
  valueMax?: number; // Maximum value for range operator

  // Position-based threshold values
  latitude?: number; // Target latitude for position operators
  longitude?: number; // Target longitude for position operators
  radius?: number; // Radius in meters for position operators
  boundingBox?: BoundingBox; // Bounding box for area operators (manual mode)
  useHomePort?: boolean; // Use home port location instead of custom lat/lon
  boxSize?: number; // Box size in meters (for home port-based bounding box)
  boxAnchor?: string; // Anchor point for home port-based box (nw, n, ne, w, center, e, sw, s, se)
  boxBuffer?: number; // Buffer in meters to add to bounding box (default: 5m for GPS accuracy)

  activateOnMatch: boolean; // true = activate command when condition met, false = deactivate
  hysteresis?: number; // Optional: prevent rapid switching (seconds)
}

export interface CommandConfig {
  command: string;
  path: string;
  registered: string;
  description?: string;
  keywords?: string[]; // For Claude context matching
  active?: boolean;
  lastExecuted?: string;
  defaultState?: boolean; // Default on/off state when no threshold or manual override
  thresholds?: ThresholdConfig[]; // Threshold-based activation (multiple thresholds supported)
  manualOverride?: boolean; // True when manually controlled via PUT
  manualOverrideUntil?: string; // ISO timestamp when override expires (optional)
}

export interface CommandRegistrationState {
  registeredCommands: Map<string, CommandConfig>;
  putHandlers: Map<string, CommandPutHandler>;
}

export interface CommandExecutionRequest {
  command: string;
  value: boolean;
  timestamp?: string;
}

export interface CommandRegistrationRequest {
  command: string;
  description?: string;
  keywords?: string[];
  defaultState?: boolean;
  thresholds?: ThresholdConfig[];
}

// Web App Configuration (stored separately from plugin config)
export interface WebAppPathConfig {
  paths: PathConfig[];
  commands: CommandConfig[];
}

export interface CloudUploadConfig {
  provider: 'none' | 's3' | 'r2';
  bucket?: string;
  region?: string; // S3 only (e.g. 'us-east-1', or 'garage' for Garage)
  endpoint?: string; // S3 only - custom endpoint URL for self-hosted S3-compatible services (Garage, MinIO, etc.)
  forcePathStyle?: boolean; // S3 only - use path-style addressing instead of virtual-hosted-style (often required by self-hosted S3-compatible services)
  accountId?: string; // R2 only (Cloudflare account ID)
  keyPrefix?: string;
  accessKeyId?: string;
  secretAccessKey?: string;
  deleteAfterUpload?: boolean;
}

// SignalK Data Structures
export interface SignalKSubscription {
  context: string;
  subscribe: Array<{
    path: string;
    period: number;
  }>;
}

// Data Record Structure
export interface DataRecord {
  received_timestamp: string;
  signalk_timestamp: string;
  context: string;
  path: string;
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  value: any;
  value_json?: string | object; // Store as object in memory, serialize when writing
  source?: string | object; // Store as object in memory, serialize when writing
  source_label?: string;
  source_type?: string;
  source_pgn?: number;
  source_src?: string;
  meta?: string | object; // Store as object in memory, serialize when writing
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  [key: string]: any; // For flattened object properties
}

// Parquet Writer Options
export interface ParquetWriterOptions {
  format: 'json' | 'csv' | 'parquet';
  app?: ServerAPI;
}

// File System Related
export interface FileInfo {
  name: string;
  path: string;
  size: number;
  modified: string;
}

export interface PathInfo {
  path: string;
  directory: string;
  fileCount: number;
}

// API Response Types
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export interface ApiResponse<T = any> {
  success: boolean;
  data?: T;
  error?: string;
  message?: string;
}

// Command API Response Types
export interface CommandApiResponse extends ApiResponse {
  commands?: CommandConfig[];
  command?: CommandConfig;
  count?: number;
}

export interface CommandExecutionResponse extends ApiResponse {
  command?: string;
  value?: boolean;
  executed?: boolean;
  timestamp?: string;
}

export interface PathsApiResponse extends ApiResponse {
  dataDirectory?: string;
  paths?: PathInfo[];
}

export interface FilesApiResponse extends ApiResponse {
  path?: string;
  directory?: string;
  files?: FileInfo[];
}

export interface QueryApiResponse extends ApiResponse {
  query?: string;
  rowCount?: number;
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  data?: any[];
}

export interface SampleApiResponse extends ApiResponse {
  path?: string;
  file?: string;
  columns?: string[];
  rowCount?: number;
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  data?: any[];
}

export interface ConfigApiResponse extends ApiResponse {
  paths?: PathConfig[];
}

export interface HealthApiResponse extends ApiResponse {
  status?: string;
  timestamp?: string;
  duckdb?: string;
}

export interface CloudTestApiResponse extends ApiResponse {
  provider?: string;
  bucket?: string;
  region?: string;
  accountId?: string;
  keyPrefix?: string;
}

// Legacy alias
export type S3TestApiResponse = CloudTestApiResponse;

export interface ValidationViolation {
  file: string;
  vessel?: string;
  issues: string[];
}

export interface ValidationApiResponse extends ApiResponse {
  totalFiles?: number;
  totalVessels?: number;
  correctSchemas?: number;
  violations?: number;
  violationDetails?: string[];
  violationFiles?: ValidationViolation[];
  debugMessages?: string[];
  processedFiles?: number;
  processedVessels?: number;
  progress?: string;
  jobId?: string;
  cancelled?: boolean;
}

export interface ProcessStatusApiResponse extends ApiResponse {
  isRunning: boolean;
  processType?: ProcessType;
  startTime?: string;
  totalFiles?: number;
  processedFiles?: number;
  currentFile?: string;
  progress?: number; // percentage 0-100
}

export interface ProcessCancelApiResponse extends ApiResponse {
  message: string;
}

// Claude Analysis API Response Types
export interface AnalysisApiResponse extends ApiResponse {
  analysis?: AnalysisResult;
  history?: AnalysisResult[];
  templates?: AnalysisTemplateInfo[];
  usage?: {
    input_tokens: number;
    output_tokens: number;
  };
}

export interface AnalysisResult {
  id: string;
  analysis: string;
  insights: string[];
  recommendations?: string[];
  anomalies?: AnomalyInfo[];
  confidence: number;
  dataQuality: string;
  timestamp: string;
  metadata: AnalysisMetadata;
}

export interface AnomalyInfo {
  timestamp: string;
  value: any;
  expectedRange: { min: number; max: number };
  severity: 'low' | 'medium' | 'high';
  description: string;
  confidence: number;
}

export interface AnalysisMetadata {
  dataPath: string;
  analysisType: string;
  recordCount: number;
  timeRange?: { start: Date; end: Date };
  templateUsed?: string;
}

export interface AnalysisTemplateInfo {
  id: string;
  name: string;
  description: string;
  category: string;
  icon: string;
  complexity: string;
  estimatedTime: string;
  requiredPaths: string[];
}

export interface ClaudeConnectionTestResponse extends ApiResponse {
  model?: string;
  responseTime?: number;
  tokenUsage?: number;
}

// Express Router Types
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export interface TypedRequest<T = any> extends Request {
  body: T;
  params: { [key: string]: string };
  query: { [key: string]: string };
}

// eslint-disable-next-line @typescript-eslint/no-explicit-any
export interface TypedResponse<T = any> extends Response {
  json: (body: T) => this;
  status: (code: number) => this;
}

// Internal Plugin State
// Process management types
export type ProcessType = 'validation' | 'repair' | 'consolidation';

export interface ProcessState {
  type: ProcessType;
  isRunning: boolean;
  startTime: Date;
  totalFiles?: number;
  processedFiles?: number;
  currentFile?: string;
  cancelRequested?: boolean;
  abortController?: AbortController;
}

// Forward declaration for SQLiteBuffer to avoid circular dependency
export interface SQLiteBufferInterface {
  isOpen(): boolean;
  insert(record: DataRecord): void;
  insertBatch(records: DataRecord[]): void;
  cleanup(): number;
  getStats(): {
    totalRecords: number;
    pendingRecords: number;
    exportedRecords: number;
    oldestPendingTimestamp: string | null;
    newestRecordTimestamp: string | null;
    dbSizeBytes: number;
    walSizeBytes: number;
  };
  getPendingCount(): number;
  getKnownPaths(): Set<string>;
  getTableColumns(signalkPath: string): Set<string> | undefined;
  getTableSchema(
    signalkPath: string
  ): Array<{ name: string; type: string }> | undefined;
  getRowsForFederation(
    signalkPath: string,
    context: string,
    fromIso: string,
    toIso: string,
    afterId: number,
    limit: number
  ): Array<Record<string, unknown>>;
  hasTable(signalkPath: string): boolean;
  getDbPath(): string;
  close(): void;
  checkpoint(): void;
  // Daily export methods
  getDatesWithUnexportedRecords(excludeToday?: boolean): string[];
  getPathsForDate(date: Date): Array<{ context: string; path: string }>;
  getRecordsForPathAndDate(
    context: string,
    signalkPath: string,
    date: Date
  ): DataRecord[];
  markDateExported(
    context: string,
    signalkPath: string,
    date: Date,
    batchId: string
  ): void;
}

// Forward declaration for ParquetExportService to avoid circular dependency
export interface ParquetExportServiceInterface {
  start(): void;
  stop(): void;
  forceExport(): Promise<{
    batchId: string;
    recordsExported: number;
    filesCreated: string[];
    duration: number;
    errors: string[];
  }>;
  // Daily export methods
  exportDayToParquet(targetDate: Date): Promise<{
    batchId: string;
    recordsExported: number;
    filesCreated: string[];
    duration: number;
    errors: string[];
  }>;
  exportAllUnexported(): Promise<{
    batchId: string;
    recordsExported: number;
    filesCreated: string[];
    duration: number;
    errors: string[];
  }>;
  getStatus(): {
    isRunning: boolean;
    isExporting: boolean;
    lastExportTime: Date | null;
    lastBatchExported: number;
    totalExported: number;
    pendingRecords: number;
    dailyExportHour: number;
    mode: 'daily';
  };
  getHealth(): {
    healthy: boolean;
    lastExportTime: Date | null;
    pendingRecords: number;
    bufferStats: ReturnType<SQLiteBufferInterface['getStats']>;
  };
}

export interface PluginState {
  unsubscribes: Array<() => void>;
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  streamSubscriptions?: any[]; // Store streambundle stream references for cleanup
  historicalStreamingService?: any; // HistoricalStreamingService - avoiding circular import
  streamingService?: any; // WebSocket streaming service for runtime control
  streamingEnabled?: boolean; // Runtime control separate from config
  restoredSubscriptions?: Map<string, any>; // Track active subscriptions
  dataBuffers: LRUCache<string, DataRecord[]>;
  activeRegimens: Set<string>;
  subscribedPaths: Set<string>;
  saveInterval?: NodeJS.Timeout;
  consolidationInterval?: NodeJS.Timeout;
  parquetWriter?: ParquetWriter;
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  cloudClient?: any; // S3 or R2 client (S3-compatible)
  currentConfig?: PluginConfig;
  getDataDirPath: () => string;
  commandState: CommandRegistrationState;
  // Process management
  currentProcess?: ProcessState;
  // SQLite buffer and export service (new)
  sqliteBuffer?: SQLiteBufferInterface;
  sqliteBufferError?: string;
  exportService?: ParquetExportServiceInterface;
  // Auto-discovery service
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  autoDiscoveryService?: any; // AutoDiscoveryService - avoiding circular import
}

// Parquet Writer Class Interface
export interface ParquetWriter {
  writeRecords(filepath: string, records: DataRecord[]): Promise<string>;
  writeJSON(filepath: string, records: DataRecord[]): Promise<string>;
  writeCSV(filepath: string, records: DataRecord[]): Promise<string>;
  writeParquet(filepath: string, records: DataRecord[]): Promise<string>;
  writeParquetBatched(
    filepath: string,
    firstBatch: DataRecord[],
    nextBatch: () => DataRecord[],
    currentPath?: string
  ): Promise<string>;
  getSchemaService(): SchemaService | undefined;
}

// S3 Related Types
export interface S3Config {
  region: string;
  credentials?: {
    accessKeyId: string;
    secretAccessKey: string;
  };
}

// Query Request/Response Types
export interface QueryRequest {
  query: string;
}

export interface PathConfigRequest {
  path: Path;
  name?: string;
  enabled?: boolean;
  regimen?: string;
  source?: string;
  context?: Context;
}
//FIXME https://github.com/SignalK/signalk-server/pull/2043
// Command Types
export type CommandPutHandler = (
  context: string,
  path: string,
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  value: any,
  callback?: (result: CommandExecutionResult) => void
) => CommandExecutionResult;

export interface CommandExecutionResult {
  success: boolean;
  state: 'COMPLETED' | 'PENDING' | 'FAILED';
  statusCode?: number;
  message?: string;
  timestamp: string;
}

export interface CommandHistoryEntry {
  command: string;
  action: 'EXECUTE' | 'STOP' | 'REGISTER' | 'UNREGISTER' | 'UPDATE';
  value?: boolean;
  timestamp: string;
  success: boolean;
  error?: string;
}

export enum CommandStatus {
  ACTIVE = 'ACTIVE',
  INACTIVE = 'INACTIVE',
  PENDING = 'PENDING',
  ERROR = 'ERROR',
}

// Utility Types
export type FileFormat = 'json' | 'csv' | 'parquet';
export type UploadTiming = 'realtime' | 'consolidation';
export type BufferKey = string; // Format: "context:path"

// Error Types
export interface PluginError extends Error {
  code?: string;
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  details?: any;
}

// Consolidation Types
export interface ConsolidationOptions {
  outputDirectory: string;
  date: Date;
  filenamePrefix: string;
}

export interface ConsolidationResult {
  processedPaths: number;
  consolidatedFiles: string[];
  errors: string[];
}

// Schema Definition Types
export interface ParquetField {
  type: string;
  optional?: boolean;
  repeated?: boolean;
}

export interface ParquetSchema {
  [fieldName: string]: ParquetField;
}

// Data Analysis Related Types
export interface DataSummary {
  rowCount: number;
  timeRange: { start: Date; end: Date };
  columns: ColumnInfo[];
  statisticalSummary: Record<string, Statistics>;
  dataQuality: DataQualityMetrics;
}

export interface ColumnInfo {
  name: string;
  type: string;
  nullCount: number;
  uniqueCount: number;
  sampleValues: any[];
}

export interface Statistics {
  count: number;
  mean?: number;
  median?: number;
  min?: any;
  max?: any;
  stdDev?: number;
}

export interface DataQualityMetrics {
  completeness: number; // Percentage of non-null values
  consistency: number; // Data format consistency
  timeliness: number; // Data freshness
  accuracy: number; // Estimated data accuracy
}

// File Processing Types
export interface ProcessingStats {
  totalBuffers: number;
  buffersWithData: number;
  totalRecords: number;
  processedPaths: string[];
}