signalk-parquet/src/api-routes.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

import * as fs from 'fs-extra';
import * as path from 'path';
import { glob } from 'glob';
import express, { Router } from 'express';
import multer from 'multer';
import { getAvailablePaths } from './utils/path-discovery';
import { DuckDBPool } from './utils/duckdb-pool';
import {
  TypedRequest,
  TypedResponse,
  PathsApiResponse,
  FilesApiResponse,
  QueryApiResponse,
  SampleApiResponse,
  ConfigApiResponse,
  HealthApiResponse,
  S3TestApiResponse,
  QueryRequest,
  PathConfigRequest,
  CommandApiResponse,
  CommandRegistrationRequest,
  CommandExecutionRequest,
  CommandExecutionResponse,
  PluginState,
  AnalysisApiResponse,
  ClaudeConnectionTestResponse,
  ValidationApiResponse,
  ValidationViolation,
} from './types';
import {
  ProcessType,
  ProcessStatusApiResponse,
  ProcessCancelApiResponse,
} from './types';
import { MigrationService } from './services/migration-service';
import { GPX_UPLOAD_MAX_FILE_BYTES, GPX_UPLOAD_MAX_FILES } from './constants';
import {
  GpxImportService,
  DEFAULT_IMPORT_PATHS,
  GpxImportPath,
} from './services/gpx-import-service';
import {
  CompactionConflictError,
  CompactionService,
} from './services/compaction-service';
import {
  AggregationService,
  buildPerTierRetention,
} from './services/aggregation-service';
import { AggregationTier, HivePathBuilder } from './utils/hive-path-builder';
import { isAngularPath } from './utils/angular-paths';
import {
  loadWebAppConfig,
  saveWebAppConfig,
  registerCommand,
  updateCommand,
  unregisterCommand,
  executeCommand,
  getCurrentCommands,
  getCommandHistory,
  getCommandState,
  setManualOverride,
  updatePluginConfig,
} from './commands';
import { updateDataSubscriptions } from './data-handler';
import { toContextFilePath, toParquetFilePath } from './utils/path-helpers';
import { ServerAPI, Context } from '@signalk/server-api';
import { ClaudeAnalyzer, AnalysisRequest } from './claude-analyzer';
import {
  AnalysisTemplateManager,
  TEMPLATE_CATEGORIES,
} from './analysis-templates';
import { VesselContextManager } from './vessel-context';
// import { initializeStreamingService, shutdownStreamingService } from './index';

// Progress tracking for validation jobs
interface ValidationProgress {
  jobId: string;
  status: 'running' | 'cancelling' | 'completed' | 'cancelled' | 'error';
  processed: number;
  total: number;
  percent: number;
  startTime: Date;
  currentFile?: string;
  currentVessel?: string;
  currentRelativePath?: string;
  cancelRequested?: boolean;
  error?: string;
  completedAt?: Date;
  result?: ValidationApiResponse;
}

const validationJobs = new Map<string, ValidationProgress>();

let lastValidationViolations: ValidationViolation[] = [];

interface RepairProgress {
  jobId: string;
  status: 'running' | 'cancelling' | 'completed' | 'cancelled' | 'error';
  processed: number;
  total: number;
  percent: number;
  startTime: Date;
  currentFile?: string;
  message?: string;
  cancelRequested?: boolean;
  completedAt?: Date;
  result?: {
    success: boolean;
    repairedFiles: number;
    backedUpFiles: number;
    skippedFiles: string[];
    quarantinedFiles: string[];
    errors: string[];
    message?: string;
  };
}

const repairJobs = new Map<string, RepairProgress>();

const VALIDATION_JOB_TTL_MS = 10 * 60 * 1000; // Retain job metadata for 10 minutes

function scheduleValidationJobCleanup(jobId: string) {
  setTimeout(() => {
    const job = validationJobs.get(jobId);
    if (job && job.status !== 'running') {
      validationJobs.delete(jobId);
    }
  }, VALIDATION_JOB_TTL_MS);
}

const REPAIR_JOB_TTL_MS = 10 * 60 * 1000;

function scheduleRepairJobCleanup(jobId: string) {
  setTimeout(() => {
    const job = repairJobs.get(jobId);
    if (job && job.status !== 'running') {
      repairJobs.delete(jobId);
    }
  }, REPAIR_JOB_TTL_MS);
}

// Shared analyzer instance to maintain conversation state across requests
let sharedAnalyzer: ClaudeAnalyzer | null = null;

/**
 * Get or create the shared Claude analyzer instance
 */
function getSharedAnalyzer(
  config: any,
  app: ServerAPI,
  dataDir: string,
  state: PluginState
): ClaudeAnalyzer {
  if (!sharedAnalyzer) {
    sharedAnalyzer = new ClaudeAnalyzer(
      {
        apiKey: config.claudeIntegration.apiKey,
        model: migrateClaudeModel(config.claudeIntegration.model, app) as any,
        maxTokens: config.claudeIntegration.maxTokens || 4000,
        temperature: config.claudeIntegration.temperature || 0.3,
      },
      app,
      dataDir,
      state
    );
    app.debug('🔧 Created shared Claude analyzer instance');
  }
  return sharedAnalyzer;
}

// AWS S3 for testing connection
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let ListObjectsV2Command: any;

import { getValidClaudeModel } from './claude-models';

// Helper function to migrate deprecated Claude model names
function migrateClaudeModel(model?: string, app?: ServerAPI): string {
  const validatedModel = getValidClaudeModel(model);
  if (model && validatedModel !== model) {
    app?.debug(`Auto-migrated Claude model ${model} to ${validatedModel}`);
  }
  return validatedModel;
}

// ===========================================
// PROCESS MANAGEMENT UTILITIES
// ===========================================

function _startProcess(
  state: PluginState,
  type: ProcessType,
  totalFiles?: number
): boolean {
  // Check if another process is already running
  if (state.currentProcess?.isRunning) {
    return false; // Cannot start, another process is active
  }

  // Initialize new process
  state.currentProcess = {
    type,
    isRunning: true,
    startTime: new Date(),
    totalFiles,
    processedFiles: 0,
    cancelRequested: false,
    abortController: new AbortController(),
  };

  return true;
}

function _updateProcessProgress(
  state: PluginState,
  processedFiles: number,
  currentFile?: string
): void {
  if (state.currentProcess?.isRunning) {
    state.currentProcess.processedFiles = processedFiles;
    state.currentProcess.currentFile = currentFile;
  }
}

function _finishProcess(state: PluginState): void {
  if (state.currentProcess) {
    state.currentProcess.isRunning = false;
    // Keep the process data for a short time for status queries
    setTimeout(() => {
      if (state.currentProcess && !state.currentProcess.isRunning) {
        state.currentProcess = undefined;
      }
    }, 30000); // Clear after 30 seconds
  }
}

function cancelProcess(state: PluginState): boolean {
  if (state.currentProcess?.isRunning) {
    state.currentProcess.cancelRequested = true;
    state.currentProcess.abortController?.abort();
    return true;
  }
  return false;
}

function getProcessStatus(state: PluginState): ProcessStatusApiResponse {
  if (!state.currentProcess) {
    return {
      success: true,
      isRunning: false,
    };
  }

  const process = state.currentProcess;
  const progress =
    process.totalFiles && process.processedFiles !== undefined
      ? Math.round((process.processedFiles / process.totalFiles) * 100)
      : undefined;

  return {
    success: true,
    isRunning: process.isRunning,
    processType: process.type,
    startTime: process.startTime.toISOString(),
    totalFiles: process.totalFiles,
    processedFiles: process.processedFiles,
    currentFile: process.currentFile,
    progress,
  };
}

export function registerApiRoutes(
  router: Router,
  state: PluginState,
  app: ServerAPI
): void {
  // Serve static files from public directory
  const publicPath = path.join(__dirname, '../public');
  if (fs.existsSync(publicPath)) {
    router.use(express.static(publicPath));
  }

  // Convert BigInt values to regular numbers for JSON serialization
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  function mapForJSON(rawData: any[]): any[] {
    return rawData.map(row => {
      const convertedRow: typeof row = {};
      for (const [key, value] of Object.entries(row)) {
        convertedRow[key] = typeof value === 'bigint' ? Number(value) : value;
      }
      return convertedRow;
    });
  }

  // Returns sorted parquet files for a SignalK path, or null if the directory doesn't exist
  function getDataFilesForPath(signalkPath: string): {
    pathDir: string;
    files: Array<{
      name: string;
      path: string;
      size: number;
      modified: string;
    }>;
  } | null {
    const selfContextPath = app.selfContext
      .replace(/\./g, '/')
      .replace(/:/g, '_');
    const pathDir = path.join(
      state.getDataDirPath(),
      selfContextPath,
      signalkPath.replace(/\./g, '/')
    );
    if (!fs.existsSync(pathDir)) return null;
    const files = fs
      .readdirSync(pathDir)
      .filter((file: string) => file.endsWith('.parquet'))
      .map((file: string) => {
        const filePath = path.join(pathDir, file);
        const stat = fs.statSync(filePath);
        return {
          name: file,
          path: filePath,
          size: stat.size,
          modified: stat.mtime.toISOString(),
        };
      })
      .sort((a, b) => b.modified.localeCompare(a.modified));
    return { pathDir, files };
  }

  // Get available SignalK paths
  router.get(
    '/api/paths',
    (_: TypedRequest, res: TypedResponse<PathsApiResponse>) => {
      try {
        const dataDir = state.getDataDirPath();
        const paths = getAvailablePaths(dataDir, app);

        return res.json({
          success: true,
          dataDirectory: dataDir,
          paths: paths,
        });
      } catch (error) {
        return res.status(500).json({
          success: false,
          error: (error as Error).message,
        });
      }
    }
  );

  // Get files for a specific path
  router.get(
    '/api/files/:path(*)',
    (req: TypedRequest, res: TypedResponse<FilesApiResponse>) => {
      try {
        const signalkPath = req.params.path;
        const result = getDataFilesForPath(signalkPath);
        if (!result) {
          return res.status(404).json({
            success: false,
            error: `Path not found: ${signalkPath}`,
          });
        }
        return res.json({
          success: true,
          path: signalkPath,
          directory: result.pathDir,
          files: result.files,
        });
      } catch (error) {
        return res.status(500).json({
          success: false,
          error: (error as Error).message,
        });
      }
    }
  );

  // Get sample data from a specific file
  router.get(
    '/api/sample/:path(*)',
    async (req: TypedRequest, res: TypedResponse<SampleApiResponse>) => {
      try {
        const signalkPath = req.params.path;
        const limit = parseInt(req.query.limit as string) || 10;

        const result = getDataFilesForPath(signalkPath);
        if (!result) {
          return res.status(404).json({
            success: false,
            error: `Path not found: ${signalkPath}`,
          });
        }

        if (result.files.length === 0) {
          return res.status(404).json({
            success: false,
            error: `No parquet files found for path: ${signalkPath}`,
          });
        }

        const sampleFile = result.files[0];
        const query = `SELECT * FROM read_parquet('${sampleFile.path}', union_by_name=true) LIMIT ${limit}`;

        // Get connection from pool (spatial extension already loaded)
        const connection = await DuckDBPool.getConnection();

        try {
          const reader = await connection.runAndReadAll(query);
          const rawData = reader.getRowObjects();

          const data = mapForJSON(rawData);

          // Get column info
          const columns = data.length > 0 ? Object.keys(data[0]) : [];

          return res.json({
            success: true,
            path: signalkPath,
            file: sampleFile.name,
            columns: columns,
            rowCount: data.length,
            data: data,
          });
        } catch (err) {
          return res.status(400).json({
            success: false,
            error: (err as Error).message,
          });
        } finally {
          connection.disconnectSync();
        }
      } catch (error) {
        return res.status(500).json({
          success: false,
          error: (error as Error).message,
        });
      }
    }
  );

  // Query parquet data (raw SQL - disabled by default for security)
  // Enable with: SIGNALK_PARQUET_RAW_SQL=true
  router.post(
    '/api/query',
    async (
      req: TypedRequest<QueryRequest>,
      res: TypedResponse<QueryApiResponse>
    ) => {
      try {
        // Security: Raw SQL is disabled by default
        const rawSqlEnabled =
          process.env.SIGNALK_PARQUET_RAW_SQL === 'true' ||
          state.currentConfig?.enableRawSql === true;
        if (!rawSqlEnabled) {
          return res.status(403).json({
            success: false,
            error:
              'Raw SQL queries are disabled. Enable in plugin settings or set SIGNALK_PARQUET_RAW_SQL=true.',
          });
        }

        const { query } = req.body;

        if (!query) {
          return res.status(400).json({
            success: false,
            error: 'Query is required',
          });
        }

        const dataDir = state.getDataDirPath();

        // Replace placeholder paths in query with actual file paths
        let processedQuery = query;

        // Find all quoted paths in the query that might be SignalK paths
        const pathMatches = query.match(/'([^']+)'/g);
        if (pathMatches) {
          pathMatches.forEach(match => {
            const quotedPath = match.slice(1, -1); // Remove quotes

            // If it looks like a SignalK path, convert to file path
            const selfContextPath = toContextFilePath(
              app.selfContext as Context
            );
            if (
              quotedPath.includes(`/${selfContextPath}/`) ||
              quotedPath.includes('.parquet')
            ) {
              // It's already a file path, use as is
              return;
            } else if (quotedPath.includes('.') && !quotedPath.includes('/')) {
              // It's a SignalK path, convert to file path
              const filePath = toParquetFilePath(
                dataDir,
                selfContextPath,
                quotedPath
              );
              processedQuery = processedQuery.replace(match, `'${filePath}'`);
            }
          });
        }

        // Get connection from pool (spatial extension already loaded)
        const connection = await DuckDBPool.getConnection();

        try {
          const reader = await connection.runAndReadAll(processedQuery);
          const rawData = reader.getRowObjects();

          const data = mapForJSON(rawData);

          return res.json({
            success: true,
            query: processedQuery,
            rowCount: data.length,
            data: data,
          });
        } catch (err) {
          app.error(`Query error: ${err}`);
          return res.status(400).json({
            success: false,
            error: (err as Error).message,
          });
        } finally {
          connection.disconnectSync();
        }
      } catch (error) {
        return res.status(500).json({
          success: false,
          error: (error as Error).message,
        });
      }
    }
  );

  // Check if raw SQL queries are enabled (for UI visibility)
  router.get('/api/query/enabled', (_req, res) => {
    const rawSqlEnabled =
      process.env.SIGNALK_PARQUET_RAW_SQL === 'true' ||
      state.currentConfig?.enableRawSql === true;
    return res.json({
      success: true,
      enabled: rawSqlEnabled,
    });
  });

  // Test cloud connection (S3 or R2)
  router.post(
    '/api/test-cloud',
    async (_: TypedRequest, res: TypedResponse<S3TestApiResponse>) => {
      try {
        if (!state.currentConfig) {
          return res.status(500).json({
            success: false,
            error: 'Plugin not started or configuration not available',
          });
        }

        const cloud = state.currentConfig.cloudUpload;
        if (cloud.provider === 'none') {
          return res.status(400).json({
            success: false,
            error: 'Cloud upload is not enabled in configuration',
          });
        }

        if (!ListObjectsV2Command || !state.cloudClient) {
          try {
            const awsS3 = await import('@aws-sdk/client-s3');
            ListObjectsV2Command = awsS3.ListObjectsV2Command;
          } catch (importError) {
            return res.status(503).json({
              success: false,
              error: 'Cloud client not available or not initialized',
            });
          }
        }

        const listCommand = new ListObjectsV2Command({
          Bucket: cloud.bucket,
          MaxKeys: 1,
        });

        await state.cloudClient.send(listCommand);

        const label = cloud.provider.toUpperCase();
        return res.json({
          success: true,
          message: `${label} connection successful`,
          provider: cloud.provider,
          bucket: cloud.bucket,
          region:
            cloud.provider === 's3' ? cloud.region || 'us-east-1' : undefined,
          accountId: cloud.provider === 'r2' ? cloud.accountId : undefined,
          keyPrefix: cloud.keyPrefix || 'none',
        });
      } catch (error) {
        return res.status(500).json({
          success: false,
          error: (error as Error).message || 'Cloud connection failed',
        });
      }
    }
  );

  // Cloud compare job storage
  interface CloudCompareJob {
    id: string;
    status:
      | 'scanning_local'
      | 'scanning_cloud'
      | 'comparing'
      | 'completed'
      | 'error';
    phase: string;
    localFilesScanned: number;
    localFilesTotal: number;
    cloudObjectsScanned: number;
    progress: number;
    result?: {
      provider: string;
      summary: {
        localTotal: number;
        cloudTotal: number;
        synced: number;
        localOnly: number;
        cloudOnly: number;
        localOnlySizeMB: string;
      };
      localOnly: Array<{ key: string; size: number }>;
      cloudOnly: string[];
      hasMore: boolean;
    };
    error?: string;
  }

  const cloudCompareJobs = new Map<string, CloudCompareJob>();

  // Start cloud compare job
  router.post('/api/cloud/compare', async (_req, res) => {
    try {
      const cloud = state.currentConfig?.cloudUpload;
      if (!cloud || cloud.provider === 'none') {
        return res.status(400).json({
          success: false,
          error: 'Cloud upload is not enabled',
        });
      }

      if (!state.cloudClient || !ListObjectsV2Command) {
        try {
          const awsS3 = await import('@aws-sdk/client-s3');
          ListObjectsV2Command = awsS3.ListObjectsV2Command;
        } catch {
          return res.status(503).json({
            success: false,
            error: 'Cloud client not available',
          });
        }
      }

      const label = cloud.provider.toUpperCase();
      const jobId = `cloud-compare-${Date.now()}`;
      const job: CloudCompareJob = {
        id: jobId,
        status: 'scanning_local',
        phase: 'Scanning local files...',
        localFilesScanned: 0,
        localFilesTotal: 0,
        cloudObjectsScanned: 0,
        progress: 0,
      };

      cloudCompareJobs.set(jobId, job);

      (async () => {
        try {
          const dataDir = state.getDataDirPath();

          job.phase = 'Discovering local files...';
          // Only scan hive-partitioned files (tier=X/context=Y/path=Z/year=YYYY/day=DDD/)
          const excludedDirs = [
            '/processed/',
            '/repaired/',
            '/failed/',
            '/quarantine/',
          ];
          const allLocalFiles = await glob(
            path.join(dataDir, 'tier=*', '**', '*.parquet')
          );
          const localFiles = allLocalFiles.filter(
            f => !excludedDirs.some(dir => f.includes(dir))
          );
          job.localFilesTotal = localFiles.length;

          const localKeys = new Map<string, { path: string; size: number }>();

          for (let i = 0; i < localFiles.length; i++) {
            const filePath = localFiles[i];
            const relativePath = path.relative(dataDir, filePath);
            let cloudKey = relativePath;
            if (cloud.keyPrefix) {
              const prefix = cloud.keyPrefix.endsWith('/')
                ? cloud.keyPrefix
                : `${cloud.keyPrefix}/`;
              cloudKey = `${prefix}${relativePath}`;
            }
            const stats = await fs.stat(filePath);
            localKeys.set(cloudKey, { path: filePath, size: stats.size });

            job.localFilesScanned = i + 1;
            job.progress = Math.round(((i + 1) / localFiles.length) * 40);
            job.phase = `Scanning local files: ${i + 1}/${localFiles.length}`;
          }

          job.status = 'scanning_cloud';
          job.phase = `Listing ${label} objects...`;
          const cloudKeys = new Set<string>();
          let continuationToken: string | undefined;
          let batches = 0;

          do {
            const listCommand = new ListObjectsV2Command({
              Bucket: cloud.bucket,
              Prefix: cloud.keyPrefix || undefined,
              ContinuationToken: continuationToken,
            });

            const response = await state.cloudClient.send(listCommand);
            batches++;

            if (response.Contents) {
              for (const obj of response.Contents) {
                if (obj.Key?.endsWith('.parquet')) {
                  cloudKeys.add(obj.Key);
                }
              }
            }

            job.cloudObjectsScanned = cloudKeys.size;
            job.progress = 40 + Math.min(batches * 5, 40);
            job.phase = `Listing ${label} objects: ${cloudKeys.size} found...`;

            continuationToken = response.IsTruncated
              ? response.NextContinuationToken
              : undefined;
          } while (continuationToken);

          job.status = 'comparing';
          job.phase = 'Comparing files...';
          job.progress = 85;

          const localOnly: Array<{ key: string; size: number }> = [];
          const cloudOnly: string[] = [];
          const synced: string[] = [];

          for (const [key, info] of localKeys) {
            if (cloudKeys.has(key)) {
              synced.push(key);
            } else {
              localOnly.push({ key, size: info.size });
            }
          }

          for (const key of cloudKeys) {
            if (!localKeys.has(key)) {
              cloudOnly.push(key);
            }
          }

          const totalLocalSize = localOnly.reduce((sum, f) => sum + f.size, 0);

          job.status = 'completed';
          job.phase = 'Complete';
          job.progress = 100;
          job.result = {
            provider: cloud.provider,
            summary: {
              localTotal: localKeys.size,
              cloudTotal: cloudKeys.size,
              synced: synced.length,
              localOnly: localOnly.length,
              cloudOnly: cloudOnly.length,
              localOnlySizeMB: (totalLocalSize / 1024 / 1024).toFixed(2),
            },
            localOnly: localOnly.slice(0, 100),
            cloudOnly: cloudOnly.slice(0, 100),
            hasMore: localOnly.length > 100 || cloudOnly.length > 100,
          };

          setTimeout(() => cloudCompareJobs.delete(jobId), 5 * 60 * 1000);
        } catch (error) {
          job.status = 'error';
          job.error = (error as Error).message;
        }
      })();

      return res.json({ success: true, jobId });
    } catch (error) {
      return res.status(500).json({
        success: false,
        error: (error as Error).message,
      });
    }
  });

  // Get cloud compare job status
  router.get('/api/cloud/compare/:jobId', async (req, res) => {
    const job = cloudCompareJobs.get(req.params.jobId);
    if (!job) {
      return res.status(404).json({ success: false, error: 'Job not found' });
    }
    return res.json({ success: true, ...job });
  });

  // Cloud sync job storage
  interface CloudSyncJob {
    id: string;
    status: 'preparing' | 'uploading' | 'completed' | 'error';
    phase: string;
    filesTotal: number;
    filesUploaded: number;
    filesFailed: number;
    currentFile: string;
    progress: number;
    errors: string[];
  }

  const cloudSyncJobs = new Map<string, CloudSyncJob>();

  // Start cloud sync job
  router.post('/api/cloud/sync', async (req, res) => {
    try {
      const cloud = state.currentConfig?.cloudUpload;
      if (!cloud || cloud.provider === 'none') {
        return res.status(400).json({
          success: false,
          error: 'Cloud upload is not enabled',
        });
      }

      if (!state.cloudClient) {
        return res.status(503).json({
          success: false,
          error: 'Cloud client not initialized',
        });
      }

      let PutObjectCommand: any;
      try {
        const awsS3 = await import('@aws-sdk/client-s3');
        PutObjectCommand = awsS3.PutObjectCommand;
        if (!ListObjectsV2Command) {
          ListObjectsV2Command = awsS3.ListObjectsV2Command;
        }
      } catch {
        return res.status(503).json({
          success: false,
          error: 'S3 SDK not available',
        });
      }

      const label = cloud.provider.toUpperCase();
      const jobId = `cloud-sync-${Date.now()}`;
      const job: CloudSyncJob = {
        id: jobId,
        status: 'preparing',
        phase: 'Finding files to sync...',
        filesTotal: 0,
        filesUploaded: 0,
        filesFailed: 0,
        currentFile: '',
        progress: 0,
        errors: [],
      };

      cloudSyncJobs.set(jobId, job);

      const dataDir = state.getDataDirPath();
      const { keys } = req.body as { keys?: string[] };

      (async () => {
        try {
          const filesToSync: Array<{ key: string; localPath: string }> = [];

          if (keys && keys.length > 0) {
            for (const key of keys) {
              const relativePath = cloud.keyPrefix
                ? key.replace(new RegExp(`^${cloud.keyPrefix}/?`), '')
                : key;
              const localPath = path.join(dataDir, relativePath);
              if (await fs.pathExists(localPath)) {
                filesToSync.push({ key, localPath });
              }
            }
          } else {
            job.phase = 'Scanning local files...';
            // Only sync hive-partitioned files (tier=X/context=Y/path=Z/year=YYYY/day=DDD/)
            const excludedDirs = [
              '/processed/',
              '/repaired/',
              '/failed/',
              '/quarantine/',
            ];
            const allLocalFiles = await glob(
              path.join(dataDir, 'tier=*', '**', '*.parquet')
            );
            const localFiles = allLocalFiles.filter(
              f => !excludedDirs.some(dir => f.includes(dir))
            );

            job.phase = `Listing ${label} objects...`;
            job.progress = 10;
            const cloudKeys = new Set<string>();
            let continuationToken: string | undefined;

            do {
              const listCommand = new ListObjectsV2Command({
                Bucket: cloud.bucket,
                Prefix: cloud.keyPrefix || undefined,
                ContinuationToken: continuationToken,
              });

              const response = await state.cloudClient.send(listCommand);

              if (response.Contents) {
                for (const obj of response.Contents) {
                  if (obj.Key) cloudKeys.add(obj.Key);
                }
              }

              continuationToken = response.IsTruncated
                ? response.NextContinuationToken
                : undefined;
            } while (continuationToken);

            job.phase = 'Comparing files...';
            job.progress = 20;

            for (const localPath of localFiles) {
              const relativePath = path.relative(dataDir, localPath);
              let cloudKey = relativePath;
              if (cloud.keyPrefix) {
                const prefix = cloud.keyPrefix.endsWith('/')
                  ? cloud.keyPrefix
                  : `${cloud.keyPrefix}/`;
                cloudKey = `${prefix}${relativePath}`;
              }

              if (!cloudKeys.has(cloudKey)) {
                filesToSync.push({ key: cloudKey, localPath });
              }
            }
          }

          job.filesTotal = filesToSync.length;
          job.status = 'uploading';
          job.progress = 25;

          if (filesToSync.length === 0) {
            job.status = 'completed';
            job.phase = 'No files to sync';
            job.progress = 100;
            setTimeout(() => cloudSyncJobs.delete(jobId), 5 * 60 * 1000);
            return;
          }

          for (let i = 0; i < filesToSync.length; i++) {
            const { key, localPath } = filesToSync[i];
            job.currentFile = path.basename(localPath);
            job.phase = `Uploading ${i + 1}/${filesToSync.length}: ${job.currentFile}`;
            job.progress = 25 + Math.round(((i + 1) / filesToSync.length) * 75);

            try {
              const fileContent = await fs.readFile(localPath);
              const command = new PutObjectCommand({
                Bucket: cloud.bucket,
                Key: key,
                Body: fileContent,
                ContentType: 'application/octet-stream',
              });

              await state.cloudClient.send(command);
              job.filesUploaded++;
              app.debug(`Synced to ${label}: ${key}`);
            } catch (err) {
              job.filesFailed++;
              job.errors.push(`${key}: ${(err as Error).message}`);
            }
          }

          job.status = 'completed';
          job.phase = 'Complete';
          job.progress = 100;
          job.currentFile = '';

          setTimeout(() => cloudSyncJobs.delete(jobId), 5 * 60 * 1000);
        } catch (error) {
          job.status = 'error';
          job.phase = (error as Error).message;
        }
      })();

      return res.json({ success: true, jobId });
    } catch (error) {
      return res.status(500).json({
        success: false,
        error: (error as Error).message,
      });
    }
  });

  // Get cloud sync job status
  router.get('/api/cloud/sync/:jobId', async (req, res) => {
    const job = cloudSyncJobs.get(req.params.jobId);
    if (!job) {
      return res.status(404).json({ success: false, error: 'Job not found' });
    }
    return res.json({ success: true, ...job });
  });

  // Web App Path Configuration API Routes (manages separate config file)

  // Get current path configurations
  router.get(
    '/api/config/paths',
    (_: TypedRequest, res: TypedResponse<ConfigApiResponse>) => {
      try {
        const webAppConfig = loadWebAppConfig(app);
        return res.json({
          success: true,
          paths: webAppConfig.paths,
        });
      } catch (error) {
        return res.status(500).json({
          success: false,
          error: (error as Error).message,
        });
      }
    }
  );

  // Add new path configuration
  router.post(
    '/api/config/paths',
    (req: TypedRequest<PathConfigRequest>, res: TypedResponse): void => {
      try {
        const newPath = req.body;

        // Validate required fields
        if (!newPath.path) {
          res.status(400).json({
            success: false,
            error: 'Path is required',
          });
          return;
        }

        // Load current configuration
        const webAppConfig = loadWebAppConfig(app);
        const currentPaths = webAppConfig.paths;
        const currentCommands = webAppConfig.commands;

        // Add to current paths
        currentPaths.push(newPath);

        // Save to web app configuration
        saveWebAppConfig(currentPaths, currentCommands, app);

        // Update subscriptions
        if (state.currentConfig) {
          updateDataSubscriptions(
            currentPaths,
            state,
            state.currentConfig,
            app
          );
        }

        res.json({
          success: true,
          message: 'Path configuration added successfully',
          path: newPath,
        });
      } catch (error) {
        res.status(500).json({
          success: false,
          error: (error as Error).message,
        });
      }
    }
  );

  // Update existing path configuration
  router.put(
    '/api/config/paths/:index',
    (req: TypedRequest<PathConfigRequest>, res: TypedResponse): void => {
      try {
        const index = parseInt(req.params.index);
        const updatedPath = req.body;

        // Load current configuration
        const webAppConfig = loadWebAppConfig(app);
        const currentPaths = webAppConfig.paths;
        const currentCommands = webAppConfig.commands;

        if (index < 0 || index >= currentPaths.length) {
          res.status(404).json({
            success: false,
            error: 'Path configuration not found',
          });
          return;
        }

        // Validate required fields
        if (!updatedPath.path) {
          res.status(400).json({
            success: false,
            error: 'Path is required',
          });
          return;
        }

        // Update the path configuration
        currentPaths[index] = updatedPath;

        // Save to web app configuration
        saveWebAppConfig(currentPaths, currentCommands, app);

        // Update subscriptions
        if (state.currentConfig) {
          updateDataSubscriptions(
            currentPaths,
            state,
            state.currentConfig,
            app
          );
        }

        res.json({
          success: true,
          message: 'Path configuration updated successfully',
          path: updatedPath,
        });
      } catch (error) {
        res.status(500).json({
          success: false,
          error: (error as Error).message,
        });
      }
    }
  );

  // Remove path configuration
  router.delete(
    '/api/config/paths/:index',
    (req: TypedRequest, res: TypedResponse): void => {
      try {
        const index = parseInt(req.params.index);

        // Load current configuration
        const webAppConfig = loadWebAppConfig(app);
        const currentPaths = webAppConfig.paths;
        const currentCommands = webAppConfig.commands;

        if (index < 0 || index >= currentPaths.length) {
          res.status(404).json({
            success: false,
            error: 'Path configuration not found',
          });
          return;
        }

        // Get the path being removed for response
        const removedPath = currentPaths[index];

        // Remove from current paths
        currentPaths.splice(index, 1);

        // Save to web app configuration
        saveWebAppConfig(currentPaths, currentCommands, app);

        // Update subscriptions
        if (state.currentConfig) {
          updateDataSubscriptions(
            currentPaths,
            state,
            state.currentConfig,
            app
          );
        }

        res.json({
          success: true,
          message: 'Path configuration removed successfully',
          removedPath: removedPath,
        });
      } catch (error) {
        res.status(500).json({
          success: false,
          error: (error as Error).message,
        });
      }
    }
  );

  // Command Management API endpoints

  // Get all registered commands
  router.get(
    '/api/commands',
    (_: TypedRequest, res: TypedResponse<CommandApiResponse>) => {
      try {
        const commands = getCurrentCommands();
        return res.json({
          success: true,
          commands: commands,
          count: commands.length,
        });
      } catch (error) {
        app.error(`Error retrieving commands: ${error}`);
        return res.status(500).json({
          success: false,
          error: 'Failed to retrieve commands',
        });
      }
    }
  );

  // ===========================================
  // HOME PORT CONFIGURATION ENDPOINTS
  // ===========================================

  // Get home port configuration
  router.get('/api/config/homeport', (_req, res) => {
    try {
      if (!state.currentConfig) {
        return res.status(500).json({
          success: false,
          error: 'Plugin configuration not available',
        });
      }

      return res.json({
        success: true,
        latitude: state.currentConfig.homePortLatitude || null,
        longitude: state.currentConfig.homePortLongitude || null,
      });
    } catch (error) {
      return res.status(500).json({
        success: false,
        error: (error as Error).message,
      });
    }
  });

  // Update home port configuration
  router.put('/api/config/homeport', (req, res): void => {
    try {
      const { latitude, longitude } = req.body;

      if (!state.currentConfig) {
        res.status(500).json({
          success: false,
          error: 'Plugin configuration not available',
        });
        return;
      }

      // Validate latitude and longitude
      if (typeof latitude !== 'number' || typeof longitude !== 'number') {
        res.status(400).json({
          success: false,
          error: 'Latitude and longitude must be numbers',
        });
        return;
      }

      if (latitude < -90 || latitude > 90) {
        res.status(400).json({
          success: false,
          error: 'Latitude must be between -90 and 90',
        });
        return;
      }

      if (longitude < -180 || longitude > 180) {
        res.status(400).json({
          success: false,
          error: 'Longitude must be between -180 and 180',
        });
        return;
      }

      // Update the config
      state.currentConfig.homePortLatitude = latitude;
      state.currentConfig.homePortLongitude = longitude;

      // Update the cached config in commands module so thresholds use new home port
      updatePluginConfig(state.currentConfig);

      // Save to plugin options
      app.savePluginOptions(state.currentConfig, (err?: unknown) => {
        if (err) {
          app.error(`Failed to save home port: ${err}`);
          res.status(500).json({
            success: false,
            error: 'Failed to save home port configuration',
          });
          return;
        }

        app.debug(`✅ Home port updated: ${latitude}, ${longitude}`);
        res.json({
          success: true,
          latitude,
          longitude,
        });
      });
    } catch (error) {
      res.status(500).json({
        success: false,
        error: (error as Error).message,
      });
    }
  });

  // Get current vessel position
  router.get('/api/position/current', (_req, res) => {
    try {
      // Cast to any for compatibility with different @signalk/server-api versions
      const position = app.getSelfPath('navigation.position') as any;
      if (position && position.value) {
        return res.json({
          success: true,
          latitude: position.value.latitude,
          longitude: position.value.longitude,
        });
      }
      return res.status(404).json({
        success: false,
        error: 'Current position not available',
      });
    } catch (error) {
      return res.status(500).json({
        success: false,
        error: (error as Error).message,
      });
    }
  });

  // ===========================================
  // PATH TYPE DETECTION ENDPOINT
  // ===========================================

  // Get data type information for a SignalK path
  router.get('/api/paths/:path/type', async (req, res) => {
    try {
      const pathParam = req.params.path;
      const { detectPathType } = await import('./utils/type-detector');

      const typeInfo = await detectPathType(pathParam, app);

      return res.json({
        success: true,
        ...typeInfo,
      });
    } catch (error) {
      return res.status(500).json({
        success: false,
        error: (error as Error).message,
      });
    }
  });

  // ===========================================
  // COMMAND MANAGEMENT ENDPOINTS
  // ===========================================

  // Register a new command
  router.post(
    '/api/commands',
    (
      req: TypedRequest<CommandRegistrationRequest>,
      res: TypedResponse<CommandApiResponse>
    ) => {
      try {
        const { command, description, keywords, defaultState, thresholds } =
          req.body;

        if (
          !command ||
          !/^[a-zA-Z0-9_]+$/.test(command) ||
          command.length === 0 ||
          command.length > 50
        ) {
          return res.status(400).json({
            success: false,
            error:
              'Invalid command name. Must be alphanumeric with underscores, 1-50 characters.',
          });
        }

        const result = registerCommand(
          command,
          description,
          keywords,
          defaultState,
          thresholds
        );

        if (result.state === 'COMPLETED') {
          // Update webapp config
          const webAppConfig = loadWebAppConfig(app);
          const currentCommands = getCurrentCommands();
          saveWebAppConfig(webAppConfig.paths, currentCommands, app);

          const commandState = getCommandState();
          const commandConfig = commandState.registeredCommands.get(command);
          return res.json({
            success: true,
            message: `Command '${command}' registered successfully`,
            command: commandConfig,
          });
        } else {
          return res.status(400).json({
            success: false,
            error: result.message || 'Failed to register command',
          });
        }
      } catch (error) {
        app.error(`Error registering command: ${error}`);
        return res.status(500).json({
          success: false,
          error: 'Internal server error',
        });
      }
    }
  );

  // Execute a command
  router.put(
    '/api/commands/:command/execute',
    (
      req: TypedRequest<CommandExecutionRequest>,
      res: TypedResponse<CommandExecutionResponse>
    ) => {
      try {
        const { command } = req.params;
        const { value } = req.body;

        if (typeof value !== 'boolean') {
          return res.status(400).json({
            success: false,
            error: 'Command value must be a boolean',
          });
        }

        const result = executeCommand(command, value);

        if (result.state === 'COMPLETED') {
          return res.json({
            success: true,
            command: command,
            value: value,
            executed: true,
            timestamp: result.timestamp,
          });
        } else {
          return res.status(400).json({
            success: false,
            error: result.message || 'Failed to execute command',
          });
        }
      } catch (error) {
        app.error(`Error executing command: ${error}`);
        return res.status(500).json({
          success: false,
          error: 'Internal server error',
        });
      }
    }
  );

  // Unregister a command
  router.delete(
    '/api/commands/:command',
    (req: TypedRequest, res: TypedResponse<CommandApiResponse>) => {
      try {
        const { command } = req.params;

        const result = unregisterCommand(command);

        if (result.state === 'COMPLETED') {
          // Update webapp config
          const webAppConfig = loadWebAppConfig(app);
          const currentCommands = getCurrentCommands();
          saveWebAppConfig(webAppConfig.paths, currentCommands, app);

          return res.json({
            success: true,
            message: `Command '${command}' unregistered successfully`,
          });
        } else {
          return res.status(404).json({
            success: false,
            error: result.message || 'Command not found',
          });
        }
      } catch (error) {
        app.error(`Error unregistering command: ${error}`);
        return res.status(500).json({
          success: false,
          error: 'Internal server error',
        });
      }
    }
  );

  // Update command (PUT)
  router.put(
    '/api/commands/:command',
    (
      req: TypedRequest<{
        description?: string;
        keywords?: string[];
        defaultState?: boolean;
        thresholds?: any[];
      }>,
      res: TypedResponse<CommandApiResponse>
    ) => {
      try {
        const { command } = req.params;
        const { description, keywords, defaultState, thresholds } = req.body;

        const result = updateCommand(
          command,
          description,
          keywords,
          defaultState,
          thresholds
        );
        if (result.state === 'COMPLETED') {
          // Update webapp config
          const webAppConfig = loadWebAppConfig(app);
          const currentCommands = getCurrentCommands();
          saveWebAppConfig(webAppConfig.paths, currentCommands, app);

          return res.json({
            success: true,
            message: result.message,
          });
        } else {
          return res.status(result.statusCode || 400).json({
            success: false,
            error: result.message,
          });
        }
      } catch (error) {
        app.error(`Error updating command: ${error}`);
        return res.status(500).json({
          success: false,
          error: 'Internal server error',
        });
      }
    }
  );

  // Manual override endpoint
  router.put(
    '/api/commands/:command/override',
    (
      req: TypedRequest<{ override: boolean; expiryMinutes?: number }>,
      res: TypedResponse<CommandApiResponse>
    ) => {
      try {
        const { command } = req.params;
        const { override, expiryMinutes } = req.body;

        const result = setManualOverride(command, override, expiryMinutes);
        if (result.success) {
          // Update webapp config
          const webAppConfig = loadWebAppConfig(app);
          const currentCommands = getCurrentCommands();
          saveWebAppConfig(webAppConfig.paths, currentCommands, app);

          return res.json({
            success: true,
            message: result.message,
          });
        } else {
          return res.status(400).json({
            success: false,
            error: result.message,
          });
        }
      } catch (error) {
        app.error(`Error setting manual override: ${error}`);
        return res.status(500).json({
          success: false,
          error: 'Internal server error',
        });
      }
    }
  );

  // Get command history
  router.get(
    '/api/commands/history',
    (_: TypedRequest, res: TypedResponse<CommandApiResponse>) => {
      try {
        // Return the last 50 history entries
        const commandHistory = getCommandHistory();
        const recentHistory = commandHistory.slice(-50);
        return res.json({
          success: true,
          data: recentHistory,
        });
      } catch (error) {
        app.error(`Error retrieving command history: ${error}`);
        return res.status(500).json({
          success: false,
          error: 'Failed to retrieve command history',
        });
      }
    }
  );

  // Get command status
  router.get(
    '/api/commands/:command/status',
    (req: TypedRequest, res: TypedResponse<CommandApiResponse>) => {
      try {
        const { command } = req.params;
        const commandState = getCommandState();
        const commandConfig = commandState.registeredCommands.get(command);

        if (!commandConfig) {
          return res.status(404).json({
            success: false,
            error: 'Command not found',
          });
        }

        // Get current value from SignalK
        // Cast to any for compatibility with different @signalk/server-api versions
        const currentValue = app.getSelfPath(`commands.${command}`) as any;

        return res.json({
          success: true,
          command: {
            ...commandConfig,
            active: currentValue === true,
          },
        });
      } catch (error) {
        app.error(`Error retrieving command status: ${error}`);
        return res.status(500).json({
          success: false,
          error: 'Failed to retrieve command status',
        });
      }
    }
  );

  // Streaming Control API endpoints - DISABLED

  // // Enable streaming at runtime
  // router.post('/api/streaming/enable', async (req: TypedRequest, res: TypedResponse) => {
  //   try {
  //     if (state.streamingService) {
  //       return res.json({
  //         success: true,
  //         message: 'Streaming service is already running',
  //         enabled: true
  //       });
  //     }

  //     // Check if streaming is enabled in config
  //     if (!state.currentConfig?.enableStreaming) {
  //       return res.status(400).json({
  //         success: false,
  //         error: 'Streaming is disabled in plugin configuration. Enable it in plugin settings first.',
  //         enabled: false
  //       });
  //     }

  //     const result = await initializeStreamingService(state, app);

  //     if (result.success) {
  //       return res.json({
  //         success: true,
  //         message: 'Streaming service enabled successfully',
  //         enabled: true
  //       });
  //     } else {
  //       return res.status(500).json({
  //         success: false,
  //         error: result.error || 'Failed to enable streaming service',
  //         enabled: false
  //       });
  //     }
  //   } catch (error) {
  //     app.error(`Error enabling streaming: ${error}`);
  //     return res.status(500).json({
  //       success: false,
  //       error: (error as Error).message,
  //       enabled: false
  //     });
  //   }
  // });

  // // Disable streaming at runtime
  // router.post('/api/streaming/disable', (req: TypedRequest, res: TypedResponse) => {