signalk-parquet/src/HistoryAPI.ts

Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.

import { Router, Request, Response } from 'express';
import {
  AggregateMethod,
  DataResult,
  FromToContextRequest,
  PathSpec,
} from './HistoryAPI-types';
import { ZonedDateTime, ZoneOffset, ZoneId } from '@js-joda/core';
import { Context, Path, Timestamp } from '@signalk/server-api';
import { ParamsDictionary } from 'express-serve-static-core';
import { ParsedQs } from 'qs';
import { DuckDBPool } from './utils/duckdb-pool';
import path from 'path';
import {
  getAvailablePathsArray,
  getAvailablePathsForTimeRange,
} from './utils/path-discovery';
import {
  getCachedPaths,
  setCachedPaths,
  getCachedContexts,
  setCachedContexts,
} from './utils/path-cache';
import {
  getAvailableContextsForTimeRange,
  getContextsInSpatialFilter,
} from './utils/context-discovery';
import { getPathComponentSchema, ComponentInfo } from './utils/schema-cache';
import { ConcurrencyLimiter } from './utils/concurrency-limiter';
import { CONCURRENCY } from './config/cache-defaults';
import { SQLiteBufferInterface } from './types';
import { HivePathBuilder, AggregationTier } from './utils/hive-path-builder';
import {
  AutoDiscoveryService,
  AutoDiscoveryResult,
} from './services/auto-discovery';
import {
  SpatialFilter,
  parseSpatialParams,
  buildSpatialSqlClause,
  isPositionPath,
} from './utils/spatial-queries';
import { calculateDistance } from './utils/geo-calculator';
import {
  buildBufferScalarSubquery,
  buildBufferObjectSubquery,
} from './utils/buffer-sql-builder';
import { stageBufferTable } from './utils/buffer-staging';
import {
  parseDurationToMillis,
  parseResolutionToMillis,
  InvalidResolutionError,
} from './utils/duration-parser';
import { isAngularPath } from './utils/angular-paths';
import { PathRetentionRule, RetentionRuleSet } from './utils/retention-rules';
import {
  parsePathFilters,
  buildParquetFilterClause,
  availableFilterColumns,
  filterEcho,
  FILTER_DELIMITERS,
} from './utils/path-filters';

// Allowed characters in the `paths` query parameter: path/aggregate tokens
// (alphanumerics, `.,:_`), the inline-filter delimiters from PATH_FILTER_DEFS,
// and `-`. Delimiters are escaped for use inside a character class; `-` is kept
// last so it stays a literal.
const PATHS_SANITIZER = new RegExp(
  `[^0-9a-z.,:_${FILTER_DELIMITERS.replace(/[\\\]^-]/g, '\\$&')}-]`,
  'gi'
);

export function registerHistoryApiRoute(
  router: Pick<Router, 'get'>,
  selfId: string,
  dataDir: string,
  debug: (k: string) => void,
  app: any,
  sqliteBuffer?: SQLiteBufferInterface,
  autoDiscoveryService?: AutoDiscoveryService,
  s3Config?: S3QueryConfig,
  pathRetentionOverrides?: PathRetentionRule[]
) {
  const historyApi = new HistoryAPI(
    selfId,
    dataDir,
    sqliteBuffer,
    autoDiscoveryService,
    s3Config,
    pathRetentionOverrides
  );
  // Handler for values endpoint
  const handleValues = (req: Request, res: Response) => {
    const { from, to, context, spatialFilter } = getRequestParams(
      req as FromToContextRequest,
      selfId
    );
    historyApi.getValues(
      context,
      from,
      to,
      spatialFilter,
      app,
      debug,
      req,
      res
    );
  };

  // V1 route: Direct routes with all signalk-parquet extensions
  // (spatial filtering, timezone conversion, moving averages, etc.)
  router.get('/signalk/v1/history/values', handleValues);
  // Handler for contexts endpoint
  const handleContexts = async (req: Request, res: Response) => {
    try {
      // Check if time range parameters are provided
      const hasTimeParams =
        req.query.duration || req.query.from || req.query.to;

      if (hasTimeParams) {
        // Time-range-aware: return only contexts with data in the specified time range
        const { from, to } = getRequestParams(
          req as FromToContextRequest,
          selfId
        );

        // Check cache first
        let contexts = getCachedContexts(from, to);

        if (!contexts) {
          // Cache miss - query the parquet files
          contexts = await getAvailableContextsForTimeRange(dataDir, from, to);
          // Cache the result
          setCachedContexts(from, to, contexts);
        }

        res.json(contexts);
      } else {
        // No time range specified: return only self context (legacy behavior)
        res.json([`vessels.${selfId}`] as Context[]);
      }
    } catch (error) {
      debug(`Error in /contexts: ${error}`);
      res.status(500).json({ error: (error as Error).message });
    }
  };

  // V1 route: Direct routes with extensions (time-range-aware context discovery)
  router.get('/signalk/v1/history/contexts', handleContexts);
  // Note: V2 routes (/signalk/v2/api/history/*) are handled by the registered
  // HistoryApi provider in history-provider.ts via app.registerHistoryApiProvider()
  // Handler for paths endpoint
  const handlePaths = async (req: Request, res: Response) => {
    try {
      // Check if time range parameters are provided
      const hasTimeParams =
        req.query.duration || req.query.from || req.query.to;

      if (hasTimeParams) {
        // Time-range-aware: return only paths with data in the specified time range
        const { from, to, context } = getRequestParams(
          req as FromToContextRequest,
          selfId
        );

        // Check cache first
        let paths = getCachedPaths(context, from, to);

        if (!paths) {
          // Cache miss - query the parquet files
          paths = await getAvailablePathsForTimeRange(
            dataDir,
            context,
            from,
            to
          );
          // Cache the result
          setCachedPaths(context, from, to, paths);
        }

        res.json(paths);
      } else {
        // No time range specified: return all available paths (legacy behavior)
        const context = req.query.context
          ? getContext(req.query.context as string, selfId)
          : undefined;
        const paths = getAvailablePathsArray(dataDir, app, context);
        res.json(paths);
      }
    } catch (error) {
      debug(`Error in /paths: ${error}`);
      res.status(500).json({ error: (error as Error).message });
    }
  };

  // V1 route: Direct routes with extensions (time-range-aware path discovery)
  router.get('/signalk/v1/history/paths', handlePaths);

  // Also register as plugin-style routes for testing
  router.get('/api/history/values', (req: Request, res: Response) => {
    const { from, to, context, spatialFilter } = getRequestParams(
      req as FromToContextRequest,
      selfId
    );
    historyApi.getValues(
      context,
      from,
      to,
      spatialFilter,
      app,
      debug,
      req,
      res
    );
  });
  router.get('/api/history/contexts', async (req: Request, res: Response) => {
    try {
      // Check if time range parameters are provided
      const hasTimeParams =
        req.query.duration || req.query.from || req.query.to;

      if (hasTimeParams) {
        // Time-range-aware: return only contexts with data in the specified time range
        const { from, to } = getRequestParams(
          req as FromToContextRequest,
          selfId
        );

        // Check cache first
        let contexts = getCachedContexts(from, to);

        if (!contexts) {
          // Cache miss - query the parquet files
          contexts = await getAvailableContextsForTimeRange(dataDir, from, to);
          // Cache the result
          setCachedContexts(from, to, contexts);
        }

        res.json(contexts);
      } else {
        // No time range specified: return only self context (legacy behavior)
        res.json([`vessels.${selfId}`] as Context[]);
      }
    } catch (error) {
      debug(`Error in /api/history/contexts: ${error}`);
      res.status(500).json({ error: (error as Error).message });
    }
  });
  router.get(
    '/api/history/contexts/spatial',
    async (req: Request, res: Response) => {
      try {
        const { from, to } = getRequestParams(
          req as FromToContextRequest,
          selfId
        );

        const spatialFilter = parseSpatialParams(
          req.query.bbox as string | undefined,
          req.query.radius as string | undefined
        );

        if (!spatialFilter) {
          res.status(400).json({
            error:
              'bbox (west,south,east,north) or radius (lon,lat,meters) is required',
          });
          return;
        }

        const contexts = await getContextsInSpatialFilter(
          dataDir,
          from,
          to,
          spatialFilter
        );
        res.json(contexts);
      } catch (error) {
        debug(`Error in /api/history/contexts/spatial: ${error}`);
        res.status(500).json({ error: (error as Error).message });
      }
    }
  );
  router.get('/api/history/paths', async (req: Request, res: Response) => {
    try {
      // Check if time range parameters are provided
      const hasTimeParams =
        req.query.duration || req.query.from || req.query.to;

      if (hasTimeParams) {
        // Time-range-aware: return only paths with data in the specified time range
        const { from, to, context } = getRequestParams(
          req as FromToContextRequest,
          selfId
        );

        // Check cache first
        let paths = getCachedPaths(context, from, to);

        if (!paths) {
          // Cache miss - query the parquet files
          paths = await getAvailablePathsForTimeRange(
            dataDir,
            context,
            from,
            to
          );
          // Cache the result
          setCachedPaths(context, from, to, paths);
        }

        res.json(paths);
      } else {
        // No time range specified: return all available paths (legacy behavior)
        const context = req.query.context
          ? getContext(req.query.context as string, selfId)
          : undefined;
        const paths = getAvailablePathsArray(dataDir, app, context);
        res.json(paths);
      }
    } catch (error) {
      debug(`Error in /api/history/paths: ${error}`);
      res.status(500).json({ error: (error as Error).message });
    }
  });
}

const getRequestParams = ({ query }: FromToContextRequest, selfId: string) => {
  try {
    let from: ZonedDateTime;
    let to: ZonedDateTime;

    // ============================================================================
    // STANDARD SIGNALK TIME RANGE PATTERNS
    // ============================================================================
    // Pattern 1: duration only → query back from now
    if (query.duration && !query.from && !query.to) {
      const durationMs = parseDuration(query.duration);
      to = ZonedDateTime.now(ZoneOffset.UTC);
      from = to.minusNanos(durationMs * 1000000);
    }
    // Pattern 2: from + duration → query forward from start
    else if (query.from && query.duration && !query.to) {
      from = parseDateTime(query.from);
      const durationMs = parseDuration(query.duration);
      to = from.plusNanos(durationMs * 1000000);
    }
    // Pattern 3: to + duration → query backward to end
    else if (query.to && query.duration && !query.from) {
      to = parseDateTime(query.to);
      const durationMs = parseDuration(query.duration);
      from = to.minusNanos(durationMs * 1000000);
    }
    // Pattern 4: from only → from start to now
    else if (query.from && !query.duration && !query.to) {
      from = parseDateTime(query.from);
      to = ZonedDateTime.now(ZoneOffset.UTC);
    }
    // Pattern 5: from + to → specific range
    else if (query.from && query.to && !query.duration) {
      from = parseDateTime(query.from);
      to = parseDateTime(query.to);
    } else {
      throw new Error(
        'Invalid time range parameters. Use one of the following patterns:\n' +
          '  1. ?duration=1h (query back from now)\n' +
          '  2. ?from=2025-08-01T00:00:00Z&duration=1h (query forward)\n' +
          '  3. ?to=2025-08-01T12:00:00Z&duration=1h (query backward)\n' +
          '  4. ?from=2025-08-01T00:00:00Z (from start to now)\n' +
          '  5. ?from=2025-08-01T00:00:00Z&to=2025-08-02T00:00:00Z (specific range)'
      );
    }

    const context: Context = getContext(query.context, selfId);
    const spatialFilter = parseSpatialParams(query.bbox, query.radius);
    return { from, to, context, spatialFilter };
  } catch (e: unknown) {
    console.error('Full error details:', e);
    throw new Error(
      `Error extracting query parameters from ${JSON.stringify(query)}: ${e instanceof Error ? e.stack : e}`
    );
  }
};

// Parse duration string (supports ISO 8601, integer seconds, shorthand)
function parseDuration(duration: string | undefined): number {
  if (!duration) {
    throw new Error('Duration parameter is required');
  }
  return parseDurationToMillis(duration);
}

// Check if datetime string has timezone information
function hasTimezoneInfo(dateTimeStr: string): boolean {
  // Check for 'Z' at the end, or '+'/'-' followed by timezone offset pattern
  return (
    dateTimeStr.endsWith('Z') ||
    /[+-]\d{2}:?\d{2}$/.test(dateTimeStr) ||
    /[+-]\d{4}$/.test(dateTimeStr)
  );
}

// Parse datetime string per ISO 8601:
// - Bare timestamps (no Z, no offset) → local time, converted to UTC
// - Timestamps with Z or offset → parsed as-is
function parseDateTime(dateTimeStr: string): ZonedDateTime {
  // Normalize the datetime string to include seconds if missing
  let normalizedStr = dateTimeStr;
  if (dateTimeStr.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$/)) {
    // Add seconds if only HH:MM is provided
    normalizedStr = dateTimeStr + ':00';
  }

  if (hasTimezoneInfo(normalizedStr)) {
    // Has timezone info (Z or offset), parse as-is and convert to UTC
    return ZonedDateTime.parse(normalizedStr).withZoneSameInstant(
      ZoneOffset.UTC
    );
  } else {
    // No timezone info — per ISO 8601, treat as local time and convert to UTC
    try {
      // JavaScript Date constructor treats ISO strings without timezone as local time
      const localDate = new Date(normalizedStr);
      if (isNaN(localDate.getTime())) {
        throw new Error('Invalid date');
      }

      // Convert to UTC ISO string and parse with ZonedDateTime
      const utcIsoString = localDate.toISOString();
      return ZonedDateTime.parse(utcIsoString);
    } catch (e) {
      throw new Error(
        `Unable to parse datetime '${dateTimeStr}': ${e}. Use format like '2025-08-13T08:00:00' or '2025-08-13T08:00:00Z'`
      );
    }
  }
}

function getContext(
  contextFromQuery: string | undefined,
  selfId: string
): Context {
  if (
    !contextFromQuery ||
    contextFromQuery === 'vessels.self' ||
    contextFromQuery === 'self'
  ) {
    return `vessels.${selfId}` as Context;
  }
  return contextFromQuery.replace(/ /gi, '') as Context;
}

type QuerySource = 'local' | 's3' | 'hybrid' | 'auto';

export interface S3QueryConfig {
  enabled: boolean;
  bucket: string;
  keyPrefix: string;
  region: string;
}

/**
 * Stream a DataResult to the response as a single JSON document,
 * progressively, instead of buffering it via res.json(). Output is
 * byte-equivalent to the prior res.json(result) path: same field
 * order (context, range, values, data, units?, meta?), same UTC->
 * local timestamp conversion on the range and on each row[0].
 *
 * Wrapper construction: build the wrapper object with a `data: []`
 * placeholder, JSON.stringify it once, then split at that placeholder.
 * The two halves frame the row-by-row writes. This avoids hand-rolling
 * `,` / `{}` / `[]` framing tokens.
 *   Input wrapper -> '{...,"data":[],"units":{...}}'
 *   prefix -> '{...,"data":['
 *   suffix -> ']' + ',"units":{...}}'
 *
 * Backpressure: if res.write returns false the loop awaits 'drain'
 * before continuing, so a slow client cannot grow Node's internal
 * write buffer beyond the high-water mark. The wait races against
 * 'close' so a client that disconnects mid-flight does not hang the
 * stream forever.
 *
 * Cancellation: a 'close' listener on res sets a flag so the loop
 * breaks early on client disconnect. The DuckDB result has already
 * been materialised before this function is called, so cancellation
 * is purely about cutting the write loop short.
 *
 * Logging: on success or post-headers failure, emits one structured
 * line via the optional `log` callback with row count, byte count
 * and wall-clock duration. Operators chasing "endpoint feels slow"
 * have a single line to grep for.
 */
async function streamDataResult(
  res: Response,
  result: DataResult,
  log?: (msg: string) => void
): Promise<void> {
  const startMs = Date.now();
  let rowsWritten = 0;
  let bytesWritten = 0;
  let clientGone = false;
  const onClose = (): void => {
    clientGone = true;
  };
  // A mid-stream socket failure (ECONNRESET / EPIPE) emits 'error' on res.
  // With no listener Node rethrows it as an uncaughtException and takes down
  // the whole server process. Treat it like a disconnect: flip the flag so the
  // write loop unwinds instead of writing into a broken socket.
  const onError = (): void => {
    clientGone = true;
  };
  res.on('close', onClose);
  res.on('error', onError);

  try {
    res.setHeader('Content-Type', 'application/json; charset=utf-8');
    // Streaming responses must not be cached by an intermediary.
    // X-Accel-Buffering is nginx-specific (no-op elsewhere) and tells
    // nginx not to buffer the response body, so chunks reach the
    // client as we write them rather than after res.end().
    res.setHeader('Cache-Control', 'no-store');
    res.setHeader('X-Accel-Buffering', 'no');

    const localRange = {
      from: utcToLocalTimestamp(result.range.from),
      to: utcToLocalTimestamp(result.range.to),
    };

    // Build the wrapper with `data: []` placeholder so JSON.stringify
    // produces the framing for us. Optional fields (units, meta) come
    // after data per DataResult declaration order; insertion order in
    // a plain object is preserved by JSON.stringify.
    const wrapper: Record<string, unknown> = {
      context: result.context,
      range: localRange,
      values: result.values,
      data: [],
    };
    if (result.units) wrapper.units = result.units;
    if (result.meta) wrapper.meta = result.meta;

    const wrapperJson = JSON.stringify(wrapper);
    const marker = '"data":[]';
    const markerIdx = wrapperJson.indexOf(marker);
    if (markerIdx === -1) {
      // The wrapper is constructed locally with data:[] as a literal
      // field; this branch only fires if a future JSON.stringify ever
      // omits empty arrays. Cheaper to assert than to debug a silent
      // wire-format break.
      throw new Error('streamDataResult: wrapper marker not found');
    }
    const prefix = wrapperJson.slice(0, markerIdx) + '"data":[';
    const suffix = ']' + wrapperJson.slice(markerIdx + marker.length);

    const writeOrAwait = async (chunk: string): Promise<void> => {
      bytesWritten += Buffer.byteLength(chunk, 'utf8');
      if (!res.write(chunk)) {
        // Wait for drain (kernel buffer flushed), close (client gone), or
        // error (socket broke). Whichever fires first wins; all three unhook
        // the others so we don't leak handlers across the loop, and we never
        // await a 'drain' that a dead socket will never emit.
        await new Promise<void>(resolve => {
          let settled = false;
          const finish = (): void => {
            if (settled) return;
            settled = true;
            res.off('drain', finish);
            res.off('close', finish);
            res.off('error', finish);
            resolve();
          };
          res.once('drain', finish);
          res.once('close', finish);
          res.once('error', finish);
        });
      }
    };

    await writeOrAwait(prefix);

    const data = result.data;
    for (let i = 0; i < data.length; i++) {
      if (clientGone) break;
      const row = data[i];
      // Allocate a converted row instead of mutating in place. The
      // input is logically owned by the caller, and a future second
      // caller (audit log, dual-format response) would silently see
      // local time where it expected UTC.
      const converted = row.slice() as typeof row;
      converted[0] = utcToLocalTimestamp(row[0]);
      const rowChunk = (i > 0 ? ',' : '') + JSON.stringify(converted);
      await writeOrAwait(rowChunk);
      rowsWritten++;
    }

    if (clientGone) {
      log?.(
        `streamDataResult: client disconnected after ${rowsWritten}/${data.length} rows, ${bytesWritten} bytes, ${Date.now() - startMs}ms`
      );
      return;
    }

    await writeOrAwait(suffix);
    res.end();
    log?.(
      `streamDataResult: rows=${rowsWritten} bytes=${bytesWritten} duration=${Date.now() - startMs}ms`
    );
  } finally {
    res.off('close', onClose);
    res.off('error', onError);
  }
}

/**
 * Buffered UTC->local conversion of a full DataResult. Used by callers
 * that pass a non-streaming response (e.g. internal mocks that only
 * implement res.json) so they receive the same locally-converted shape
 * the streaming path produces.
 */
function convertToLocalTime(result: DataResult): DataResult {
  return {
    ...result,
    range: {
      from: utcToLocalTimestamp(result.range.from),
      to: utcToLocalTimestamp(result.range.to),
    },
    data: result.data.map(row => {
      const newRow = row.slice() as typeof row;
      newRow[0] = utcToLocalTimestamp(row[0]);
      return newRow;
    }),
  };
}

/**
 * Returns true when `res` exposes the Node stream methods streamDataResult
 * relies on. The plugin's HTTP routes pass a real Express Response, but
 * historical-streaming.ts reuses getValues with a mock that only implements
 * res.json / res.status().json — those callers go through the buffered path.
 */
function canStreamResponse(res: Response): boolean {
  return (
    typeof (res as Partial<Response>).write === 'function' &&
    typeof (res as Partial<Response>).end === 'function' &&
    typeof (res as Partial<Response>).setHeader === 'function' &&
    typeof (res as Partial<Response>).on === 'function'
  );
}

/**
 * Convert a UTC timestamp string to server-local time with offset suffix.
 * e.g. "2026-03-06T19:00:00Z" → "2026-03-06T14:00:00-05:00" (EST)
 */
function utcToLocalTimestamp(utcTs: Timestamp): Timestamp {
  try {
    const zdt = ZonedDateTime.parse(utcTs).withZoneSameInstant(
      ZoneId.systemDefault()
    );
    // ZonedDateTime.toString() appends "[SYSTEM]" zone ID — strip it
    return zdt.toString().replace(/\[.*\]$/, '') as Timestamp;
  } catch {
    // If parsing fails (e.g. already has offset or unusual format), try via Date
    const d = new Date(utcTs);
    if (isNaN(d.getTime())) return utcTs;
    const offsetMin = d.getTimezoneOffset();
    const local = new Date(d.getTime() - offsetMin * 60000);
    const sign = offsetMin <= 0 ? '+' : '-';
    const absH = String(Math.floor(Math.abs(offsetMin) / 60)).padStart(2, '0');
    const absM = String(Math.abs(offsetMin) % 60).padStart(2, '0');
    return (local.toISOString().replace('Z', '') +
      sign +
      absH +
      ':' +
      absM) as Timestamp;
  }
}

// A response `values` entry: path + aggregate method, optional smoothing
// metadata, plus any echoed inline-filter fields (e.g. `sourceRef`). The index
// signature keeps it open to new filters without a type change here.
interface HistoryValueEntry {
  path: Path;
  method: AggregateMethod;
  smoothing?: string;
  window?: number;
  [field: string]: unknown;
}

export class HistoryAPI {
  private sqliteBuffer?: SQLiteBufferInterface;
  private hivePathBuilder: HivePathBuilder;
  private autoDiscoveryService?: AutoDiscoveryService;
  private s3Config?: S3QueryConfig;
  // Mirrors AggregationService's rule set so the read path can fall
  // back to tier=raw for skipAggregation paths. The aggregation
  // pipeline doesn't write 5s/60s/1h files for those, so a higher-
  // resolution query against an aggregated tier would otherwise return
  // empty.
  private retentionRules: RetentionRuleSet;

  constructor(
    private selfId: string,
    private dataDir: string,
    sqliteBuffer?: SQLiteBufferInterface,
    autoDiscoveryService?: AutoDiscoveryService,
    s3Config?: S3QueryConfig,
    pathRetentionOverrides?: PathRetentionRule[]
  ) {
    this.sqliteBuffer = sqliteBuffer;
    this.hivePathBuilder = new HivePathBuilder();
    this.autoDiscoveryService = autoDiscoveryService;
    this.s3Config = s3Config;
    this.retentionRules = new RetentionRuleSet(pathRetentionOverrides || []);
  }

  /**
   * Set S3 configuration for federated queries
   */
  setS3Config(config: S3QueryConfig | undefined): void {
    this.s3Config = config;
  }

  /**
   * Set the auto-discovery service
   */
  setAutoDiscoveryService(service: AutoDiscoveryService | undefined): void {
    this.autoDiscoveryService = service;
  }

  /**
   * Set the SQLite buffer for federated queries
   */
  setSqliteBuffer(buffer: SQLiteBufferInterface | undefined): void {
    this.sqliteBuffer = buffer;
  }

  /**
   * Auto-select the optimal tier based on requested resolution
   * Returns undefined to use raw/flat data, or a tier name for aggregated data
   *
   * Logic:
   * - resolution >= 1 hour (3600000ms) → use 1h tier
   * - resolution >= 1 minute (60000ms) → use 60s tier
   * - resolution >= 5 seconds (5000ms) → use 5s tier
   * - resolution < 5 seconds → use raw data (no tier)
   *
   * Falls back through tiers if preferred tier doesn't exist
   */
  private selectOptimalTier(
    resolutionMillis: number
  ): AggregationTier | undefined {
    // eslint-disable-next-line @typescript-eslint/no-require-imports
    const fs = require('fs');

    // Determine preferred tier based on resolution
    let preferredTiers: AggregationTier[] = [];

    if (resolutionMillis >= 3600000) {
      preferredTiers = ['1h', '60s', '5s'];
    } else if (resolutionMillis >= 60000) {
      preferredTiers = ['60s', '5s'];
    } else if (resolutionMillis >= 5000) {
      preferredTiers = ['5s'];
    } else {
      // Use raw data for sub-5-second resolution
      return undefined;
    }

    // Check which tiers exist and return the best available
    for (const tier of preferredTiers) {
      const tierPath = path.join(this.dataDir, `tier=${tier}`);
      try {
        if (fs.existsSync(tierPath)) {
          return tier;
        }
      } catch {
        // Directory doesn't exist or not accessible
      }
    }

    // No aggregated tiers available, use raw data
    return undefined;
  }

  /**
   * Get timestamps where vessel position was within the spatial filter
   * Used to correlate non-position paths with spatial filtering
   */
  private async getSpatialTimestamps(
    context: Context,
    from: ZonedDateTime,
    to: ZonedDateTime,
    timeResolutionMillis: number,
    spatialFilter: SpatialFilter,
    positionPath: string,
    _tier: AggregationTier | undefined,
    _querySource: QuerySource,
    debug: (k: string) => void
  ): Promise<Set<string>> {
    const timestamps = new Set<string>();

    // Build file path for raw position data
    const sanitizedContext = this.hivePathBuilder.sanitizeContext(context);
    const sanitizedPath = this.hivePathBuilder.sanitizePath(positionPath);
    const localFilePath = path.join(
      this.dataDir,
      'tier=raw',
      `context=${sanitizedContext}`,
      `path=${sanitizedPath}`,
      '**',
      '*.parquet'
    );

    const fromIso = from.toInstant().toString();
    const toIso = to.toInstant().toString();

    try {
      const hasBuffer = DuckDBPool.isSQLiteBufferInitialized();
      const connection = await DuckDBPool.getConnection();
      try {
        // Bucket-lookup approach: instead of scanning all raw position data,
        // bucket by time resolution, grab FIRST lat/lon per bucket, then filter by bbox/radius.
        // This reads far less data than a full scan with spatial SQL.
        const bucketExpr = `strftime(DATE_TRUNC('seconds',
          EPOCH_MS(CAST(FLOOR(EPOCH_MS(signalk_timestamp::TIMESTAMP) / ${timeResolutionMillis}) * ${timeResolutionMillis} AS BIGINT))
        ), '%Y-%m-%dT%H:%M:%SZ')`;

        // Build FROM: parquet UNION ALL buffer
        const parquetFrom = `SELECT signalk_timestamp, value_latitude, value_longitude FROM (
          SELECT * FROM read_parquet('${localFilePath}', union_by_name=true, filename=true)
          WHERE filename NOT LIKE '%/processed/%'
          AND filename NOT LIKE '%/quarantine/%'
          AND filename NOT LIKE '%/failed/%'
          AND filename NOT LIKE '%/repaired/%')`;

        let fromSource = `(${parquetFrom})`;
        if (hasBuffer && this.sqliteBuffer) {
          const bufferTableCols =
            this.sqliteBuffer.getTableColumns(positionPath);
          const stagedTable = await stageBufferTable(
            connection,
            this.sqliteBuffer,
            String(context),
            positionPath,
            fromIso,
            toIso,
            debug
          );
          const bufferSubquery = stagedTable
            ? buildBufferObjectSubquery(
                stagedTable,
                context,
                fromIso,
                toIso,
                new Map([
                  [
                    'latitude',
                    {
                      name: 'latitude',
                      columnName: 'value_latitude',
                      dataType: 'numeric' as const,
                    },
                  ],
                  [
                    'longitude',
                    {
                      name: 'longitude',
                      columnName: 'value_longitude',
                      dataType: 'numeric' as const,
                    },
                  ],
                ]),
                bufferTableCols
              )
            : null;
          if (bufferSubquery) {
            fromSource = `(${parquetFrom} UNION ALL SELECT signalk_timestamp, TRY_CAST(value_latitude AS DOUBLE) as value_latitude, TRY_CAST(value_longitude AS DOUBLE) as value_longitude FROM ${bufferSubquery})`;
          }
        }

        const query = `
          SELECT
            ${bucketExpr} as timestamp,
            FIRST(TRY_CAST(value_latitude AS DOUBLE)) as lat,
            FIRST(TRY_CAST(value_longitude AS DOUBLE)) as lon
          FROM ${fromSource}
          WHERE
            signalk_timestamp >= '${fromIso}'
            AND signalk_timestamp < '${toIso}'
            AND TRY_CAST(value_latitude AS DOUBLE) IS NOT NULL
            AND TRY_CAST(value_longitude AS DOUBLE) IS NOT NULL
          GROUP BY timestamp
          ORDER BY timestamp
        `;

        const result = await connection.runAndReadAll(query);
        const rows = result.getRowObjects() as Array<{
          timestamp: string;
          lat: number;
          lon: number;
        }>;

        // Filter buckets by spatial filter in JS
        const { bbox } = spatialFilter;
        for (const row of rows) {
          if (row.lat === null || row.lon === null) continue;

          // Lat check
          if (row.lat < bbox.south || row.lat > bbox.north) continue;

          // Lon check (handles 180° meridian crossing)
          if (bbox.west <= bbox.east) {
            if (row.lon < bbox.west || row.lon > bbox.east) continue;
          } else {
            if (row.lon < bbox.west && row.lon > bbox.east) continue;
          }

          // Precise radius check if applicable
          if (
            spatialFilter.type === 'radius' &&
            spatialFilter.centerLat !== undefined &&
            spatialFilter.centerLon !== undefined &&
            spatialFilter.radiusMeters !== undefined
          ) {
            const dist = calculateDistance(
              row.lat,
              row.lon,
              spatialFilter.centerLat,
              spatialFilter.centerLon
            );
            if (dist > spatialFilter.radiusMeters) continue;
          }

          timestamps.add(row.timestamp);
        }

        debug(
          `[Spatial Correlation] Bucketed ${rows.length} positions, ${timestamps.size} within spatial filter`
        );
      } finally {
        connection.disconnectSync();
      }
    } catch (error) {
      debug(`[Spatial Correlation] Error querying position data: ${error}`);
      // On error, return empty set (no filtering will occur)
    }

    return timestamps;
  }

  async getValues(
    context: Context,
    from: ZonedDateTime,
    to: ZonedDateTime,
    spatialFilter: SpatialFilter | null,
    app: any,
    debug: (k: string) => void,
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    req: Request<ParamsDictionary, any, any, ParsedQs, Record<string, any>>,
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    res: Response<any, Record<string, any>>
  ) {
    try {
      // Resolution now in SECONDS (breaking change from v0.7.0)
      const timeResolutionMillis = req.query.resolution
        ? parseResolutionToMillis(req.query.resolution as string)
        : ((to.toEpochSecond() - from.toEpochSecond()) / 500) * 1000;
      // Strip everything except path/aggregate characters, the filter
      // delimiters from PATH_FILTER_DEFS, and `-` (common in source
      // references). This is a coarse injection guard; values are also SQL-
      // escaped downstream. Adding a delimiter to the registry extends this
      // automatically.
      const pathExpressions = ((req.query.paths as string) || '')
        .replace(PATHS_SANITIZER, '')
        .split(',');
      const pathSpecs: PathSpec[] = pathExpressions.map(splitPathExpression);

      // Auto-select tier based on resolution (provider selects automatically)
      const tier = this.selectOptimalTier(timeResolutionMillis);
      if (tier) {
        debug(
          `Auto-selected tier=${tier} for resolution=${timeResolutionMillis}ms`
        );
      }

      // Log spatial filter if present
      if (spatialFilter) {
        debug(
          `Spatial filter: type=${spatialFilter.type}, bbox=[${spatialFilter.bbox.west},${spatialFilter.bbox.south},${spatialFilter.bbox.east},${spatialFilter.bbox.north}]`
        );
      }

      // Handle position and numeric paths together
      const positionPath = 'navigation.position';
      const allResult = pathSpecs.length
        ? await this.getNumericValues(
            context,
            from,
            to,
            timeResolutionMillis,
            pathSpecs,
            debug,
            tier,
            spatialFilter,
            'local',
            positionPath,
            app
          )
        : {
            context,
            range: {
              from: from.toString() as Timestamp,
              to: to.toString() as Timestamp,
            },
            values: [],
            data: [],
          };

      // Check for auto-discovery on paths with no data
      debug(
        `[AutoDiscovery] Checking auto-discovery: service=${!!this.autoDiscoveryService}, pathSpecs.length=${pathSpecs.length}`
      );
      if (this.autoDiscoveryService && pathSpecs.length > 0) {
        const autoConfiguredPaths: AutoDiscoveryResult[] = [];

        for (let i = 0; i < pathSpecs.length; i++) {
          const pathSpec = pathSpecs[i];
          // Check if this path has any data in the result
          // Data array format: [timestamp, value1, value2, ...]
          // Each path corresponds to a position in the values array
          const hasData = allResult.data.some(row => {
            const valueIndex = i + 1; // +1 because index 0 is timestamp
            return row[valueIndex] !== null && row[valueIndex] !== undefined;
          });

          if (!hasData) {
            debug(
              `[AutoDiscovery] No data found for path ${pathSpec.path}, checking auto-discovery`
            );
            const result =
              await this.autoDiscoveryService.maybeAutoConfigurePath(
                pathSpec.path as Path,
                context
              );
            if (result.configured) {
              autoConfiguredPaths.push(result);
              debug(`[AutoDiscovery] Auto-configured path: ${pathSpec.path}`);
            } else {
              debug(
                `[AutoDiscovery] Path ${pathSpec.path} not auto-configured: ${result.reason}`
              );
            }
          }
        }

        // Add meta to response if any paths were auto-configured
        if (autoConfiguredPaths.length > 0) {
          allResult.meta = {
            autoConfigured: true,
            paths: autoConfiguredPaths.map(r => r.path),
            message: `${autoConfiguredPaths.length} path(s) auto-configured for recording. Data will be available shortly.`,
          };
        }
      }

      if (canStreamResponse(res)) {
        await streamDataResult(res, allResult, debug);
      } else {
        // Internal callers (historical-streaming.ts) pass a mock res with
        // only json()/status().json(); fall back to the buffered shape so
        // they keep working without an Express Response.
        res.json(convertToLocalTime(allResult));
      }
    } catch (error) {
      if (error instanceof InvalidResolutionError) {
        debug(`Bad request in getValues: ${error.message}`);
        res.status(400).json({ error: error.message });
        return;
      }
      // Note: the response may have been streamed, so by the time an
      // error reaches here, headers may already be on the wire. The two
      // branches below are deliberately not consolidated into a single
      // res.status(500) call as in the rest of this module.
      if (res.headersSent) {
        // Stream already started -- can't switch to a 500. Destroy the socket
        // rather than res.end(): ending would close a half-written JSON array
        // cleanly, so the client would see an orderly 200 indistinguishable
        // from a complete body. Destroying surfaces a broken transfer instead.
        // The message is distinct from the pre-stream failure path so an
        // operator can tell a truncated mid-stream failure apart from it.
        debug(
          `Error in getValues after headersSent (truncated response): ${error}`
        );
        res.destroy();
        return;
      }
      debug(`Error in getValues: ${error}`);
      res.status(500).json({
        error: 'Internal server error',
        message: error instanceof Error ? error.message : String(error),
      });
    }
  }

  async getNumericValues(
    context: Context,
    from: ZonedDateTime,
    to: ZonedDateTime,
    timeResolutionMillis: number,
    pathSpecs: PathSpec[],
    debug: (k: string) => void,
    tier?: AggregationTier,
    spatialFilter?: SpatialFilter | null,
    querySource: QuerySource = 'local',
    positionPath: string = 'navigation.position',
    app?: any
  ): Promise<DataResult> {
    // Keyed by pathSpecKey(spec), not bare path, so the same path requested
    // with different sources/aggregates keeps separate series.
    const allData: { [key: string]: Array<[Timestamp, unknown]> } = {};
    const objectPaths = new Set<string>(); // Track which paths are object paths

    // Convert ZonedDateTime to Date for S3 pattern building
    const fromDate = new Date(from.toInstant().toString());
    const toDate = new Date(to.toInstant().toString());

    // If spatial filter is set, get valid timestamps from position data
    // This allows filtering non-position paths by "when vessel was in this area"
    let spatialTimestamps: Set<string> | null = null;
    const hasPositionPath =
      spatialFilter && pathSpecs.some(ps => isPositionPath(ps.path));
    if (spatialFilter) {
      const hasNonPositionPaths = pathSpecs.some(
        ps => !isPositionPath(ps.path)
      );
      if (hasNonPositionPaths && !hasPositionPath) {
        // Position not in requested paths — need a separate scan for correlation
        debug(
          `[Spatial Correlation] Querying ${positionPath} for valid timestamps within spatial filter`
        );
        spatialTimestamps = await this.getSpatialTimestamps(
          context,
          from,
          to,
          timeResolutionMillis,
          spatialFilter,
          positionPath,
          undefined, // Force raw tier — position is an object path, aggregated tiers lack value_latitude/value_longitude
          querySource,
          debug
        );
        debug(
          `[Spatial Correlation] Found ${spatialTimestamps.size} valid timestamps`
        );
      }

      if (hasPositionPath) {
        // Position IS in requested paths — use fast bucket+FIRST query for both
        // position results AND spatial correlation timestamps (one query, no full raw scan)
        const posPathSpec = pathSpecs.find(ps => isPositionPath(ps.path))!;
        debug(
          `[Spatial] Fast bucket query for ${posPathSpec.path} (FIRST lat/lon per bucket + JS filter)`
        );

        const sanitizedCtx = this.hivePathBuilder.sanitizeContext(context);
        const sanitizedPos = this.hivePathBuilder.sanitizePath(
          posPathSpec.path
        );
        const posFilePath = path.join(
          this.dataDir,
          'tier=raw',
          `context=${sanitizedCtx}`,
          `path=${sanitizedPos}`,
          '**',
          '*.parquet'
        );
        const fromIso = from.toInstant().toString();
        const toIso = to.toInstant().toString();

        try {
          const hasBuffer = DuckDBPool.isSQLiteBufferInitialized();
          const connection = await DuckDBPool.getConnection();
          try {
            const bucketExpr = `strftime(DATE_TRUNC('seconds',
              EPOCH_MS(CAST(FLOOR(EPOCH_MS(signalk_timestamp::TIMESTAMP) / ${timeResolutionMillis}) * ${timeResolutionMillis} AS BIGINT))
            ), '%Y-%m-%dT%H:%M:%SZ')`;

            // Optional filters for the position path (e.g. one of several GPS
            // receivers). This fast path is local-only (raw parquet + buffer),
            // so probe just the local glob for the filter columns.
            const posAvailable = await availableFilterColumns(
              connection,
              [posFilePath],
              posPathSpec.filters
            );
            const posSourceFilter = buildParquetFilterClause(
              posPathSpec.filters,
              posAvailable
            );

            // Build FROM: parquet UNION ALL buffer (for today's unexported data)
            const parquetFrom = `SELECT signalk_timestamp, value_latitude, value_longitude FROM (
              SELECT * FROM read_parquet('${posFilePath}', union_by_name=true, filename=true)
              WHERE filename NOT LIKE '%/processed/%'
              AND filename NOT LIKE '%/quarantine/%'
              AND filename NOT LIKE '%/failed/%'
              AND filename NOT LIKE '%/repaired/%'${posSourceFilter})`;

            let fromSource = `(${parquetFrom})`;
            if (hasBuffer && this.sqliteBuffer) {
              const bufferTableCols = this.sqliteBuffer.getTableColumns(
                posPathSpec.path
              );
              const stagedTable = await stageBufferTable(
                connection,
                this.sqliteBuffer,
                String(context),
                posPathSpec.path,
                fromIso,
                toIso,
                debug
              );
              const bufferSubquery = stagedTable
                ? buildBufferObjectSubquery(
                    stagedTable,
                    context,
                    fromIso,
                    toIso,
                    new Map([
                      [
                        'latitude',
                        {
                          name: 'latitude',
                          columnName: 'value_latitude',
                          dataType: 'numeric' as const,
                        },
                      ],
                      [
                        'longitude',
                        {
                          name: 'longitude',
                          columnName: 'value_longitude',
                          dataType: 'numeric' as const,
                        },
                      ],
                    ]),
                    bufferTableCols,
                    posPathSpec.filters
                  )
                : null;
              if (bufferSubquery) {
                fromSource = `(${parquetFrom} UNION ALL SELECT signalk_timestamp, TRY_CAST(value_latitude AS DOUBLE) as value_latitude, TRY_CAST(value_longitude AS DOUBLE) as value_longitude FROM ${bufferSubquery})`;
              }
            }

            const query = `
              SELECT
                ${bucketExpr} as timestamp,
                FIRST(TRY_CAST(value_latitude AS DOUBLE)) as lat,
                FIRST(TRY_CAST(value_longitude AS DOUBLE)) as lon
              FROM ${fromSource}
              WHERE
                signalk_timestamp >= '${fromIso}'
                AND signalk_timestamp < '${toIso}'
                AND TRY_CAST(value_latitude AS DOUBLE) IS NOT NULL
                AND TRY_CAST(value_longitude AS DOUBLE) IS NOT NULL
              GROUP BY timestamp
              ORDER BY timestamp
            `;

            const result = await connection.runAndReadAll(query);
            const rows = result.getRowObjects() as Array<{
              timestamp: string;
              lat: number;
              lon: number;
            }>;

            // Filter by spatial and build position results + timestamps
            const posData: Array<[Timestamp, unknown]> = [];
            spatialTimestamps = new Set<string>();
            const { bbox } = spatialFilter;

            for (const row of rows) {
              if (row.lat === null || row.lon === null) continue;

              let inArea = true;

              // Lat check
              if (row.lat < bbox.south || row.lat > bbox.north) inArea = false;

              // Lon check
              if (inArea) {
                if (bbox.west <= bbox.east) {
                  if (row.lon < bbox.west || row.lon > bbox.east)
                    inArea = false;
                } else {
                  if (row.lon < bbox.west && row.lon > bbox.east)
                    inArea = false;
                }
              }

              // Precise radius check
              if (
                inArea &&
                spatialFilter.type === 'radius' &&
                spatialFilter.centerLat !== undefined &&
                spatialFilter.centerLon !== undefined &&
                spatialFilter.radiusMeters !== undefined
              ) {
                const dist = calculateDistance(
                  row.lat,
                  row.lon,
                  spatialFilter.centerLat,
                  spatialFilter.centerLon
                );
                if (dist > spatialFilter.radiusMeters) inArea = false;
              }

              if (inArea) {
                spatialTimestamps.add(row.timestamp);
                posData.push([
                  row.timestamp as Timestamp,
                  { latitude: row.lat, longitude: row.lon },
                ]);
              }
            }

            allData[pathSpecKey(posPathSpec)] = posData;
            debug(
              `[Spatial] Bucketed ${rows.length} positions, ${posData.length} within filter`
            );
          } finally {
            connection.disconnectSync();
          }
        } catch (error) {
          debug(`[Spatial] Error in fast position query: ${error}`);
          allData[pathSpecKey(posPathSpec)] = [];
          spatialTimestamps = new Set<string>();
        }
      }
    }

    // Process each path and collect data with concurrency limiting
    // Limit concurrent queries to prevent resource exhaustion (configured in cache-defaults)
    const limiter = new ConcurrencyLimiter(CONCURRENCY.MAX_QUERIES);
    await limiter.map(pathSpecs, async pathSpec => {
      try {
        // Skip position if already handled by fast spatial bucket query
        if (
          hasPositionPath &&
          isPositionPath(pathSpec.path) &&
          allData[pathSpecKey(pathSpec)]
        ) {
          return;
        }

        // Build file patterns based on query source
        let localFilePath: string | null = null;
        let s3FilePath: string | null = null;
        let effectiveTier = tier || 'raw';

        // Honour per-path skipAggregation: AggregationService never wrote
        // 5s/60s/1h files for these paths, so reading any aggregated tier
        // would return empty. Fall back to raw — the existing object-path
        // and string-path overrides further down handle their own special
        // cases and run after this.
        if (
          effectiveTier !== 'raw' &&
          this.retentionRules.shouldSkipAggregation(pathSpec.path)
        ) {
          debug(
            `Path ${pathSpec.path}: skipAggregation rule matched — overriding tier=${effectiveTier} to raw`
          );
          effectiveTier = 'raw';
        }

        // Inline filters require raw data: aggregated tiers (5s/60s/1h) blend
        // every source into each bucket and don't carry the source_* columns,
        // so they cannot be filtered.
        if (effectiveTier !== 'raw' && pathSpec.filters.length > 0) {
          debug(
            `Path ${pathSpec.path}: filter present — overriding tier=${effectiveTier} to raw`
          );
          effectiveTier = 'raw';
        }

        // Always query local first
        const sanitizedContext = this.hivePathBuilder.sanitizeContext(context);
        const sanitizedSkPath = this.hivePathBuilder.sanitizePath(
          pathSpec.path
        );
        localFilePath = path.join(
          this.dataDir,
          `tier=${effectiveTier}`,
          `context=${sanitizedContext}`,
          `path=${sanitizedSkPath}`,
          '**',
          '*.parquet'
        );
        debug(`Querying local Hive tier=${effectiveTier} at: ${localFilePath}`);