signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
974 lines • 104 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HistoryAPI = void 0;
exports.registerHistoryApiRoute = registerHistoryApiRoute;
const core_1 = require("@js-joda/core");
const duckdb_pool_1 = require("./utils/duckdb-pool");
const path_1 = __importDefault(require("path"));
const path_discovery_1 = require("./utils/path-discovery");
const path_cache_1 = require("./utils/path-cache");
const context_discovery_1 = require("./utils/context-discovery");
const schema_cache_1 = require("./utils/schema-cache");
const concurrency_limiter_1 = require("./utils/concurrency-limiter");
const cache_defaults_1 = require("./config/cache-defaults");
const hive_path_builder_1 = require("./utils/hive-path-builder");
const spatial_queries_1 = require("./utils/spatial-queries");
const geo_calculator_1 = require("./utils/geo-calculator");
const buffer_sql_builder_1 = require("./utils/buffer-sql-builder");
const buffer_staging_1 = require("./utils/buffer-staging");
const duration_parser_1 = require("./utils/duration-parser");
const angular_paths_1 = require("./utils/angular-paths");
const retention_rules_1 = require("./utils/retention-rules");
const path_filters_1 = require("./utils/path-filters");
// Allowed characters in the `paths` query parameter: path/aggregate tokens
// (alphanumerics, `.,:_`), the inline-filter delimiters from PATH_FILTER_DEFS,
// and `-`. Delimiters are escaped for use inside a character class; `-` is kept
// last so it stays a literal.
const PATHS_SANITIZER = new RegExp(`[^0-9a-z.,:_${path_filters_1.FILTER_DELIMITERS.replace(/[\\\]^-]/g, '\\$&')}-]`, 'gi');
function registerHistoryApiRoute(router, selfId, dataDir, debug, app, sqliteBuffer, autoDiscoveryService, s3Config, pathRetentionOverrides) {
const historyApi = new HistoryAPI(selfId, dataDir, sqliteBuffer, autoDiscoveryService, s3Config, pathRetentionOverrides);
// Handler for values endpoint
const handleValues = (req, res) => {
const { from, to, context, spatialFilter } = getRequestParams(req, selfId);
historyApi.getValues(context, from, to, spatialFilter, app, debug, req, res);
};
// V1 route: Direct routes with all signalk-parquet extensions
// (spatial filtering, timezone conversion, moving averages, etc.)
router.get('/signalk/v1/history/values', handleValues);
// Handler for contexts endpoint
const handleContexts = async (req, res) => {
try {
// Check if time range parameters are provided
const hasTimeParams = req.query.duration || req.query.from || req.query.to;
if (hasTimeParams) {
// Time-range-aware: return only contexts with data in the specified time range
const { from, to } = getRequestParams(req, selfId);
// Check cache first
let contexts = (0, path_cache_1.getCachedContexts)(from, to);
if (!contexts) {
// Cache miss - query the parquet files
contexts = await (0, context_discovery_1.getAvailableContextsForTimeRange)(dataDir, from, to);
// Cache the result
(0, path_cache_1.setCachedContexts)(from, to, contexts);
}
res.json(contexts);
}
else {
// No time range specified: return only self context (legacy behavior)
res.json([`vessels.${selfId}`]);
}
}
catch (error) {
debug(`Error in /contexts: ${error}`);
res.status(500).json({ error: error.message });
}
};
// V1 route: Direct routes with extensions (time-range-aware context discovery)
router.get('/signalk/v1/history/contexts', handleContexts);
// Note: V2 routes (/signalk/v2/api/history/*) are handled by the registered
// HistoryApi provider in history-provider.ts via app.registerHistoryApiProvider()
// Handler for paths endpoint
const handlePaths = async (req, res) => {
try {
// Check if time range parameters are provided
const hasTimeParams = req.query.duration || req.query.from || req.query.to;
if (hasTimeParams) {
// Time-range-aware: return only paths with data in the specified time range
const { from, to, context } = getRequestParams(req, selfId);
// Check cache first
let paths = (0, path_cache_1.getCachedPaths)(context, from, to);
if (!paths) {
// Cache miss - query the parquet files
paths = await (0, path_discovery_1.getAvailablePathsForTimeRange)(dataDir, context, from, to);
// Cache the result
(0, path_cache_1.setCachedPaths)(context, from, to, paths);
}
res.json(paths);
}
else {
// No time range specified: return all available paths (legacy behavior)
const context = req.query.context
? getContext(req.query.context, selfId)
: undefined;
const paths = (0, path_discovery_1.getAvailablePathsArray)(dataDir, app, context);
res.json(paths);
}
}
catch (error) {
debug(`Error in /paths: ${error}`);
res.status(500).json({ error: error.message });
}
};
// V1 route: Direct routes with extensions (time-range-aware path discovery)
router.get('/signalk/v1/history/paths', handlePaths);
// Also register as plugin-style routes for testing
router.get('/api/history/values', (req, res) => {
const { from, to, context, spatialFilter } = getRequestParams(req, selfId);
historyApi.getValues(context, from, to, spatialFilter, app, debug, req, res);
});
router.get('/api/history/contexts', async (req, res) => {
try {
// Check if time range parameters are provided
const hasTimeParams = req.query.duration || req.query.from || req.query.to;
if (hasTimeParams) {
// Time-range-aware: return only contexts with data in the specified time range
const { from, to } = getRequestParams(req, selfId);
// Check cache first
let contexts = (0, path_cache_1.getCachedContexts)(from, to);
if (!contexts) {
// Cache miss - query the parquet files
contexts = await (0, context_discovery_1.getAvailableContextsForTimeRange)(dataDir, from, to);
// Cache the result
(0, path_cache_1.setCachedContexts)(from, to, contexts);
}
res.json(contexts);
}
else {
// No time range specified: return only self context (legacy behavior)
res.json([`vessels.${selfId}`]);
}
}
catch (error) {
debug(`Error in /api/history/contexts: ${error}`);
res.status(500).json({ error: error.message });
}
});
router.get('/api/history/contexts/spatial', async (req, res) => {
try {
const { from, to } = getRequestParams(req, selfId);
const spatialFilter = (0, spatial_queries_1.parseSpatialParams)(req.query.bbox, req.query.radius);
if (!spatialFilter) {
res.status(400).json({
error: 'bbox (west,south,east,north) or radius (lon,lat,meters) is required',
});
return;
}
const contexts = await (0, context_discovery_1.getContextsInSpatialFilter)(dataDir, from, to, spatialFilter);
res.json(contexts);
}
catch (error) {
debug(`Error in /api/history/contexts/spatial: ${error}`);
res.status(500).json({ error: error.message });
}
});
router.get('/api/history/paths', async (req, res) => {
try {
// Check if time range parameters are provided
const hasTimeParams = req.query.duration || req.query.from || req.query.to;
if (hasTimeParams) {
// Time-range-aware: return only paths with data in the specified time range
const { from, to, context } = getRequestParams(req, selfId);
// Check cache first
let paths = (0, path_cache_1.getCachedPaths)(context, from, to);
if (!paths) {
// Cache miss - query the parquet files
paths = await (0, path_discovery_1.getAvailablePathsForTimeRange)(dataDir, context, from, to);
// Cache the result
(0, path_cache_1.setCachedPaths)(context, from, to, paths);
}
res.json(paths);
}
else {
// No time range specified: return all available paths (legacy behavior)
const context = req.query.context
? getContext(req.query.context, selfId)
: undefined;
const paths = (0, path_discovery_1.getAvailablePathsArray)(dataDir, app, context);
res.json(paths);
}
}
catch (error) {
debug(`Error in /api/history/paths: ${error}`);
res.status(500).json({ error: error.message });
}
});
}
const getRequestParams = ({ query }, selfId) => {
try {
let from;
let to;
// ============================================================================
// STANDARD SIGNALK TIME RANGE PATTERNS
// ============================================================================
// Pattern 1: duration only → query back from now
if (query.duration && !query.from && !query.to) {
const durationMs = parseDuration(query.duration);
to = core_1.ZonedDateTime.now(core_1.ZoneOffset.UTC);
from = to.minusNanos(durationMs * 1000000);
}
// Pattern 2: from + duration → query forward from start
else if (query.from && query.duration && !query.to) {
from = parseDateTime(query.from);
const durationMs = parseDuration(query.duration);
to = from.plusNanos(durationMs * 1000000);
}
// Pattern 3: to + duration → query backward to end
else if (query.to && query.duration && !query.from) {
to = parseDateTime(query.to);
const durationMs = parseDuration(query.duration);
from = to.minusNanos(durationMs * 1000000);
}
// Pattern 4: from only → from start to now
else if (query.from && !query.duration && !query.to) {
from = parseDateTime(query.from);
to = core_1.ZonedDateTime.now(core_1.ZoneOffset.UTC);
}
// Pattern 5: from + to → specific range
else if (query.from && query.to && !query.duration) {
from = parseDateTime(query.from);
to = parseDateTime(query.to);
}
else {
throw new Error('Invalid time range parameters. Use one of the following patterns:\n' +
' 1. ?duration=1h (query back from now)\n' +
' 2. ?from=2025-08-01T00:00:00Z&duration=1h (query forward)\n' +
' 3. ?to=2025-08-01T12:00:00Z&duration=1h (query backward)\n' +
' 4. ?from=2025-08-01T00:00:00Z (from start to now)\n' +
' 5. ?from=2025-08-01T00:00:00Z&to=2025-08-02T00:00:00Z (specific range)');
}
const context = getContext(query.context, selfId);
const spatialFilter = (0, spatial_queries_1.parseSpatialParams)(query.bbox, query.radius);
return { from, to, context, spatialFilter };
}
catch (e) {
console.error('Full error details:', e);
throw new Error(`Error extracting query parameters from ${JSON.stringify(query)}: ${e instanceof Error ? e.stack : e}`);
}
};
// Parse duration string (supports ISO 8601, integer seconds, shorthand)
function parseDuration(duration) {
if (!duration) {
throw new Error('Duration parameter is required');
}
return (0, duration_parser_1.parseDurationToMillis)(duration);
}
// Check if datetime string has timezone information
function hasTimezoneInfo(dateTimeStr) {
// Check for 'Z' at the end, or '+'/'-' followed by timezone offset pattern
return (dateTimeStr.endsWith('Z') ||
/[+-]\d{2}:?\d{2}$/.test(dateTimeStr) ||
/[+-]\d{4}$/.test(dateTimeStr));
}
// Parse datetime string per ISO 8601:
// - Bare timestamps (no Z, no offset) → local time, converted to UTC
// - Timestamps with Z or offset → parsed as-is
function parseDateTime(dateTimeStr) {
// Normalize the datetime string to include seconds if missing
let normalizedStr = dateTimeStr;
if (dateTimeStr.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$/)) {
// Add seconds if only HH:MM is provided
normalizedStr = dateTimeStr + ':00';
}
if (hasTimezoneInfo(normalizedStr)) {
// Has timezone info (Z or offset), parse as-is and convert to UTC
return core_1.ZonedDateTime.parse(normalizedStr).withZoneSameInstant(core_1.ZoneOffset.UTC);
}
else {
// No timezone info — per ISO 8601, treat as local time and convert to UTC
try {
// JavaScript Date constructor treats ISO strings without timezone as local time
const localDate = new Date(normalizedStr);
if (isNaN(localDate.getTime())) {
throw new Error('Invalid date');
}
// Convert to UTC ISO string and parse with ZonedDateTime
const utcIsoString = localDate.toISOString();
return core_1.ZonedDateTime.parse(utcIsoString);
}
catch (e) {
throw new Error(`Unable to parse datetime '${dateTimeStr}': ${e}. Use format like '2025-08-13T08:00:00' or '2025-08-13T08:00:00Z'`);
}
}
}
function getContext(contextFromQuery, selfId) {
if (!contextFromQuery ||
contextFromQuery === 'vessels.self' ||
contextFromQuery === 'self') {
return `vessels.${selfId}`;
}
return contextFromQuery.replace(/ /gi, '');
}
/**
* Stream a DataResult to the response as a single JSON document,
* progressively, instead of buffering it via res.json(). Output is
* byte-equivalent to the prior res.json(result) path: same field
* order (context, range, values, data, units?, meta?), same UTC->
* local timestamp conversion on the range and on each row[0].
*
* Wrapper construction: build the wrapper object with a `data: []`
* placeholder, JSON.stringify it once, then split at that placeholder.
* The two halves frame the row-by-row writes. This avoids hand-rolling
* `,` / `{}` / `[]` framing tokens.
* Input wrapper -> '{...,"data":[],"units":{...}}'
* prefix -> '{...,"data":['
* suffix -> ']' + ',"units":{...}}'
*
* Backpressure: if res.write returns false the loop awaits 'drain'
* before continuing, so a slow client cannot grow Node's internal
* write buffer beyond the high-water mark. The wait races against
* 'close' so a client that disconnects mid-flight does not hang the
* stream forever.
*
* Cancellation: a 'close' listener on res sets a flag so the loop
* breaks early on client disconnect. The DuckDB result has already
* been materialised before this function is called, so cancellation
* is purely about cutting the write loop short.
*
* Logging: on success or post-headers failure, emits one structured
* line via the optional `log` callback with row count, byte count
* and wall-clock duration. Operators chasing "endpoint feels slow"
* have a single line to grep for.
*/
async function streamDataResult(res, result, log) {
const startMs = Date.now();
let rowsWritten = 0;
let bytesWritten = 0;
let clientGone = false;
const onClose = () => {
clientGone = true;
};
// A mid-stream socket failure (ECONNRESET / EPIPE) emits 'error' on res.
// With no listener Node rethrows it as an uncaughtException and takes down
// the whole server process. Treat it like a disconnect: flip the flag so the
// write loop unwinds instead of writing into a broken socket.
const onError = () => {
clientGone = true;
};
res.on('close', onClose);
res.on('error', onError);
try {
res.setHeader('Content-Type', 'application/json; charset=utf-8');
// Streaming responses must not be cached by an intermediary.
// X-Accel-Buffering is nginx-specific (no-op elsewhere) and tells
// nginx not to buffer the response body, so chunks reach the
// client as we write them rather than after res.end().
res.setHeader('Cache-Control', 'no-store');
res.setHeader('X-Accel-Buffering', 'no');
const localRange = {
from: utcToLocalTimestamp(result.range.from),
to: utcToLocalTimestamp(result.range.to),
};
// Build the wrapper with `data: []` placeholder so JSON.stringify
// produces the framing for us. Optional fields (units, meta) come
// after data per DataResult declaration order; insertion order in
// a plain object is preserved by JSON.stringify.
const wrapper = {
context: result.context,
range: localRange,
values: result.values,
data: [],
};
if (result.units)
wrapper.units = result.units;
if (result.meta)
wrapper.meta = result.meta;
const wrapperJson = JSON.stringify(wrapper);
const marker = '"data":[]';
const markerIdx = wrapperJson.indexOf(marker);
if (markerIdx === -1) {
// The wrapper is constructed locally with data:[] as a literal
// field; this branch only fires if a future JSON.stringify ever
// omits empty arrays. Cheaper to assert than to debug a silent
// wire-format break.
throw new Error('streamDataResult: wrapper marker not found');
}
const prefix = wrapperJson.slice(0, markerIdx) + '"data":[';
const suffix = ']' + wrapperJson.slice(markerIdx + marker.length);
const writeOrAwait = async (chunk) => {
bytesWritten += Buffer.byteLength(chunk, 'utf8');
if (!res.write(chunk)) {
// Wait for drain (kernel buffer flushed), close (client gone), or
// error (socket broke). Whichever fires first wins; all three unhook
// the others so we don't leak handlers across the loop, and we never
// await a 'drain' that a dead socket will never emit.
await new Promise(resolve => {
let settled = false;
const finish = () => {
if (settled)
return;
settled = true;
res.off('drain', finish);
res.off('close', finish);
res.off('error', finish);
resolve();
};
res.once('drain', finish);
res.once('close', finish);
res.once('error', finish);
});
}
};
await writeOrAwait(prefix);
const data = result.data;
for (let i = 0; i < data.length; i++) {
if (clientGone)
break;
const row = data[i];
// Allocate a converted row instead of mutating in place. The
// input is logically owned by the caller, and a future second
// caller (audit log, dual-format response) would silently see
// local time where it expected UTC.
const converted = row.slice();
converted[0] = utcToLocalTimestamp(row[0]);
const rowChunk = (i > 0 ? ',' : '') + JSON.stringify(converted);
await writeOrAwait(rowChunk);
rowsWritten++;
}
if (clientGone) {
log?.(`streamDataResult: client disconnected after ${rowsWritten}/${data.length} rows, ${bytesWritten} bytes, ${Date.now() - startMs}ms`);
return;
}
await writeOrAwait(suffix);
res.end();
log?.(`streamDataResult: rows=${rowsWritten} bytes=${bytesWritten} duration=${Date.now() - startMs}ms`);
}
finally {
res.off('close', onClose);
res.off('error', onError);
}
}
/**
* Buffered UTC->local conversion of a full DataResult. Used by callers
* that pass a non-streaming response (e.g. internal mocks that only
* implement res.json) so they receive the same locally-converted shape
* the streaming path produces.
*/
function convertToLocalTime(result) {
return {
...result,
range: {
from: utcToLocalTimestamp(result.range.from),
to: utcToLocalTimestamp(result.range.to),
},
data: result.data.map(row => {
const newRow = row.slice();
newRow[0] = utcToLocalTimestamp(row[0]);
return newRow;
}),
};
}
/**
* Returns true when `res` exposes the Node stream methods streamDataResult
* relies on. The plugin's HTTP routes pass a real Express Response, but
* historical-streaming.ts reuses getValues with a mock that only implements
* res.json / res.status().json — those callers go through the buffered path.
*/
function canStreamResponse(res) {
return (typeof res.write === 'function' &&
typeof res.end === 'function' &&
typeof res.setHeader === 'function' &&
typeof res.on === 'function');
}
/**
* Convert a UTC timestamp string to server-local time with offset suffix.
* e.g. "2026-03-06T19:00:00Z" → "2026-03-06T14:00:00-05:00" (EST)
*/
function utcToLocalTimestamp(utcTs) {
try {
const zdt = core_1.ZonedDateTime.parse(utcTs).withZoneSameInstant(core_1.ZoneId.systemDefault());
// ZonedDateTime.toString() appends "[SYSTEM]" zone ID — strip it
return zdt.toString().replace(/\[.*\]$/, '');
}
catch {
// If parsing fails (e.g. already has offset or unusual format), try via Date
const d = new Date(utcTs);
if (isNaN(d.getTime()))
return utcTs;
const offsetMin = d.getTimezoneOffset();
const local = new Date(d.getTime() - offsetMin * 60000);
const sign = offsetMin <= 0 ? '+' : '-';
const absH = String(Math.floor(Math.abs(offsetMin) / 60)).padStart(2, '0');
const absM = String(Math.abs(offsetMin) % 60).padStart(2, '0');
return (local.toISOString().replace('Z', '') +
sign +
absH +
':' +
absM);
}
}
class HistoryAPI {
constructor(selfId, dataDir, sqliteBuffer, autoDiscoveryService, s3Config, pathRetentionOverrides) {
this.selfId = selfId;
this.dataDir = dataDir;
this.sqliteBuffer = sqliteBuffer;
this.hivePathBuilder = new hive_path_builder_1.HivePathBuilder();
this.autoDiscoveryService = autoDiscoveryService;
this.s3Config = s3Config;
this.retentionRules = new retention_rules_1.RetentionRuleSet(pathRetentionOverrides || []);
}
/**
* Set S3 configuration for federated queries
*/
setS3Config(config) {
this.s3Config = config;
}
/**
* Set the auto-discovery service
*/
setAutoDiscoveryService(service) {
this.autoDiscoveryService = service;
}
/**
* Set the SQLite buffer for federated queries
*/
setSqliteBuffer(buffer) {
this.sqliteBuffer = buffer;
}
/**
* Auto-select the optimal tier based on requested resolution
* Returns undefined to use raw/flat data, or a tier name for aggregated data
*
* Logic:
* - resolution >= 1 hour (3600000ms) → use 1h tier
* - resolution >= 1 minute (60000ms) → use 60s tier
* - resolution >= 5 seconds (5000ms) → use 5s tier
* - resolution < 5 seconds → use raw data (no tier)
*
* Falls back through tiers if preferred tier doesn't exist
*/
selectOptimalTier(resolutionMillis) {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const fs = require('fs');
// Determine preferred tier based on resolution
let preferredTiers = [];
if (resolutionMillis >= 3600000) {
preferredTiers = ['1h', '60s', '5s'];
}
else if (resolutionMillis >= 60000) {
preferredTiers = ['60s', '5s'];
}
else if (resolutionMillis >= 5000) {
preferredTiers = ['5s'];
}
else {
// Use raw data for sub-5-second resolution
return undefined;
}
// Check which tiers exist and return the best available
for (const tier of preferredTiers) {
const tierPath = path_1.default.join(this.dataDir, `tier=${tier}`);
try {
if (fs.existsSync(tierPath)) {
return tier;
}
}
catch {
// Directory doesn't exist or not accessible
}
}
// No aggregated tiers available, use raw data
return undefined;
}
/**
* Get timestamps where vessel position was within the spatial filter
* Used to correlate non-position paths with spatial filtering
*/
async getSpatialTimestamps(context, from, to, timeResolutionMillis, spatialFilter, positionPath, _tier, _querySource, debug) {
const timestamps = new Set();
// Build file path for raw position data
const sanitizedContext = this.hivePathBuilder.sanitizeContext(context);
const sanitizedPath = this.hivePathBuilder.sanitizePath(positionPath);
const localFilePath = path_1.default.join(this.dataDir, 'tier=raw', `context=${sanitizedContext}`, `path=${sanitizedPath}`, '**', '*.parquet');
const fromIso = from.toInstant().toString();
const toIso = to.toInstant().toString();
try {
const hasBuffer = duckdb_pool_1.DuckDBPool.isSQLiteBufferInitialized();
const connection = await duckdb_pool_1.DuckDBPool.getConnection();
try {
// Bucket-lookup approach: instead of scanning all raw position data,
// bucket by time resolution, grab FIRST lat/lon per bucket, then filter by bbox/radius.
// This reads far less data than a full scan with spatial SQL.
const bucketExpr = `strftime(DATE_TRUNC('seconds',
EPOCH_MS(CAST(FLOOR(EPOCH_MS(signalk_timestamp::TIMESTAMP) / ${timeResolutionMillis}) * ${timeResolutionMillis} AS BIGINT))
), '%Y-%m-%dT%H:%M:%SZ')`;
// Build FROM: parquet UNION ALL buffer
const parquetFrom = `SELECT signalk_timestamp, value_latitude, value_longitude FROM (
SELECT * FROM read_parquet('${localFilePath}', union_by_name=true, filename=true)
WHERE filename NOT LIKE '%/processed/%'
AND filename NOT LIKE '%/quarantine/%'
AND filename NOT LIKE '%/failed/%'
AND filename NOT LIKE '%/repaired/%')`;
let fromSource = `(${parquetFrom})`;
if (hasBuffer && this.sqliteBuffer) {
const bufferTableCols = this.sqliteBuffer.getTableColumns(positionPath);
const stagedTable = await (0, buffer_staging_1.stageBufferTable)(connection, this.sqliteBuffer, String(context), positionPath, fromIso, toIso, debug);
const bufferSubquery = stagedTable
? (0, buffer_sql_builder_1.buildBufferObjectSubquery)(stagedTable, context, fromIso, toIso, new Map([
[
'latitude',
{
name: 'latitude',
columnName: 'value_latitude',
dataType: 'numeric',
},
],
[
'longitude',
{
name: 'longitude',
columnName: 'value_longitude',
dataType: 'numeric',
},
],
]), bufferTableCols)
: null;
if (bufferSubquery) {
fromSource = `(${parquetFrom} UNION ALL SELECT signalk_timestamp, TRY_CAST(value_latitude AS DOUBLE) as value_latitude, TRY_CAST(value_longitude AS DOUBLE) as value_longitude FROM ${bufferSubquery})`;
}
}
const query = `
SELECT
${bucketExpr} as timestamp,
FIRST(TRY_CAST(value_latitude AS DOUBLE)) as lat,
FIRST(TRY_CAST(value_longitude AS DOUBLE)) as lon
FROM ${fromSource}
WHERE
signalk_timestamp >= '${fromIso}'
AND signalk_timestamp < '${toIso}'
AND TRY_CAST(value_latitude AS DOUBLE) IS NOT NULL
AND TRY_CAST(value_longitude AS DOUBLE) IS NOT NULL
GROUP BY timestamp
ORDER BY timestamp
`;
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
// Filter buckets by spatial filter in JS
const { bbox } = spatialFilter;
for (const row of rows) {
if (row.lat === null || row.lon === null)
continue;
// Lat check
if (row.lat < bbox.south || row.lat > bbox.north)
continue;
// Lon check (handles 180° meridian crossing)
if (bbox.west <= bbox.east) {
if (row.lon < bbox.west || row.lon > bbox.east)
continue;
}
else {
if (row.lon < bbox.west && row.lon > bbox.east)
continue;
}
// Precise radius check if applicable
if (spatialFilter.type === 'radius' &&
spatialFilter.centerLat !== undefined &&
spatialFilter.centerLon !== undefined &&
spatialFilter.radiusMeters !== undefined) {
const dist = (0, geo_calculator_1.calculateDistance)(row.lat, row.lon, spatialFilter.centerLat, spatialFilter.centerLon);
if (dist > spatialFilter.radiusMeters)
continue;
}
timestamps.add(row.timestamp);
}
debug(`[Spatial Correlation] Bucketed ${rows.length} positions, ${timestamps.size} within spatial filter`);
}
finally {
connection.disconnectSync();
}
}
catch (error) {
debug(`[Spatial Correlation] Error querying position data: ${error}`);
// On error, return empty set (no filtering will occur)
}
return timestamps;
}
async getValues(context, from, to, spatialFilter, app, debug,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
req,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
res) {
try {
// Resolution now in SECONDS (breaking change from v0.7.0)
const timeResolutionMillis = req.query.resolution
? (0, duration_parser_1.parseResolutionToMillis)(req.query.resolution)
: ((to.toEpochSecond() - from.toEpochSecond()) / 500) * 1000;
// Strip everything except path/aggregate characters, the filter
// delimiters from PATH_FILTER_DEFS, and `-` (common in source
// references). This is a coarse injection guard; values are also SQL-
// escaped downstream. Adding a delimiter to the registry extends this
// automatically.
const pathExpressions = (req.query.paths || '')
.replace(PATHS_SANITIZER, '')
.split(',');
const pathSpecs = pathExpressions.map(splitPathExpression);
// Auto-select tier based on resolution (provider selects automatically)
const tier = this.selectOptimalTier(timeResolutionMillis);
if (tier) {
debug(`Auto-selected tier=${tier} for resolution=${timeResolutionMillis}ms`);
}
// Log spatial filter if present
if (spatialFilter) {
debug(`Spatial filter: type=${spatialFilter.type}, bbox=[${spatialFilter.bbox.west},${spatialFilter.bbox.south},${spatialFilter.bbox.east},${spatialFilter.bbox.north}]`);
}
// Handle position and numeric paths together
const positionPath = 'navigation.position';
const allResult = pathSpecs.length
? await this.getNumericValues(context, from, to, timeResolutionMillis, pathSpecs, debug, tier, spatialFilter, 'local', positionPath, app)
: {
context,
range: {
from: from.toString(),
to: to.toString(),
},
values: [],
data: [],
};
// Check for auto-discovery on paths with no data
debug(`[AutoDiscovery] Checking auto-discovery: service=${!!this.autoDiscoveryService}, pathSpecs.length=${pathSpecs.length}`);
if (this.autoDiscoveryService && pathSpecs.length > 0) {
const autoConfiguredPaths = [];
for (let i = 0; i < pathSpecs.length; i++) {
const pathSpec = pathSpecs[i];
// Check if this path has any data in the result
// Data array format: [timestamp, value1, value2, ...]
// Each path corresponds to a position in the values array
const hasData = allResult.data.some(row => {
const valueIndex = i + 1; // +1 because index 0 is timestamp
return row[valueIndex] !== null && row[valueIndex] !== undefined;
});
if (!hasData) {
debug(`[AutoDiscovery] No data found for path ${pathSpec.path}, checking auto-discovery`);
const result = await this.autoDiscoveryService.maybeAutoConfigurePath(pathSpec.path, context);
if (result.configured) {
autoConfiguredPaths.push(result);
debug(`[AutoDiscovery] Auto-configured path: ${pathSpec.path}`);
}
else {
debug(`[AutoDiscovery] Path ${pathSpec.path} not auto-configured: ${result.reason}`);
}
}
}
// Add meta to response if any paths were auto-configured
if (autoConfiguredPaths.length > 0) {
allResult.meta = {
autoConfigured: true,
paths: autoConfiguredPaths.map(r => r.path),
message: `${autoConfiguredPaths.length} path(s) auto-configured for recording. Data will be available shortly.`,
};
}
}
if (canStreamResponse(res)) {
await streamDataResult(res, allResult, debug);
}
else {
// Internal callers (historical-streaming.ts) pass a mock res with
// only json()/status().json(); fall back to the buffered shape so
// they keep working without an Express Response.
res.json(convertToLocalTime(allResult));
}
}
catch (error) {
if (error instanceof duration_parser_1.InvalidResolutionError) {
debug(`Bad request in getValues: ${error.message}`);
res.status(400).json({ error: error.message });
return;
}
// Note: the response may have been streamed, so by the time an
// error reaches here, headers may already be on the wire. The two
// branches below are deliberately not consolidated into a single
// res.status(500) call as in the rest of this module.
if (res.headersSent) {
// Stream already started -- can't switch to a 500. Destroy the socket
// rather than res.end(): ending would close a half-written JSON array
// cleanly, so the client would see an orderly 200 indistinguishable
// from a complete body. Destroying surfaces a broken transfer instead.
// The message is distinct from the pre-stream failure path so an
// operator can tell a truncated mid-stream failure apart from it.
debug(`Error in getValues after headersSent (truncated response): ${error}`);
res.destroy();
return;
}
debug(`Error in getValues: ${error}`);
res.status(500).json({
error: 'Internal server error',
message: error instanceof Error ? error.message : String(error),
});
}
}
async getNumericValues(context, from, to, timeResolutionMillis, pathSpecs, debug, tier, spatialFilter, querySource = 'local', positionPath = 'navigation.position', app) {
// Keyed by pathSpecKey(spec), not bare path, so the same path requested
// with different sources/aggregates keeps separate series.
const allData = {};
const objectPaths = new Set(); // Track which paths are object paths
// Convert ZonedDateTime to Date for S3 pattern building
const fromDate = new Date(from.toInstant().toString());
const toDate = new Date(to.toInstant().toString());
// If spatial filter is set, get valid timestamps from position data
// This allows filtering non-position paths by "when vessel was in this area"
let spatialTimestamps = null;
const hasPositionPath = spatialFilter && pathSpecs.some(ps => (0, spatial_queries_1.isPositionPath)(ps.path));
if (spatialFilter) {
const hasNonPositionPaths = pathSpecs.some(ps => !(0, spatial_queries_1.isPositionPath)(ps.path));
if (hasNonPositionPaths && !hasPositionPath) {
// Position not in requested paths — need a separate scan for correlation
debug(`[Spatial Correlation] Querying ${positionPath} for valid timestamps within spatial filter`);
spatialTimestamps = await this.getSpatialTimestamps(context, from, to, timeResolutionMillis, spatialFilter, positionPath, undefined, // Force raw tier — position is an object path, aggregated tiers lack value_latitude/value_longitude
querySource, debug);
debug(`[Spatial Correlation] Found ${spatialTimestamps.size} valid timestamps`);
}
if (hasPositionPath) {
// Position IS in requested paths — use fast bucket+FIRST query for both
// position results AND spatial correlation timestamps (one query, no full raw scan)
const posPathSpec = pathSpecs.find(ps => (0, spatial_queries_1.isPositionPath)(ps.path));
debug(`[Spatial] Fast bucket query for ${posPathSpec.path} (FIRST lat/lon per bucket + JS filter)`);
const sanitizedCtx = this.hivePathBuilder.sanitizeContext(context);
const sanitizedPos = this.hivePathBuilder.sanitizePath(posPathSpec.path);
const posFilePath = path_1.default.join(this.dataDir, 'tier=raw', `context=${sanitizedCtx}`, `path=${sanitizedPos}`, '**', '*.parquet');
const fromIso = from.toInstant().toString();
const toIso = to.toInstant().toString();
try {
const hasBuffer = duckdb_pool_1.DuckDBPool.isSQLiteBufferInitialized();
const connection = await duckdb_pool_1.DuckDBPool.getConnection();
try {
const bucketExpr = `strftime(DATE_TRUNC('seconds',
EPOCH_MS(CAST(FLOOR(EPOCH_MS(signalk_timestamp::TIMESTAMP) / ${timeResolutionMillis}) * ${timeResolutionMillis} AS BIGINT))
), '%Y-%m-%dT%H:%M:%SZ')`;
// Optional filters for the position path (e.g. one of several GPS
// receivers). This fast path is local-only (raw parquet + buffer),
// so probe just the local glob for the filter columns.
const posAvailable = await (0, path_filters_1.availableFilterColumns)(connection, [posFilePath], posPathSpec.filters);
const posSourceFilter = (0, path_filters_1.buildParquetFilterClause)(posPathSpec.filters, posAvailable);
// Build FROM: parquet UNION ALL buffer (for today's unexported data)
const parquetFrom = `SELECT signalk_timestamp, value_latitude, value_longitude FROM (
SELECT * FROM read_parquet('${posFilePath}', union_by_name=true, filename=true)
WHERE filename NOT LIKE '%/processed/%'
AND filename NOT LIKE '%/quarantine/%'
AND filename NOT LIKE '%/failed/%'
AND filename NOT LIKE '%/repaired/%'${posSourceFilter})`;
let fromSource = `(${parquetFrom})`;
if (hasBuffer && this.sqliteBuffer) {
const bufferTableCols = this.sqliteBuffer.getTableColumns(posPathSpec.path);
const stagedTable = await (0, buffer_staging_1.stageBufferTable)(connection, this.sqliteBuffer, String(context), posPathSpec.path, fromIso, toIso, debug);
const bufferSubquery = stagedTable
? (0, buffer_sql_builder_1.buildBufferObjectSubquery)(stagedTable, context, fromIso, toIso, new Map([
[
'latitude',
{
name: 'latitude',
columnName: 'value_latitude',
dataType: 'numeric',
},
],
[
'longitude',
{
name: 'longitude',
columnName: 'value_longitude',
dataType: 'numeric',
},
],
]), bufferTableCols, posPathSpec.filters)
: null;
if (bufferSubquery) {
fromSource = `(${parquetFrom} UNION ALL SELECT signalk_timestamp, TRY_CAST(value_latitude AS DOUBLE) as value_latitude, TRY_CAST(value_longitude AS DOUBLE) as value_longitude FROM ${bufferSubquery})`;
}
}
const query = `
SELECT
${bucketExpr} as timestamp,
FIRST(TRY_CAST(value_latitude AS DOUBLE)) as lat,
FIRST(TRY_CAST(value_longitude AS DOUBLE)) as lon
FROM ${fromSource}
WHERE
signalk_timestamp >= '${fromIso}'
AND signalk_timestamp < '${toIso}'
AND TRY_CAST(value_latitude AS DOUBLE) IS NOT NULL
AND TRY_CAST(value_longitude AS DOUBLE) IS NOT NULL
GROUP BY timestamp
ORDER BY timestamp
`;
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
// Filter by spatial and build position results + timestamps
const posData = [];
spatialTimestamps = new Set();
const { bbox } = spatialFilter;
for (const row of rows) {
if (row.lat === null || row.lon === null)
continue;
let inArea = true;
// Lat check
if (row.lat < bbox.south || row.lat > bbox.north)
inArea = false;
// Lon check
if (inArea) {
if (bbox.west <= bbox.east) {
if (row.lon < bbox.west || row.lon > bbox.east)
inArea = false;
}
else {
if (row.lon < bbox.west && row.lon > bbox.east)
inArea = false;
}
}
// Precise radius check
if (inArea &&
spatialFilter.type === 'radius' &&
spatialFilter.centerLat !== undefined &&
spatialFilter.centerLon !== undefined &&
spatialFilter.radiusMeters !== undefined) {
const dist = (0, geo_calculator_1.calculateDistance)(row.lat, row.lon, spatialFilter.centerLat, spatialFilter.centerLon);
if (dist > spatialFilter.radiusMeters)
inArea = false;
}
if (inArea) {
spatialTimestamps.add(row.timestamp);
posData.push([
row.timestamp,
{ latitude: row.lat, longitude: row.lon },
]);
}
}
allData[pathSpecKey(posPathSpec)] = posData;
debug(`[Spatial] Bucketed ${rows.length} positions, ${posData.length} within filter`);
}
finally {
connection.disconnectSync();
}
}
catch (error) {
debug(`[Spatial] Error in fast position query: ${error}`);
allData[pathSpecKey(posPathSpec)] = [];
spatialTimestamps = new Set();
}
}
}
// Process each path and collect data with concurrency limiting
// Limit concurrent queries to prevent resource exhaustion (configured in cache-defaults)
const limiter = new concurrency_limiter_1.ConcurrencyLimiter(cache_defaults_1.CONCURRENCY.MAX_QUERIES);
await limiter.map(pathSpecs, async (pathSpec) => {
try {
// Skip position if already handled by fast spatial bucket query
if (hasPositionPath &&
(0, spatial_queries_1.isPositionPath)(pathSpec.path) &&
allData[pathSpecKey(pathSpec)]) {
return;
}
// Build file patterns based on query source
let localFilePath = null;
let s3FilePath = null;
let effectiveTier = tier || 'raw';
// Honour per-path skipAggregation: AggregationService never wrote
// 5s/60s/1h files for these paths, so reading any aggregated tier
// would return empty. Fall back to raw — the existing object-path
// and string-path overrides further down handle their own special
// cases and run after this.
if (effectiveTier !== 'raw' &&
this.retentionRules.shouldSkipAggregation(pathSpec.path)) {
debug(`Path ${pathSpec.path}: skipAggregation rule matched — overriding tier=${effectiveTier} to raw`);
effectiveTier = 'raw';
}
// Inline filters require raw data: aggregated tiers (5s/60s/1h) blend
// every source into each bucket and don't carry the source_* columns,
// so they cannot be filtered.
if (effectiveTier !== 'raw' && pathSpec.filters.length > 0) {
debug(`Path ${pathSpec.path}: filter present — overriding tier=${effectiveTier} to raw`);
effectiveTier = 'raw';
}
// Always query local first
const sanitizedContext = this.hivePathBuilder.sanitizeContext(context);
const sanitizedSkPath = this.hivePathBuilder.sanitizePath(pathSpec.path);
localFilePath = path_1.default.joi