signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
436 lines • 19.8 kB
JavaScript
;
/**
* SignalK History API Provider
*
* This module implements the SignalK HistoryApi interface to register
* this plugin as the official history data provider for the SignalK server.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.HistoryProvider = void 0;
exports.registerHistoryApiProvider = registerHistoryApiProvider;
exports.unregisterHistoryApiProvider = unregisterHistoryApiProvider;
const core_1 = require("@js-joda/core");
const path_filters_1 = require("./utils/path-filters");
const path_discovery_1 = require("./utils/path-discovery");
const context_discovery_1 = require("./utils/context-discovery");
const duckdb_pool_1 = require("./utils/duckdb-pool");
const schema_cache_1 = require("./utils/schema-cache");
const hive_path_builder_1 = require("./utils/hive-path-builder");
const angular_paths_1 = require("./utils/angular-paths");
const buffer_sql_builder_1 = require("./utils/buffer-sql-builder");
const buffer_staging_1 = require("./utils/buffer-staging");
/**
* Convert Temporal.Instant or ISO string to ZonedDateTime (UTC)
*/
function temporalToZonedDateTime(instant) {
// Handle ISO string (e.g., "2025-06-06T00:00:00Z")
if (typeof instant === 'string') {
return core_1.ZonedDateTime.parse(instant);
}
// Handle Temporal.Instant
const epochMillis = instant.epochMilliseconds;
return core_1.ZonedDateTime.ofInstant(core_1.Instant.ofEpochMilli(epochMillis), core_1.ZoneOffset.UTC);
}
/**
* Convert Temporal.Duration to milliseconds
*/
function durationToMillis(duration) {
if (typeof duration === 'number') {
return duration;
}
return duration.total({ unit: 'milliseconds' });
}
/**
* Parse time range parameters into from/to ZonedDateTime
*/
function parseTimeRange(params) {
const now = core_1.ZonedDateTime.now(core_1.ZoneOffset.UTC);
if ('from' in params && params.from && 'to' in params && params.to) {
// Both from and to specified
return {
from: temporalToZonedDateTime(params.from),
to: temporalToZonedDateTime(params.to),
};
}
else if ('from' in params &&
params.from &&
'duration' in params &&
params.duration) {
// From + duration: query forward
const from = temporalToZonedDateTime(params.from);
const durationMs = durationToMillis(params.duration);
const to = from.plusNanos(durationMs * 1000000);
return { from, to };
}
else if ('to' in params &&
params.to &&
'duration' in params &&
params.duration) {
// To + duration: query backward
const to = temporalToZonedDateTime(params.to);
const durationMs = durationToMillis(params.duration);
const from = to.minusNanos(durationMs * 1000000);
return { from, to };
}
else if ('from' in params && params.from) {
// From only: query to now
return {
from: temporalToZonedDateTime(params.from),
to: now,
};
}
else if ('duration' in params && params.duration) {
// Duration only: query back from now
const durationMs = durationToMillis(params.duration);
return {
from: now.minusNanos(durationMs * 1000000),
to: now,
};
}
// Default: last hour
return {
from: now.minusHours(1),
to: now,
};
}
/**
* Unique key for a requested path spec. The same path may be requested more
* than once with a different filter, aggregate, or parameters; keying stored
* results by path alone would collapse those into one column, so the key
* includes every distinguishing field. Fields never contain spaces
* (paths/aggregates/filter values are sanitised upstream), so a space
* separator is unambiguous.
*/
function pathSpecKey(ps) {
const parameter = (ps.parameter ?? []).join(',');
const filters = (0, path_filters_1.filtersFromFields)(ps)
.map(f => `${f.column}=${f.value}`)
.join(' ');
return [ps.path, ps.aggregate, parameter, filters].join(' ');
}
/**
* History API Provider implementation
*/
class HistoryProvider {
constructor(selfId, dataDir, app, debug) {
this.selfId = selfId;
this.dataDir = dataDir;
this.app = app;
this.debug = debug;
}
setSqliteBuffer(buffer) {
this.sqliteBuffer = buffer;
}
/**
* Get historical values for the specified query
*/
async getValues(query) {
this.debug(`[HistoryProvider] getValues called with: ${JSON.stringify(query, (_, v) => (typeof v === 'bigint' ? v.toString() : v))}`);
const { from, to } = parseTimeRange(query);
// Translate 'vessels.self' to actual vessel URN (same as HTTP endpoint)
const context = !query.context ||
query.context === 'vessels.self' ||
query.context === 'self'
? `vessels.${this.selfId}`
: query.context;
// query.resolution is in seconds per the SignalK History API spec;
// the DuckDB bucketing SQL below works in milliseconds. Reject 0,
// negative, NaN, and Infinity here so they don't reach SQL as a
// divide-by-zero, negative bucket, or unbounded cardinality query.
const r = query.resolution;
const resolutionFromQuery = r != null && Number.isFinite(r) && r > 0;
// Compute the auto fallback in milliseconds so a sub-second range
// doesn't truncate to a 0 ms divisor in the SQL bucketing.
const autoResolutionMs = Math.max(1, Math.round((to.toInstant().toEpochMilli() - from.toInstant().toEpochMilli()) / 500));
const resolutionMs = resolutionFromQuery ? r * 1000 : autoResolutionMs;
this.debug(`[HistoryProvider] getValues: context=${context}, from=${from}, to=${to}, resolution=${resolutionMs}ms (${resolutionFromQuery ? 'from query' : 'auto'}), paths=${query.pathSpecs.length}`);
const fromIso = from.toInstant().toString();
const toIso = to.toInstant().toString();
// Query each path
const allData = {};
for (const pathSpec of query.pathSpecs) {
// Key by the full spec, not just path: the same path may appear multiple
// times with different sourceRef/aggregate and must stay separate.
const key = pathSpecKey(pathSpec);
try {
const pathData = await this.queryPath(context, pathSpec, fromIso, toIso, resolutionMs);
allData[key] = pathData;
}
catch (error) {
this.debug(`[HistoryProvider] Error querying path ${pathSpec.path}: ${error}`);
allData[key] = [];
}
}
// Merge all path data into time-ordered rows
const mergedData = this.mergePathData(allData, query.pathSpecs);
return {
context,
range: {
from: fromIso,
to: toIso,
},
// Echo each filter (e.g. sourceRef) back per path. Cast covers the
// @signalk/server-api version gap until ValueList declares the field.
values: query.pathSpecs.map(ps => {
const echo = (0, path_filters_1.filterEcho)((0, path_filters_1.filtersFromFields)(ps));
return { path: ps.path, method: ps.aggregate, ...echo };
}),
data: mergedData,
};
}
/**
* Get available contexts for the time range
*/
async getContexts(query) {
const { from, to } = parseTimeRange(query);
this.debug(`[HistoryProvider] getContexts: from=${from}, to=${to}`);
const contexts = await (0, context_discovery_1.getAvailableContextsForTimeRange)(this.dataDir, from, to);
return contexts;
}
/**
* Get available paths for the time range
*/
async getPaths(query) {
const { from, to } = parseTimeRange(query);
this.debug(`[HistoryProvider] getPaths: from=${from}, to=${to}`);
// Extract context if present (PathsRequest type doesn't include context, but callers may pass it)
const queryContext = query.context;
const context = queryContext
? !queryContext ||
queryContext === 'vessels.self' ||
queryContext === 'self'
? `vessels.${this.selfId}`
: queryContext.replace(/ /gi, '')
: undefined;
const paths = (0, path_discovery_1.getAvailablePathsArray)(this.dataDir, this.app, context);
return paths;
}
/**
* Query a single path from parquet files
*/
async queryPath(context, pathSpec, fromIso, toIso, resolutionMs) {
// Use HivePathBuilder for correct Hive-partitioned paths
const hiveBuilder = new hive_path_builder_1.HivePathBuilder();
// Build glob pattern for Hive partitions
const filePath = hiveBuilder.getGlobPattern(this.dataDir, 'raw', context, pathSpec.path);
this.debug(`[HistoryProvider] Querying Hive path: ${filePath}`);
// Stage this path's buffer rows into a temp table if the buffer is available
const hasBuffer = duckdb_pool_1.DuckDBPool.isSQLiteBufferInitialized();
const connection = await duckdb_pool_1.DuckDBPool.getConnection();
try {
const stagedBufferTable = hasBuffer && this.sqliteBuffer
? await (0, buffer_staging_1.stageBufferTable)(connection, this.sqliteBuffer, String(context), String(pathSpec.path), fromIso, toIso, (msg) => this.debug(msg))
: null;
// Check if this is an object path (has value_* columns)
const componentSchema = await (0, schema_cache_1.getPathComponentSchema)(this.dataDir, context, pathSpec.path);
const aggFunc = this.getAggregateFunction(pathSpec.aggregate);
// Inline filters (e.g. sourceRef) come from the server-parsed PathSpec.
// The fields are populated by a newer @signalk/server-api than this plugin
// pins, so they are read defensively via the registry. This provider only
// queries raw-tier parquet; probe it for the filter columns so files
// without them are excluded rather than throwing.
const filters = (0, path_filters_1.filtersFromFields)(pathSpec);
const available = await (0, path_filters_1.availableFilterColumns)(connection, [filePath], filters);
const sourceFilter = (0, path_filters_1.buildParquetFilterClause)(filters, available);
// Build parquet FROM clause with filename filtering
const parquetFrom = `(SELECT * FROM read_parquet('${filePath}', union_by_name=true, filename=true) WHERE filename NOT LIKE '%/processed/%' AND filename NOT LIKE '%/quarantine/%' AND filename NOT LIKE '%/failed/%' AND filename NOT LIKE '%/repaired/%'${sourceFilter})`;
if (componentSchema && componentSchema.components.size > 0) {
// Object path - aggregate each component
const componentSelects = Array.from(componentSchema.components.entries())
.map(([name, comp]) => {
const compAggFunc = comp.dataType === 'numeric' ? aggFunc : 'FIRST';
// TRY_CAST handles mixed-type parquet files (some store lat/lon as VARCHAR)
const colExpr = comp.dataType === 'numeric'
? `TRY_CAST(${comp.columnName} AS DOUBLE)`
: comp.columnName;
return `${compAggFunc}(${colExpr}) as ${name}`;
})
.join(', ');
const componentWhereConditions = Array.from(componentSchema.components.values())
.map(comp => `${comp.columnName} IS NOT NULL`)
.join(' OR ');
const componentCols = Array.from(componentSchema.components.values())
.map(c => c.columnName)
.join(', ');
// Build federated FROM: parquet UNION ALL buffer
let federatedFrom;
if (stagedBufferTable) {
const bufferTableCols = this.sqliteBuffer?.getTableColumns(pathSpec.path);
const bufferSubquery = (0, buffer_sql_builder_1.buildBufferObjectSubquery)(stagedBufferTable, context, fromIso, toIso, componentSchema.components, bufferTableCols, filters);
federatedFrom = `(
SELECT signalk_timestamp, ${componentCols} FROM ${parquetFrom}
UNION ALL
SELECT signalk_timestamp, ${componentCols} FROM ${bufferSubquery}
)`;
}
else {
federatedFrom = parquetFrom;
}
const query = `
SELECT
strftime(DATE_TRUNC('seconds',
EPOCH_MS(CAST(FLOOR(EPOCH_MS(signalk_timestamp::TIMESTAMP) / ${resolutionMs}) * ${resolutionMs} AS BIGINT))
), '%Y-%m-%dT%H:%M:%SZ') as timestamp,
${componentSelects}
FROM ${federatedFrom} AS source_data
WHERE
signalk_timestamp >= '${fromIso}'
AND signalk_timestamp < '${toIso}'
AND (${componentWhereConditions})
GROUP BY timestamp
ORDER BY timestamp
`;
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
return rows.map((row) => {
const timestamp = row.timestamp;
// For navigation.position, return as [longitude, latitude] array for compatibility
// with plugins like signalk-pmtiles-plugin that expect this format
if (pathSpec.path === 'navigation.position' &&
row.longitude !== undefined &&
row.latitude !== undefined) {
return [timestamp, [row.longitude, row.latitude]];
}
// For other object paths, return as object
const obj = {};
componentSchema.components.forEach((_, name) => {
if (row[name] !== null && row[name] !== undefined) {
obj[name] = row[name];
}
});
return [timestamp, obj];
});
}
else {
// Scalar path — use vector averaging for angular paths when aggregating by average
const angular = (0, angular_paths_1.isAngularPath)(pathSpec.path, this.app, context);
const valueExpression = angular && (pathSpec.aggregate === 'average' || !pathSpec.aggregate)
? 'ATAN2(AVG(SIN(TRY_CAST(value AS DOUBLE))), AVG(COS(TRY_CAST(value AS DOUBLE))))'
: `${aggFunc}(TRY_CAST(value AS DOUBLE))`;
// Build federated FROM: parquet UNION ALL buffer
let federatedFrom;
if (stagedBufferTable) {
const bufferSubquery = (0, buffer_sql_builder_1.buildBufferScalarSubquery)(stagedBufferTable, context, pathSpec.path, fromIso, toIso, filters);
federatedFrom = `(
SELECT signalk_timestamp, value FROM ${parquetFrom}
UNION ALL
SELECT signalk_timestamp, value FROM ${bufferSubquery}
)`;
}
else {
federatedFrom = parquetFrom;
}
const query = `
SELECT
strftime(DATE_TRUNC('seconds',
EPOCH_MS(CAST(FLOOR(EPOCH_MS(signalk_timestamp::TIMESTAMP) / ${resolutionMs}) * ${resolutionMs} AS BIGINT))
), '%Y-%m-%dT%H:%M:%SZ') as timestamp,
${valueExpression} as value
FROM ${federatedFrom} AS source_data
WHERE
signalk_timestamp >= '${fromIso}'
AND signalk_timestamp < '${toIso}'
AND value IS NOT NULL
GROUP BY timestamp
ORDER BY timestamp
`;
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
return rows.map((row) => [row.timestamp, row.value]);
}
}
finally {
connection.disconnectSync();
}
}
/**
* Convert aggregate method to SQL function
*/
getAggregateFunction(method) {
switch (method) {
case 'average':
return 'AVG';
case 'min':
return 'MIN';
case 'max':
return 'MAX';
case 'first':
return 'FIRST';
case 'last':
return 'LAST';
case 'mid':
return 'MEDIAN';
case 'middle_index':
return 'FIRST'; // Fallback
default:
return 'AVG';
}
}
/**
* Merge data from multiple paths into time-aligned rows
*/
mergePathData(allData, pathSpecs) {
// Collect all unique timestamps
const timestampSet = new Set();
Object.values(allData).forEach(pathData => {
pathData.forEach(([ts]) => timestampSet.add(ts));
});
// Sort timestamps
const timestamps = Array.from(timestampSet).sort();
// One lookup map per spec (by position), each fetched with the same
// composite key used to store it, so duplicate paths with different
// sourceRef/aggregate remain distinct columns.
const specMaps = pathSpecs.map(ps => {
const map = new Map();
(allData[pathSpecKey(ps)] || []).forEach(([ts, val]) => map.set(ts, val));
return map;
});
// Build merged rows
return timestamps.map(ts => {
const row = [ts];
specMaps.forEach(map => row.push(map.get(ts) ?? null));
return row;
});
}
}
exports.HistoryProvider = HistoryProvider;
/**
* Register this plugin as the History API provider
*/
function registerHistoryApiProvider(app, selfId, dataDir, debug, sqliteBuffer) {
const provider = new HistoryProvider(selfId, dataDir, app, debug);
if (sqliteBuffer) {
provider.setSqliteBuffer(sqliteBuffer);
}
// Debug: Check if registerHistoryApiProvider exists on app
console.log('[signalk-parquet] app.registerHistoryApiProvider exists:', typeof app.registerHistoryApiProvider);
if (typeof app.registerHistoryApiProvider !== 'function') {
console.error('[signalk-parquet] ERROR: app.registerHistoryApiProvider is not a function!');
console.error('[signalk-parquet] Available app methods:', Object.keys(app)
.filter(k => typeof app[k] === 'function')
.join(', '));
return;
}
try {
app.registerHistoryApiProvider(provider);
debug('[HistoryProvider] Successfully registered as History API provider');
console.log('[signalk-parquet] Registered as SignalK History API provider');
}
catch (error) {
console.error('[signalk-parquet] Failed to register as History API provider:', error);
debug(`[HistoryProvider] Registration failed: ${error}`);
}
}
/**
* Unregister this plugin as the History API provider
*/
function unregisterHistoryApiProvider(app) {
try {
app.unregisterHistoryApiProvider();
console.log('[signalk-parquet] Unregistered as SignalK History API provider');
}
catch (error) {
// Ignore errors during unregistration
}
}
//# sourceMappingURL=history-provider.js.map