signalk-parquet
Version:
SignalK plugin to save marine data directly to Parquet files with regimen-based control
425 lines • 18.5 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HistoryAPI = void 0;
exports.registerHistoryApiRoute = registerHistoryApiRoute;
const core_1 = require("@js-joda/core");
const node_api_1 = require("@duckdb/node-api");
const _1 = require(".");
const path_1 = __importDefault(require("path"));
const path_discovery_1 = require("./utils/path-discovery");
function registerHistoryApiRoute(router, selfId, dataDir, debug, app) {
const historyApi = new HistoryAPI(selfId, dataDir);
router.get('/signalk/v1/history/values', (req, res) => {
const { from, to, context, shouldRefresh } = getRequestParams(req, selfId);
historyApi.getValues(context, from, to, shouldRefresh, debug, req, res);
});
router.get('/signalk/v1/history/contexts', (req, res) => {
//TODO implement retrieval of contexts for the given period
res.json([`vessels.${selfId}`]);
});
router.get('/signalk/v1/history/paths', (req, res) => {
try {
const paths = (0, path_discovery_1.getAvailablePathsArray)(dataDir, app);
res.json(paths);
}
catch (error) {
res.status(500).json({ error: error.message });
}
});
// Also register as plugin-style routes for testing
router.get('/api/history/values', (req, res) => {
const { from, to, context, shouldRefresh } = getRequestParams(req, selfId);
historyApi.getValues(context, from, to, shouldRefresh, debug, req, res);
});
router.get('/api/history/contexts', (req, res) => {
res.json([`vessels.${selfId}`]);
});
router.get('/api/history/paths', (req, res) => {
res.json(['navigation.speedOverGround']);
});
}
const getRequestParams = ({ query }, selfId) => {
try {
let from;
let to;
let shouldRefresh = false;
// Check if user wants to work in UTC (default: false, use local timezone)
const useUTC = query.useUTC === 'true' || query.useUTC === '1';
// Handle new backwards querying with start + duration
if (query.start && query.duration) {
const durationMs = parseDuration(query.duration);
if (query.start === 'now') {
// Always use current UTC time for 'now' regardless of useUTC setting
to = core_1.ZonedDateTime.now(core_1.ZoneOffset.UTC);
from = to.minusNanos(durationMs * 1000000); // Convert ms to nanoseconds
shouldRefresh = query.refresh === 'true' || query.refresh === '1';
}
else {
// Parse start time with timezone conversion if needed
to = parseDateTime(query.start, useUTC);
from = to.minusNanos(durationMs * 1000000);
}
}
else if (query.from && query.to) {
// Traditional from/to querying (forward in time) with timezone conversion
from = parseDateTime(query.from, useUTC);
to = parseDateTime(query.to, useUTC);
}
else {
throw new Error('Either (from + to) or (start + duration) parameters are required');
}
const context = getContext(query.context, selfId);
const bbox = query.bbox;
return { from, to, context, bbox, shouldRefresh };
}
catch (e) {
console.error('Full error details:', e);
throw new Error(`Error extracting query parameters from ${JSON.stringify(query)}: ${e instanceof Error ? e.stack : e}`);
}
};
// Parse duration string (e.g., "1h", "30m", "5s", "2d")
function parseDuration(duration) {
const match = duration.match(/^(\d+)([smhd])$/);
if (!match) {
throw new Error(`Invalid duration format: ${duration}. Use format like "1h", "30m", "5s", "2d"`);
}
const value = parseInt(match[1]);
const unit = match[2];
switch (unit) {
case 's': return value * 1000; // seconds to milliseconds
case 'm': return value * 60 * 1000; // minutes to milliseconds
case 'h': return value * 60 * 60 * 1000; // hours to milliseconds
case 'd': return value * 24 * 60 * 60 * 1000; // days to milliseconds
default: throw new Error(`Unknown duration unit: ${unit}`);
}
}
// Check if datetime string has timezone information
function hasTimezoneInfo(dateTimeStr) {
// Check for 'Z' at the end, or '+'/'-' followed by timezone offset pattern
return dateTimeStr.endsWith('Z') ||
/[+-]\d{2}:?\d{2}$/.test(dateTimeStr) ||
/[+-]\d{4}$/.test(dateTimeStr);
}
// Parse datetime string and convert to UTC if needed
function parseDateTime(dateTimeStr, useUTC) {
// Normalize the datetime string to include seconds if missing
let normalizedStr = dateTimeStr;
if (dateTimeStr.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$/)) {
// Add seconds if only HH:MM is provided
normalizedStr = dateTimeStr + ':00';
}
if (useUTC) {
// When useUTC=true, treat the datetime as UTC
if (hasTimezoneInfo(normalizedStr)) {
// Already has timezone info, parse as-is
return core_1.ZonedDateTime.parse(normalizedStr);
}
else {
// No timezone info, assume UTC by adding 'Z'
return core_1.ZonedDateTime.parse(normalizedStr + 'Z');
}
}
else {
// When useUTC=false, handle timezone conversion
if (hasTimezoneInfo(normalizedStr)) {
// Already has timezone info, parse as-is (will be in UTC or specified timezone)
return core_1.ZonedDateTime.parse(normalizedStr).withZoneSameInstant(core_1.ZoneOffset.UTC);
}
else {
// No timezone info, treat as local time and convert to UTC
try {
// JavaScript Date constructor treats ISO strings without timezone as local time
const localDate = new Date(normalizedStr);
if (isNaN(localDate.getTime())) {
throw new Error('Invalid date');
}
// Convert to UTC ISO string and parse with ZonedDateTime
const utcIsoString = localDate.toISOString();
return core_1.ZonedDateTime.parse(utcIsoString);
}
catch (e) {
throw new Error(`Unable to parse datetime '${dateTimeStr}': ${e}. Use format like '2025-08-13T08:00:00' or '2025-08-13T08:00:00Z'`);
}
}
}
}
function getContext(contextFromQuery, selfId) {
if (!contextFromQuery ||
contextFromQuery === 'vessels.self' ||
contextFromQuery === 'self') {
return `vessels.${selfId}`;
}
return contextFromQuery.replace(/ /gi, '');
}
class HistoryAPI {
constructor(selfId, dataDir) {
this.selfId = selfId;
this.dataDir = dataDir;
this.selfContextPath = (0, _1.toContextFilePath)(`vessels.${selfId}`);
}
async getValues(context, from, to, shouldRefresh, debug,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
req,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
res) {
try {
const timeResolutionMillis = req.query.resolution
? Number.parseFloat(req.query.resolution)
: (to.toEpochSecond() - from.toEpochSecond()) / 500 * 1000;
const pathExpressions = (req.query.paths || '')
.replace(/[^0-9a-z.,:_]/gi, '')
.split(',');
const pathSpecs = pathExpressions.map(splitPathExpression);
// Handle position and numeric paths together
const allResult = pathSpecs.length
? await this.getNumericValues(context, from, to, timeResolutionMillis, pathSpecs, debug)
: Promise.resolve({
context,
range: {
from: from.toString(),
to: to.toString(),
},
values: [],
data: [],
});
// Add refresh headers if shouldRefresh is enabled
if (shouldRefresh) {
const refreshIntervalSeconds = Math.max(Math.round(timeResolutionMillis / 1000), 1); // At least 1 second
res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate');
res.setHeader('Pragma', 'no-cache');
res.setHeader('Expires', '0');
res.setHeader('Refresh', refreshIntervalSeconds.toString());
// Add refresh info to response
allResult.refresh = {
enabled: true,
intervalSeconds: refreshIntervalSeconds,
nextRefresh: new Date(Date.now() + refreshIntervalSeconds * 1000).toISOString()
};
}
res.json(allResult);
}
catch (error) {
debug(`Error in getValues: ${error}`);
res.status(500).json({
error: 'Internal server error',
message: error instanceof Error ? error.message : String(error),
});
}
}
async getNumericValues(context, from, to, timeResolutionMillis, pathSpecs, debug) {
const allData = {};
// Process each path and collect data
await Promise.all(pathSpecs.map(async (pathSpec) => {
try {
// Sanitize the path to prevent directory traversal and SQL injection
const sanitizedPath = pathSpec.path
.replace(/[^a-zA-Z0-9._]/g, '') // Only allow alphanumeric, dots, underscores
.replace(/\./g, '/');
const filePath = path_1.default.join(this.dataDir, this.selfContextPath, sanitizedPath, '*.parquet');
// Convert ZonedDateTime to ISO string format matching parquet schema
const fromIso = from.toInstant().toString();
const toIso = to.toInstant().toString();
// Build query with time bucketing - fix type casting
const query = `
SELECT
strftime(DATE_TRUNC('seconds',
EPOCH_MS(CAST(FLOOR(EPOCH_MS(signalk_timestamp::TIMESTAMP) / ${timeResolutionMillis}) * ${timeResolutionMillis} AS BIGINT))
), '%Y-%m-%dT%H:%M:%SZ') as timestamp,
${getAggregateExpression(pathSpec.aggregateMethod, pathSpec.path)} as value,
FIRST(value_json) as value_json
FROM '${filePath}'
WHERE
signalk_timestamp >= '${fromIso}'
AND
signalk_timestamp < '${toIso}'
AND (value IS NOT NULL OR value_json IS NOT NULL)
GROUP BY timestamp
ORDER BY timestamp
`;
const duckDB = await node_api_1.DuckDBInstance.create();
const connection = await duckDB.connect();
try {
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
// Convert rows to the expected format using bucketed timestamps
const pathData = rows.map((row) => {
const rowData = row;
const { timestamp } = rowData;
// Handle both JSON values (like position objects) and simple values
const value = rowData.value_json
? JSON.parse(String(rowData.value_json))
: rowData.value;
// For position paths, ensure we return the full position object
if (pathSpec.path === 'navigation.position' &&
value &&
typeof value === 'object') {
// Position data is already an object with latitude/longitude
// No reassignment needed, keeping original value
}
return [timestamp, value];
});
allData[pathSpec.path] = pathData;
}
finally {
connection.disconnectSync();
}
}
catch (error) {
debug(`Error querying path ${pathSpec.path}: ${error}`);
allData[pathSpec.path] = [];
}
}));
// Merge all path data into time-ordered rows
const mergedData = this.mergePathData(allData, pathSpecs);
// Add EMA and SMA calculations to numeric columns
const enhancedData = this.addMovingAverages(mergedData, pathSpecs);
return {
context,
range: {
from: from.toString(),
to: to.toString(),
},
values: this.buildValuesWithMovingAverages(pathSpecs),
data: enhancedData,
};
}
mergePathData(allData, pathSpecs) {
// Create a map of all unique timestamps
const timestampMap = new Map();
pathSpecs.forEach((pathSpec, index) => {
const pathData = allData[pathSpec.path] || [];
pathData.forEach(([timestamp, value]) => {
if (!timestampMap.has(timestamp)) {
timestampMap.set(timestamp, new Array(pathSpecs.length).fill(null));
}
timestampMap.get(timestamp)[index] = value;
});
});
// Convert to sorted array format
return Array.from(timestampMap.entries())
.sort(([a], [b]) => a.localeCompare(b))
.map(([timestamp, values]) => [timestamp, ...values]);
}
addMovingAverages(data, pathSpecs) {
if (data.length === 0)
return data;
const smaPeriod = 10;
const emaAlpha = 0.2;
// For each column, track EMA and SMA state
const columnEMAs = new Array(pathSpecs.length).fill(null);
const columnSMAWindows = pathSpecs.map(() => []);
return data.map((row, rowIndex) => {
const [timestamp, ...values] = row;
const enhancedValues = [];
values.forEach((value, colIndex) => {
enhancedValues.push(value);
// Calculate EMA and SMA for numeric values only
if (typeof value === 'number' && !isNaN(value)) {
// Calculate EMA
if (columnEMAs[colIndex] === null) {
columnEMAs[colIndex] = value; // First value
}
else {
columnEMAs[colIndex] = emaAlpha * value + (1 - emaAlpha) * columnEMAs[colIndex];
}
// Calculate SMA
columnSMAWindows[colIndex].push(value);
if (columnSMAWindows[colIndex].length > smaPeriod) {
columnSMAWindows[colIndex] = columnSMAWindows[colIndex].slice(-smaPeriod);
}
const sma = columnSMAWindows[colIndex].reduce((sum, val) => sum + val, 0) / columnSMAWindows[colIndex].length;
// Add EMA and SMA as additional values (rounded to 3 decimal places)
enhancedValues.push(Math.round(columnEMAs[colIndex] * 1000) / 1000); // EMA
enhancedValues.push(Math.round(sma * 1000) / 1000); // SMA
}
else {
// Non-numeric values get null for EMA/SMA
enhancedValues.push(null); // EMA
enhancedValues.push(null); // SMA
}
});
return [timestamp, ...enhancedValues];
});
}
buildValuesWithMovingAverages(pathSpecs) {
const result = [];
pathSpecs.forEach(({ path, aggregateMethod }) => {
// Add original path
result.push({ path, method: aggregateMethod });
// Add EMA and SMA paths for this column
result.push({ path: `${path}.ema`, method: 'ema' });
result.push({ path: `${path}.sma`, method: 'sma' });
});
return result;
}
}
exports.HistoryAPI = HistoryAPI;
function splitPathExpression(pathExpression) {
const parts = pathExpression.split(':');
let aggregateMethod = (parts[1] || 'average');
// Auto-select appropriate default method for complex data types
if (parts[0] === 'navigation.position' && !parts[1]) {
aggregateMethod = 'first';
}
// Validate the aggregation method
const validMethods = ['average', 'min', 'max', 'first', 'last', 'mid', 'middle_index'];
if (parts[1] && !validMethods.includes(parts[1])) {
aggregateMethod = 'average';
}
return {
path: parts[0],
queryResultName: parts[0].replace(/\./g, '_'),
aggregateMethod,
aggregateFunction: functionForAggregate[aggregateMethod] || 'avg',
};
}
const functionForAggregate = {
average: 'avg',
min: 'min',
max: 'max',
first: 'first',
last: 'last',
mid: 'median',
middle_index: 'nth_value',
};
function getAggregateFunction(method) {
switch (method) {
case 'average':
return 'AVG';
case 'min':
return 'MIN';
case 'max':
return 'MAX';
case 'first':
return 'FIRST';
case 'last':
return 'LAST';
case 'mid':
return 'MEDIAN';
case 'middle_index':
return 'NTH_VALUE';
default:
return 'AVG';
}
}
function getValueExpression(pathName) {
// For position data, use value_json since the value is an object
if (pathName === 'navigation.position') {
return 'value_json';
}
// For numeric data, try to cast to DOUBLE, fallback to the original value
return 'TRY_CAST(value AS DOUBLE)';
}
function getAggregateExpression(method, pathName) {
const valueExpr = getValueExpression(pathName);
if (method === 'middle_index') {
// For middle_index, use FIRST as a simple fallback for now
// TODO: Implement proper middle index selection
return `FIRST(${valueExpr})`;
}
return `${getAggregateFunction(method)}(${valueExpr})`;
}
//# sourceMappingURL=HistoryAPI.js.map