signalk-parquet
Version:
SignalK plugin to save marine data directly to Parquet files with regimen-based control
214 lines • 9.03 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.registerHistoryApiRoute = registerHistoryApiRoute;
const core_1 = require("@js-joda/core");
const node_api_1 = require("@duckdb/node-api");
const _1 = require(".");
const path_1 = __importDefault(require("path"));
function registerHistoryApiRoute(router, selfId, dataDir, debug) {
const historyApi = new HistoryAPI(selfId, dataDir);
router.get('/signalk/v1/history/values', (req, res) => {
const { from, to, context } = getRequestParams(req, selfId);
historyApi.getValues(context, from, to, debug, req, res);
});
router.get('/signalk/v1/history/contexts', (req, res) => {
//TODO implement retrieval of contexts for the given period
res.json([`vessels.${selfId}`]);
});
router.get('/signalk/v1/history/paths', (req, res) => {
//TODO implement retrieval of paths for the given period
// const { from, to } = getRequestParams(req as FromToContextRequest, selfId);
// getPaths(influx, from, to, res);
res.json(['navigation.speedOverGround']);
});
// Also register as plugin-style routes for testing
router.get('/api/history/values', (req, res) => {
const { from, to, context } = getRequestParams(req, selfId);
historyApi.getValues(context, from, to, debug, req, res);
});
router.get('/api/history/contexts', (req, res) => {
res.json([`vessels.${selfId}`]);
});
router.get('/api/history/paths', (req, res) => {
res.json(['navigation.speedOverGround']);
});
}
const getRequestParams = ({ query }, selfId) => {
try {
const from = core_1.ZonedDateTime.parse(query['from']);
const to = core_1.ZonedDateTime.parse(query['to']);
const context = getContext(query.context, selfId);
const bbox = query['bbox'];
return { from, to, context, bbox };
}
catch (e) {
throw new Error(`Error extracting from/to query parameters from ${JSON.stringify(query)}`);
}
};
function getContext(contextFromQuery, selfId) {
if (!contextFromQuery ||
contextFromQuery === 'vessels.self' ||
contextFromQuery === 'self') {
return `vessels.${selfId}`;
}
return contextFromQuery.replace(/ /gi, '');
}
class HistoryAPI {
constructor(selfId, dataDir) {
this.selfId = selfId;
this.dataDir = dataDir;
this.selfContextPath = (0, _1.toContextFilePath)(`vessels.${selfId}`);
}
async getValues(context, from, to, debug,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
req,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
res) {
try {
const timeResolutionMillis = (req.query.resolution
? Number.parseFloat(req.query.resolution)
: (to.toEpochSecond() - from.toEpochSecond()) / 500) * 1000;
const pathExpressions = (req.query.paths || '')
.replace(/[^0-9a-z.,:]/gi, '')
.split(',');
const pathSpecs = pathExpressions.map(splitPathExpression);
// Handle position and numeric paths together
const allResult = pathSpecs.length
? await this.getNumericValues(context, from, to, timeResolutionMillis, pathSpecs, debug)
: Promise.resolve({
context,
range: {
from: from.toString(),
to: to.toString(),
},
values: [],
data: [],
});
res.json(allResult);
}
catch (error) {
debug(`Error in getValues: ${error}`);
res.status(500).json({
error: 'Internal server error',
message: error instanceof Error ? error.message : String(error),
});
}
}
async getNumericValues(context, from, to, timeResolutionMillis, pathSpecs, debug) {
const allData = {};
// Process each path and collect data
await Promise.all(pathSpecs.map(async (pathSpec) => {
try {
// Sanitize the path to prevent directory traversal and SQL injection
const sanitizedPath = pathSpec.path
.replace(/[^a-zA-Z0-9._]/g, '') // Only allow alphanumeric, dots, underscores
.replace(/\./g, '/');
const filePath = path_1.default.join(this.dataDir, this.selfContextPath, sanitizedPath, '*.parquet');
// Convert ZonedDateTime to ISO string format matching parquet schema
const fromIso = from.toInstant().toString();
const toIso = to.toInstant().toString();
// Build query with sanitized inputs (filePath is already sanitized above)
const query = `
SELECT
signalk_timestamp,
value,
value_json
FROM '${filePath}'
WHERE
signalk_timestamp >= '${fromIso}'
AND
signalk_timestamp < '${toIso}'
ORDER BY signalk_timestamp
`;
debug(`Executing query for path ${pathSpec.path}: ${query}`);
const duckDB = await node_api_1.DuckDBInstance.create();
const connection = await duckDB.connect();
try {
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
// Convert rows to the expected format
const pathData = rows.map((row) => {
const rowData = row;
const timestamp = rowData.signalk_timestamp;
// Handle both JSON values (like position objects) and simple values
const value = rowData.value_json
? JSON.parse(String(rowData.value_json))
: rowData.value;
// For position paths, ensure we return the full position object
if (pathSpec.path === 'navigation.position' &&
value &&
typeof value === 'object') {
// Position data is already an object with latitude/longitude
// No reassignment needed, keeping original value
}
return [timestamp, value];
});
allData[pathSpec.path] = pathData;
debug(`Retrieved ${pathData.length} data points for ${pathSpec.path}`);
}
finally {
connection.disconnectSync();
}
}
catch (error) {
debug(`Error querying path ${pathSpec.path}: ${error}`);
allData[pathSpec.path] = [];
}
}));
// Merge all path data into time-ordered rows
const mergedData = this.mergePathData(allData, pathSpecs);
return {
context,
range: {
from: from.toString(),
to: to.toString(),
},
values: pathSpecs.map(({ path, aggregateMethod }) => ({
path,
method: aggregateMethod,
})),
data: mergedData,
};
}
mergePathData(allData, pathSpecs) {
// Create a map of all unique timestamps
const timestampMap = new Map();
pathSpecs.forEach((pathSpec, index) => {
const pathData = allData[pathSpec.path] || [];
pathData.forEach(([timestamp, value]) => {
const timestampStr = timestamp.toString();
if (!timestampMap.has(timestampStr)) {
timestampMap.set(timestampStr, new Array(pathSpecs.length).fill(null));
}
timestampMap.get(timestampStr)[index] = value;
});
});
// Convert to sorted array format
return Array.from(timestampMap.entries())
.sort(([a], [b]) => a.localeCompare(b))
.map(([timestamp, values]) => [timestamp, ...values]);
}
}
function splitPathExpression(pathExpression) {
const parts = pathExpression.split(':');
let aggregateMethod = (parts[1] || 'average');
if (parts[0] === 'navigation.position') {
aggregateMethod = 'first';
}
return {
path: parts[0],
queryResultName: parts[0].replace(/\./g, '_'),
aggregateMethod,
aggregateFunction: functionForAggregate[aggregateMethod] || 'mean()',
};
}
const functionForAggregate = {
average: 'avg',
min: 'min',
max: 'max',
first: 'first',
};
//# sourceMappingURL=HistoryAPI.js.map