signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
172 lines • 6.9 kB
JavaScript
;
/**
* Inline per-path query filters for the History API.
*
* A request can narrow a single path to rows whose stored column matches a
* value. The first such filter is `sourceRef`:
*
* navigation.headingMagnetic:average|n2k-on-ve.can0.115
*
* filters to the source whose `$source` (stored in the `source_label` column)
* is `n2k-on-ve.can0.115`.
*
* ── Adding another filter ────────────────────────────────────────────────
* Add one entry to PATH_FILTER_DEFS below. Everything else in the read path is
* generic over the filter list: raw-tier forcing, parquet/buffer SQL, parquet
* schema probing, result keying, and the response echo all iterate the filters
* and need no per-filter changes.
*
* Requirements for a new filter:
* - `column` must be written into raw-tier parquet AND the SQLite buffer.
* The `source_*` columns already are (see data-handler.ts / sqlite-buffer.ts).
* - `delimiter` must be a single character that cannot appear in a SignalK
* path or aggregate token (the path sanitiser in HistoryAPI must also allow
* it, alongside `|`).
* - `field` is the property name echoed back in each response `values` entry.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.FILTER_DELIMITERS = exports.PATH_FILTER_DEFS = void 0;
exports.parsePathFilters = parsePathFilters;
exports.filtersFromFields = filtersFromFields;
exports.filterColumns = filterColumns;
exports.buildParquetFilterClause = buildParquetFilterClause;
exports.buildBufferFilterClause = buildBufferFilterClause;
exports.filterEcho = filterEcho;
exports.parquetHasColumn = parquetHasColumn;
exports.availableFilterColumns = availableFilterColumns;
const sql_escape_1 = require("./sql-escape");
/**
* The registry of supported inline path filters — the single place to add one.
*/
exports.PATH_FILTER_DEFS = [
// `path:aggregate|sourceRef` — restrict to one SignalK source.
{ delimiter: '|', field: 'sourceRef', column: 'source_label' },
];
const DEF_BY_DELIMITER = new Map(exports.PATH_FILTER_DEFS.map(def => [def.delimiter, def]));
/** The set of delimiter characters, for the path sanitiser to preserve. */
exports.FILTER_DELIMITERS = exports.PATH_FILTER_DEFS.map(d => d.delimiter).join('');
/**
* Split inline filters off a path expression, returning the base expression
* (path + aggregate + smoothing) and the parsed filters.
*
* Input: 'navigation.position:first|gps-1'
* Output: { base: 'navigation.position:first',
* filters: [{ field: 'sourceRef', column: 'source_label', value: 'gps-1' }] }
*/
function parsePathFilters(expr) {
// The base expression ends at the first registered delimiter.
let firstDelim = -1;
for (let i = 0; i < expr.length; i++) {
if (DEF_BY_DELIMITER.has(expr[i])) {
firstDelim = i;
break;
}
}
if (firstDelim < 0) {
return { base: expr, filters: [] };
}
const base = expr.substring(0, firstDelim);
const filters = [];
let i = firstDelim;
while (i < expr.length) {
const def = DEF_BY_DELIMITER.get(expr[i]);
let end = i + 1;
while (end < expr.length && !DEF_BY_DELIMITER.has(expr[end])) {
end++;
}
const value = expr.substring(i + 1, end);
if (value.length > 0) {
filters.push({ field: def.field, column: def.column, value });
}
i = end;
}
return { base, filters };
}
/**
* Build filters from an already-parsed spec object (e.g. a server-provided
* History API PathSpec) by reading each registered field off the object. Used
* by the v2 provider, where the server — not this plugin — parses the request.
*/
function filtersFromFields(spec) {
const filters = [];
for (const def of exports.PATH_FILTER_DEFS) {
const value = spec[def.field];
if (typeof value === 'string' && value.length > 0) {
filters.push({ field: def.field, column: def.column, value });
}
}
return filters;
}
/** Distinct stored columns referenced by the given filters. */
function filterColumns(filters) {
return [...new Set(filters.map(f => f.column))];
}
/**
* Build the parquet-side filter fragment to append to a WHERE clause.
*
* For each filter: if its column exists in `availableColumns`, append
* `AND column = 'value'`; otherwise append `AND 1=0` so the parquet side
* contributes nothing — legacy/imported files without the column cannot match,
* and the always-tagged SQLite buffer answers through its own filtered subquery.
*/
function buildParquetFilterClause(filters, availableColumns) {
return filters
.map(f => availableColumns.has(f.column)
? ` AND ${f.column} = '${(0, sql_escape_1.escapeSqlString)(f.value)}'`
: ' AND 1=0')
.join('');
}
/**
* Build the buffer-side filter fragment. The buffer schema always carries the
* registered filter columns, so no existence check is needed.
*/
function buildBufferFilterClause(filters) {
if (!filters || filters.length === 0) {
return '';
}
return filters
.map(f => `\n AND ${f.column} = '${(0, sql_escape_1.escapeSqlString)(f.value)}'`)
.join('');
}
/** Response-echo properties for a set of filters, e.g. { sourceRef: '...' }. */
function filterEcho(filters) {
return Object.fromEntries(filters.map(f => [f.field, f.value]));
}
/**
* Returns true if any of the given parquet globs exposes `column`. Null/empty
* paths, globs that match no files, and probe errors are treated as "absent",
* so a query that reads from S3 isn't penalised by a local glob that matches no
* files (and vice versa).
*/
async function parquetHasColumn(connection, filePaths, column) {
for (const filePath of filePaths) {
if (!filePath) {
continue;
}
try {
const result = await connection.runAndReadAll(`SELECT 1 FROM parquet_schema('${filePath}') WHERE name = '${column}' LIMIT 1`);
if (result.getRowObjects().length > 0) {
return true;
}
}
catch {
// Path may match no files; treat as absent and try the next.
}
}
return false;
}
/**
* The subset of the filters' columns that actually exist in the given parquet
* globs. Pass the result to buildParquetFilterClause so absent columns become
* `AND 1=0` rather than a "column not found" error.
*/
async function availableFilterColumns(connection, filePaths, filters) {
const available = new Set();
for (const column of filterColumns(filters)) {
if (await parquetHasColumn(connection, filePaths, column)) {
available.add(column);
}
}
return available;
}
//# sourceMappingURL=path-filters.js.map