signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
241 lines • 10.1 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.getAvailableContextsForTimeRange = getAvailableContextsForTimeRange;
exports.getContextsInSpatialFilter = getContextsInSpatialFilter;
exports.clearFileListCache = clearFileListCache;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const debug_logger_1 = require("./debug-logger");
const cache_defaults_1 = require("../config/cache-defaults");
const hive_path_builder_1 = require("./hive-path-builder");
const duckdb_pool_1 = require("./duckdb-pool");
const spatial_queries_1 = require("./spatial-queries");
let contextListCache = null;
const hiveBuilder = new hive_path_builder_1.HivePathBuilder();
/**
* Read context=* directory names under tier=raw/ and unsanitize them.
* No file scanning needed — just directory name reading.
*/
async function discoverContextsFromHiveDirs(dataDir) {
const tierRawDir = path.join(dataDir, 'tier=raw');
try {
const entries = await fs.readdir(tierRawDir, { withFileTypes: true });
const contexts = [];
for (const entry of entries) {
if (!entry.isDirectory())
continue;
const match = entry.name.match(/^context=(.+)$/);
if (match) {
const unsanitized = hiveBuilder.unsanitizeContext(match[1]);
contexts.push(unsanitized);
}
}
return contexts.sort();
}
catch (error) {
debug_logger_1.debugLogger.error('[Context Discovery] Error reading tier=raw/ directory:', error);
return [];
}
}
/**
* Get available SignalK contexts that have data within a specific time range.
* Reads context=* dirs under tier=raw/, then filters by checking if any
* year=YYYY/day=DDD subdirectories fall within the from/to range.
*/
async function getAvailableContextsForTimeRange(dataDir, from, to) {
try {
// Get all contexts (cached)
const now = Date.now();
let allContexts;
if (contextListCache &&
contextListCache.dataDir === dataDir &&
now - contextListCache.timestamp < cache_defaults_1.CACHE_TTL.FILE_LIST) {
allContexts = contextListCache.contexts;
debug_logger_1.debugLogger.log(`[Context Discovery] Using cached context list (${allContexts.length} contexts, age: ${Math.round((now - contextListCache.timestamp) / 1000)}s)`);
}
else {
debug_logger_1.debugLogger.log(`[Context Discovery] Scanning hive directories for contexts...`);
allContexts = await discoverContextsFromHiveDirs(dataDir);
contextListCache = {
contexts: allContexts,
timestamp: now,
dataDir,
};
debug_logger_1.debugLogger.log(`[Context Discovery] Cached ${allContexts.length} contexts`);
}
if (allContexts.length === 0) {
return [];
}
// Compute from/to as year + dayOfYear for range comparison
const fromDate = new Date(from.toInstant().toString());
const toDate = new Date(to.toInstant().toString());
const fromYearDay = dateToYearDay(fromDate);
const toYearDay = dateToYearDay(toDate);
// Filter contexts by checking if they have any matching year/day subdirs
const tierRawDir = path.join(dataDir, 'tier=raw');
const matchingContexts = [];
for (const context of allContexts) {
const sanitizedContext = hiveBuilder.sanitizeContext(context);
const contextDir = path.join(tierRawDir, `context=${sanitizedContext}`);
const hasData = await contextHasDataInRange(contextDir, fromYearDay, toYearDay);
if (hasData) {
matchingContexts.push(context);
}
}
debug_logger_1.debugLogger.log(`[Context Discovery] Found ${matchingContexts.length} contexts with data in time range`);
return matchingContexts.sort();
}
catch (error) {
debug_logger_1.debugLogger.error('Error scanning contexts:', error);
return [];
}
}
/**
* Convert a Date to { year, day } for range comparison.
* Matches HivePathBuilder.getDayOfYear() logic exactly.
*/
function dateToYearDay(d) {
const year = d.getUTCFullYear();
const start = new Date(Date.UTC(year, 0, 0)); // day 0 — same as HivePathBuilder
const day = Math.floor((d.getTime() - start.getTime()) / (24 * 60 * 60 * 1000));
return { year, day };
}
/**
* Check if a year/day falls within [from, to] range (inclusive).
*/
function yearDayInRange(year, day, from, to) {
const val = year * 1000 + day;
return val >= from.year * 1000 + from.day && val <= to.year * 1000 + to.day;
}
/**
* Check if a context directory has any path=* / year=YYYY / day=DDD
* subdirectories that fall within the from/to range.
*/
async function contextHasDataInRange(contextDir, from, to) {
try {
const pathEntries = await fs.readdir(contextDir, { withFileTypes: true });
for (const pathEntry of pathEntries) {
if (!pathEntry.isDirectory() || !pathEntry.name.startsWith('path=')) {
continue;
}
const pathDir = path.join(contextDir, pathEntry.name);
let yearEntries;
try {
yearEntries = await fs.readdir(pathDir, { withFileTypes: true });
}
catch {
continue;
}
for (const yearEntry of yearEntries) {
if (!yearEntry.isDirectory())
continue;
const yearMatch = yearEntry.name.match(/^year=(\d+)$/);
if (!yearMatch)
continue;
const year = parseInt(yearMatch[1], 10);
// Quick skip: if this year is entirely outside the range
if (year < from.year || year > to.year)
continue;
const yearDir = path.join(pathDir, yearEntry.name);
let dayEntries;
try {
dayEntries = await fs.readdir(yearDir, { withFileTypes: true });
}
catch {
continue;
}
for (const dayEntry of dayEntries) {
if (!dayEntry.isDirectory())
continue;
const dayMatch = dayEntry.name.match(/^day=(\d+)$/);
if (!dayMatch)
continue;
const day = parseInt(dayMatch[1], 10);
if (yearDayInRange(year, day, from, to)) {
return true;
}
}
}
}
}
catch {
// Context dir doesn't exist or can't be read
}
return false;
}
/**
* Find all vessel contexts with position data inside a spatial filter and time range.
* Accepts bbox ("west,south,east,north") or radius ("lon,lat,meters") via SpatialFilter.
* Uses a single DuckDB query with hive_partitioning on navigation__position files only.
*/
async function getContextsInSpatialFilter(dataDir, from, to, filter) {
const fromIso = from.toInstant().toString();
const toIso = to.toInstant().toString();
const glob = path.join(dataDir, 'tier=raw', 'context=*', 'path=navigation__position', 'year=*', 'day=*', '*.parquet');
const spatialClause = (0, spatial_queries_1.buildSpatialSqlClause)(filter);
const query = `
SELECT DISTINCT context
FROM read_parquet('${glob}', hive_partitioning=true, union_by_name=true)
WHERE signalk_timestamp >= '${fromIso}'
AND signalk_timestamp < '${toIso}'
AND ${spatialClause}
`;
const connection = await duckdb_pool_1.DuckDBPool.getConnection();
try {
debug_logger_1.debugLogger.log(`[Context Discovery] Spatial query (${filter.type}): ${JSON.stringify(filter.bbox)}`);
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
const contexts = rows
.map(row => {
const sanitized = row.context;
return hiveBuilder.unsanitizeContext(sanitized);
})
.sort();
debug_logger_1.debugLogger.log(`[Context Discovery] Found ${contexts.length} contexts in spatial filter`);
return contexts;
}
finally {
connection.disconnectSync();
}
}
/**
* Clear the context list cache (useful for testing or when data structure changes)
*/
function clearFileListCache() {
contextListCache = null;
debug_logger_1.debugLogger.log('[Context Discovery] Context list cache cleared');
}
//# sourceMappingURL=context-discovery.js.map