signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
309 lines • 12.6 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.getAvailablePaths = getAvailablePaths;
exports.getAvailablePathsArray = getAvailablePathsArray;
exports.getAvailablePathsForTimeRange = getAvailablePathsForTimeRange;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const hive_path_builder_1 = require("./hive-path-builder");
const duckdb_pool_1 = require("./duckdb-pool");
/**
* Get available SignalK paths from Hive directory structure
* Scans tier=raw/context={ctx}/path={path}/ and returns paths that contain data files
*/
function getAvailablePaths(dataDir, app, context,
// When false, a path is included as soon as one parquet file is found rather
// than counting every file. countParquetFilesRecursive descends every
// year=/day= partition and stats every file — on a large store that is
// millions of synchronous calls that block the event loop. Callers that
// display the count (UI, analyzer) keep the default; the History API path
// list, which discards the count, passes false. Both return the same paths.
countFiles = true) {
const paths = [];
const hiveBuilder = new hive_path_builder_1.HivePathBuilder();
// Scan Hive structure: tier=raw/context=*/path=*/
const hiveRawDir = path.join(dataDir, 'tier=raw');
if (!fs.existsSync(hiveRawDir)) {
return paths;
}
try {
// Get target context sanitized for matching
const targetContext = context || app.selfContext;
const sanitizedTargetContext = hiveBuilder.sanitizeContext(targetContext);
// Iterate context= directories
const contextDirs = fs.readdirSync(hiveRawDir);
for (const contextDir of contextDirs) {
if (!contextDir.startsWith('context='))
continue;
const contextPath = path.join(hiveRawDir, contextDir);
const stat = fs.statSync(contextPath);
if (!stat.isDirectory())
continue;
// Extract and unsanitize context name
const sanitizedContext = contextDir.replace('context=', '');
// Only include paths for target context
if (sanitizedContext !== sanitizedTargetContext)
continue;
// Iterate path= directories within each context
const pathDirs = fs.readdirSync(contextPath);
for (const pathDir of pathDirs) {
if (!pathDir.startsWith('path='))
continue;
const pathPath = path.join(contextPath, pathDir);
const pathStat = fs.statSync(pathPath);
if (!pathStat.isDirectory())
continue;
// Extract and unsanitize path name
const sanitizedPath = pathDir.replace('path=', '');
const unsanitizedPath = hiveBuilder.unsanitizePath(sanitizedPath);
// Count parquet files across all year=/day=/ subdirs, or just confirm
// at least one exists when the count is not needed (far cheaper, same
// inclusion result).
const fileCount = countFiles
? countParquetFilesRecursive(pathPath)
: hasParquetFilesRecursiveSync(pathPath)
? 1
: 0;
if (fileCount > 0) {
paths.push({
path: unsanitizedPath,
directory: pathPath,
fileCount: fileCount,
});
}
}
}
}
catch (error) {
// Error reading directory - skip
}
return paths;
}
/**
* Count parquet files recursively in a directory
*/
function countParquetFilesRecursive(dir) {
let count = 0;
try {
const items = fs.readdirSync(dir);
for (const item of items) {
const fullPath = path.join(dir, item);
const stat = fs.statSync(fullPath);
if (stat.isDirectory()) {
count += countParquetFilesRecursive(fullPath);
}
else if (item.endsWith('.parquet')) {
count++;
}
}
}
catch (error) {
// Error reading directory - skip
}
return count;
}
/**
* Return true as soon as a single parquet file is found anywhere under `dir`.
* Mirrors countParquetFilesRecursive's traversal exactly (recurses every
* subdirectory, no special-dir skipping) so it includes the same paths, but
* short-circuits on the first hit and uses Dirent types (falling back to stat
* only for symlinks/unknown entries) instead of a statSync per entry — turning
* a full census into a handful of readdirs.
*/
function hasParquetFilesRecursiveSync(dir) {
try {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
let isDir = entry.isDirectory();
let isParquet = !isDir && entry.isFile() && entry.name.endsWith('.parquet');
// Dirent types are unreliable for symlinks (reported as the link, not its
// target) and on filesystems that return unknown types. For those rare
// entries fall back to stat (which follows the link) so traversal matches
// countParquetFilesRecursive. The data tree has no symlinks in practice,
// so the fast Dirent path covers essentially everything.
if (!isDir && !isParquet && !entry.isFile()) {
try {
isDir = fs.statSync(path.join(dir, entry.name)).isDirectory();
isParquet = !isDir && entry.name.endsWith('.parquet');
}
catch {
// broken symlink / unreadable entry — skip
}
}
if (isDir) {
if (hasParquetFilesRecursiveSync(path.join(dir, entry.name))) {
return true;
}
}
else if (isParquet) {
return true;
}
}
}
catch (error) {
// Error reading directory - skip
}
return false;
}
/**
* Get available SignalK paths as simple string array
* Useful for SignalK history API compliance
*/
function getAvailablePathsArray(dataDir, app, context) {
// Path names only — skip the per-file count so this stays cheap even on a
// store with millions of parquet files (this is the History API hot path).
const pathInfos = getAvailablePaths(dataDir, app, context, false);
return pathInfos.map(pathInfo => pathInfo.path);
}
/**
* Get available SignalK paths that have data within a specific time range
* This is compliant with SignalK History API specification
* Uses Hive-partitioned directory structure
*/
async function getAvailablePathsForTimeRange(dataDir, context, from, to) {
const hiveBuilder = new hive_path_builder_1.HivePathBuilder();
const fromIso = from.toInstant().toString();
const toIso = to.toInstant().toString();
// Build Hive-style context directory
const sanitizedContext = hiveBuilder.sanitizeContext(context);
const contextDir = path.join(dataDir, 'tier=raw', `context=${sanitizedContext}`);
// Get all path= directories for this context
const allPaths = await scanHivePathDirectories(contextDir, hiveBuilder);
// Then, check each path to see if it has data in the time range
const pathsWithData = [];
await Promise.all(allPaths.map(async (pathStr) => {
const hasData = await checkPathHasDataInRangeHive(dataDir, context, pathStr, fromIso, toIso, hiveBuilder);
if (hasData) {
pathsWithData.push(pathStr);
}
}));
return pathsWithData.sort();
}
/**
* Scan Hive-style path= directories to find all SignalK paths
*/
async function scanHivePathDirectories(contextDir, hiveBuilder) {
const paths = [];
try {
if (!(await fs.pathExists(contextDir))) {
return paths;
}
const entries = await fs.readdir(contextDir, { withFileTypes: true });
for (const entry of entries) {
// Only look at path= directories
if (!entry.isDirectory() || !entry.name.startsWith('path=')) {
continue;
}
const sanitizedPath = entry.name.replace('path=', '');
const unsanitizedPath = hiveBuilder.unsanitizePath(sanitizedPath);
// Check if this path directory has any parquet files (recursively in year/day subdirs)
const pathDir = path.join(contextDir, entry.name);
const hasParquet = await hasParquetFilesRecursive(pathDir);
if (hasParquet) {
paths.push(unsanitizedPath);
}
}
}
catch (error) {
// Directory doesn't exist or not accessible - skip
}
return paths;
}
/**
* Check if a directory (or its subdirectories) contains any parquet files
*/
async function hasParquetFilesRecursive(dir) {
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (entry.isFile() && entry.name.endsWith('.parquet')) {
return true;
}
if (entry.isDirectory()) {
// Skip special directories
if (entry.name === 'processed' ||
entry.name === 'failed' ||
entry.name === 'quarantine' ||
entry.name === 'repaired') {
continue;
}
const hasFiles = await hasParquetFilesRecursive(path.join(dir, entry.name));
if (hasFiles)
return true;
}
}
}
catch (error) {
// Skip on error
}
return false;
}
/**
* Check if a specific path has data within the given time range
* Uses Hive-partitioned directory structure
*/
async function checkPathHasDataInRangeHive(dataDir, context, pathStr, fromIso, toIso, hiveBuilder) {
// Build Hive-style glob pattern for this path
const filePath = hiveBuilder.getGlobPattern(dataDir, 'raw', context, pathStr);
try {
const connection = await duckdb_pool_1.DuckDBPool.getConnection();
try {
// Fast query: just check if ANY row exists in time range
const query = `
SELECT 1 as found
FROM read_parquet('${filePath}', union_by_name=true, filename=true)
WHERE signalk_timestamp >= '${fromIso}'
AND signalk_timestamp < '${toIso}'
AND filename NOT LIKE '%/processed/%'
AND filename NOT LIKE '%/quarantine/%'
AND filename NOT LIKE '%/failed/%'
AND filename NOT LIKE '%/repaired/%'
LIMIT 1
`;
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
return rows.length > 0;
}
finally {
connection.disconnectSync();
}
}
catch (error) {
// If path doesn't exist or has no parquet files, return false
return false;
}
}
//# sourceMappingURL=path-discovery.js.map