signalk-parquet
Version:
SignalK plugin and webapp that archives SK data to Parquet files with a regimen control system, advanced querying, Claude integrated AI analysis, spatial capabilities, and REST API.
199 lines • 7.91 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.getAvailablePaths = getAvailablePaths;
exports.getAvailablePathsArray = getAvailablePathsArray;
exports.getAvailablePathsForTimeRange = getAvailablePathsForTimeRange;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const node_api_1 = require("@duckdb/node-api");
const path_helpers_1 = require("./path-helpers");
/**
* Get available SignalK paths from directory structure
* Scans the parquet data directory and returns paths that contain data files
*/
function getAvailablePaths(dataDir, app) {
const paths = [];
// Clean the self context for filesystem usage (replace dots with slashes, colons with underscores)
const selfContextPath = app.selfContext
.replace(/\./g, '/')
.replace(/:/g, '_');
const vesselsDir = path.join(dataDir, selfContextPath);
if (!fs.existsSync(vesselsDir)) {
return paths;
}
function walkPaths(currentPath, relativePath = '') {
try {
const items = fs.readdirSync(currentPath);
items.forEach((item) => {
const fullPath = path.join(currentPath, item);
const stat = fs.statSync(fullPath);
if (stat.isDirectory() &&
item !== 'processed' &&
item !== 'failed' &&
item !== 'quarantine' &&
item !== 'claude-schemas' &&
item !== 'repaired') {
const newRelativePath = relativePath
? `${relativePath}.${item}`
: item;
// Check if this directory has parquet files
const hasParquetFiles = fs
.readdirSync(fullPath)
.some((file) => file.endsWith('.parquet'));
if (hasParquetFiles) {
const fileCount = fs
.readdirSync(fullPath)
.filter((file) => file.endsWith('.parquet')).length;
paths.push({
path: newRelativePath,
directory: fullPath,
fileCount: fileCount,
});
}
walkPaths(fullPath, newRelativePath);
}
});
}
catch (error) {
// Error reading directory - skip
}
}
if (fs.existsSync(vesselsDir)) {
walkPaths(vesselsDir);
}
return paths;
}
/**
* Get available SignalK paths as simple string array
* Useful for SignalK history API compliance
*/
function getAvailablePathsArray(dataDir, app) {
const pathInfos = getAvailablePaths(dataDir, app);
return pathInfos.map(pathInfo => pathInfo.path);
}
/**
* Get available SignalK paths that have data within a specific time range
* This is compliant with SignalK History API specification
*/
async function getAvailablePathsForTimeRange(dataDir, context, from, to) {
const contextPath = (0, path_helpers_1.toContextFilePath)(context);
const fromIso = from.toInstant().toString();
const toIso = to.toInstant().toString();
// First, get all possible paths by scanning the directory structure
const allPaths = await scanPathDirectories(path.join(dataDir, contextPath));
// Then, check each path to see if it has data in the time range
const pathsWithData = [];
await Promise.all(allPaths.map(async (pathStr) => {
const hasData = await checkPathHasDataInRange(dataDir, contextPath, pathStr, fromIso, toIso);
if (hasData) {
pathsWithData.push(pathStr);
}
}));
return pathsWithData.sort();
}
/**
* Recursively scan directories to find all paths with parquet files
*/
async function scanPathDirectories(contextDir) {
const paths = [];
async function scan(dir, prefix = '') {
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
// Skip special directories
if (entry.name === 'processed' ||
entry.name === 'failed' ||
entry.name === 'quarantine' ||
entry.name === 'claude-schemas' ||
entry.name === 'repaired') {
continue;
}
if (entry.isDirectory()) {
const currentPath = prefix ? `${prefix}.${entry.name}` : entry.name;
const fullPath = path.join(dir, entry.name);
// Check if this directory contains parquet files
const files = await fs.readdir(fullPath);
const hasParquet = files.some(f => f.endsWith('.parquet'));
if (hasParquet) {
paths.push(currentPath);
}
// Recurse into subdirectories
await scan(fullPath, currentPath);
}
}
}
catch (error) {
// Directory doesn't exist or not accessible - skip
}
}
await scan(contextDir);
return paths;
}
/**
* Check if a specific path has data within the given time range
*/
async function checkPathHasDataInRange(dataDir, contextPath, pathStr, fromIso, toIso) {
// Sanitize the path for filesystem use
const sanitizedPath = pathStr
.replace(/[^a-zA-Z0-9._]/g, '')
.replace(/\./g, '/');
const filePath = path.join(dataDir, contextPath, sanitizedPath, '*.parquet');
try {
const duckDB = await node_api_1.DuckDBInstance.create();
const connection = await duckDB.connect();
try {
// Fast query: just check if ANY row exists in time range
const query = `
SELECT COUNT(*) as count
FROM read_parquet('${filePath}', union_by_name=true)
WHERE signalk_timestamp >= '${fromIso}'
AND signalk_timestamp < '${toIso}'
LIMIT 1
`;
const result = await connection.runAndReadAll(query);
const rows = result.getRowObjects();
return rows.length > 0 && rows[0].count > 0;
}
finally {
connection.disconnectSync();
}
}
catch (error) {
// If path doesn't exist or has no parquet files, return false
return false;
}
}
//# sourceMappingURL=path-discovery.js.map