signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
208 lines • 8.37 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.getPathComponentSchema = getPathComponentSchema;
exports.clearSchemaCache = clearSchemaCache;
exports.inferDataTypeCategory = inferDataTypeCategory;
const duckdb_pool_1 = require("./duckdb-pool");
const path = __importStar(require("path"));
const fs = __importStar(require("fs-extra"));
const debug_logger_1 = require("./debug-logger");
const cache_defaults_1 = require("../config/cache-defaults");
const directory_scanner_1 = require("./directory-scanner");
const hive_path_builder_1 = require("./hive-path-builder");
/**
* Cache for path component schemas
* Key: `${context}:${path}`
*/
const schemaCache = new Map();
/**
* Directory scanner for finding parquet files
* Reused across multiple schema discovery operations
*/
const directoryScanner = new directory_scanner_1.DirectoryScanner();
/**
* Hive path builder for constructing Hive-style paths
*/
const hivePathBuilder = new hive_path_builder_1.HivePathBuilder();
/**
* Get the component schema for an object-valued path across all parquet files
* Returns the union of all value_* columns found in any file for this path
* Uses Hive-partitioned directory structure: tier=raw/context=.../path=.../
*/
async function getPathComponentSchema(dataDir, context, pathStr) {
const cacheKey = `${context}:${pathStr}`;
const now = Date.now();
// Check cache first
const cached = schemaCache.get(cacheKey);
if (cached && now - cached.timestamp < cache_defaults_1.CACHE_TTL.SCHEMA) {
return cached;
}
try {
// Build Hive-style path for this context and path
// Default to 'raw' tier for schema discovery
const sanitizedContext = hivePathBuilder.sanitizeContext(context);
const sanitizedPath = hivePathBuilder.sanitizePath(pathStr);
const pathDir = path.join(dataDir, 'tier=raw', `context=${sanitizedContext}`, `path=${sanitizedPath}`);
if (!(await fs.pathExists(pathDir))) {
return null;
}
// Recursively find all .parquet files
const parquetFiles = await findParquetFiles(pathDir);
if (parquetFiles.length === 0) {
return null;
}
// Query schemas from all files to get union of components
const allComponents = new Map();
// Get connection from pool
const connection = await duckdb_pool_1.DuckDBPool.getConnection();
try {
for (const filePath of parquetFiles) {
try {
// First check if this file has a 'value' column
const valueColQuery = `
SELECT name
FROM parquet_schema('${filePath.replace(/'/g, "''")}')
WHERE name = 'value'
`;
const valueColResult = await connection.runAndReadAll(valueColQuery);
const hasValueColumn = valueColResult.getRowObjects().length > 0;
// If 'value' column exists, skip this file - it's a scalar path
if (hasValueColumn) {
continue;
}
// Query the parquet schema for data component columns
// Exclude metadata columns like value_units, value_description, value_json
const schemaQuery = `
SELECT name, type
FROM parquet_schema('${filePath.replace(/'/g, "''")}')
WHERE name LIKE 'value_%'
AND name NOT IN ('value_json', 'value_units', 'value_description', 'value_age')
`;
const result = await connection.runAndReadAll(schemaQuery);
const rows = result.getRowObjects();
rows.forEach(row => {
const columnName = row.name;
const columnType = row.type;
const componentName = columnName.replace(/^value_/, '');
// Skip if we already have this component
if (allComponents.has(componentName)) {
return;
}
// Determine data type category
const dataType = inferDataTypeCategory(columnType);
allComponents.set(componentName, {
name: componentName,
columnName: columnName,
dataType: dataType,
});
});
}
catch (error) {
// Skip files with errors (corrupted, etc.)
debug_logger_1.debugLogger.warn(`[Schema Cache] Error reading schema from ${filePath}:`, error);
}
}
}
finally {
connection.disconnectSync();
}
if (allComponents.size === 0) {
// No value_* columns found - this is a simple scalar path
return null;
}
const schema = {
components: allComponents,
timestamp: now,
};
// Cache it
schemaCache.set(cacheKey, schema);
return schema;
}
catch (error) {
debug_logger_1.debugLogger.error(`[Schema Cache] Error getting schema for ${pathStr}:`, error);
return null;
}
}
/**
* Clear the schema cache (useful for testing or when data structure changes)
*/
function clearSchemaCache() {
schemaCache.clear();
debug_logger_1.debugLogger.log('[Schema Cache] Schema cache cleared');
}
/**
* Infer data type category from DuckDB type string
*/
function inferDataTypeCategory(duckdbType) {
const typeUpper = duckdbType.toUpperCase();
// Numeric types
if (typeUpper.includes('INT') ||
typeUpper.includes('DOUBLE') ||
typeUpper.includes('FLOAT') ||
typeUpper.includes('DECIMAL') ||
typeUpper.includes('NUMERIC') ||
typeUpper.includes('REAL') ||
typeUpper.includes('BIGINT') ||
typeUpper.includes('SMALLINT') ||
typeUpper.includes('TINYINT')) {
return 'numeric';
}
// String types
if (typeUpper.includes('VARCHAR') ||
typeUpper.includes('CHAR') ||
typeUpper.includes('TEXT') ||
typeUpper.includes('STRING') ||
typeUpper.includes('UTF8') ||
typeUpper.includes('BYTE_ARRAY')) {
return 'string';
}
// Boolean
if (typeUpper.includes('BOOL')) {
return 'boolean';
}
return 'unknown';
}
/**
* Recursively find all .parquet files in a directory
* Uses DirectoryScanner for cached, efficient file discovery
*/
async function findParquetFiles(dir) {
// Use DirectoryScanner with pattern matching for .parquet files
const fileInfos = await directoryScanner.scanDirectory(dir, /\.parquet$/);
// Convert FileInfo[] to string[] for compatibility
return fileInfos.map(f => f.path);
}
//# sourceMappingURL=schema-cache.js.map