signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
781 lines • 37.4 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.ParquetWriter = void 0;
exports.quarantineEmptyParquetFiles = quarantineEmptyParquetFiles;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const schema_service_1 = require("./schema-service");
const directory_scanner_1 = require("./utils/directory-scanner");
// Try to import ParquetJS, fall back if not available
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let parquet;
try {
// eslint-disable-next-line @typescript-eslint/no-require-imports
parquet = require('@dsnp/parquetjs');
}
catch (error) {
parquet = null;
}
class ParquetWriter {
constructor(options = { format: 'json' }) {
this.format = options.format || 'json';
this.app = options.app;
// Initialize schema service if app is available
if (this.app) {
this.schemaService = new schema_service_1.SchemaService(this.app);
}
// Initialize directory scanner with 5-minute cache
this.directoryScanner = new directory_scanner_1.DirectoryScanner(5 * 60 * 1000);
}
getSchemaService() {
return this.schemaService;
}
async writeRecords(filepath, records) {
try {
const directory = path.dirname(filepath);
await fs.ensureDir(directory);
let result;
switch (this.format) {
case 'json':
result = await this.writeJSON(filepath, records);
break;
case 'csv':
result = await this.writeCSV(filepath, records);
break;
case 'parquet':
result = await this.writeParquet(filepath, records);
break;
default:
throw new Error(`Unsupported format: ${this.format}`);
}
// Invalidate directory cache since we just wrote a file
// Get the base data directory (go up until we find the root data dir)
const baseDir = this.findBaseDataDir(directory);
this.directoryScanner.invalidateCache(baseDir);
return result;
}
catch (error) {
throw new Error(`Failed to write records: ${error.message}`);
}
}
/**
* Find the base data directory from a nested path
* This helps invalidate the right cache entry
*/
findBaseDataDir(filePath) {
// Walk up the directory tree to find a reasonable cache boundary
// Typically 2-3 levels up from the leaf file
const current = path.dirname(filePath);
const parts = current.split(path.sep);
// Go up to the path level (usually 2-3 directories up)
// This provides a good balance between cache granularity and performance
if (parts.length > 3) {
return parts.slice(0, -2).join(path.sep);
}
return current;
}
async writeJSON(filepath, records) {
const jsonPath = filepath.replace(/\.(parquet|csv)$/, '.json');
await fs.writeJson(jsonPath, records, { spaces: 2 });
return jsonPath;
}
async writeCSV(filepath, records) {
if (records.length === 0)
return filepath;
const csvPath = filepath.replace(/\.(parquet|json)$/, '.csv');
// Get all unique keys from all records
const allKeys = new Set();
records.forEach(record => {
Object.keys(record).forEach(key => allKeys.add(key));
});
const headers = Array.from(allKeys).sort();
const csvRows = [headers.join(',')];
records.forEach(record => {
const row = headers.map(header => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const value = record[header];
if (value === null || value === undefined)
return '';
if (typeof value === 'string' &&
(value.includes(',') || value.includes('"'))) {
return `"${value.replace(/"/g, '""')}"`;
}
return String(value);
});
csvRows.push(row.join(','));
});
await fs.writeFile(csvPath, csvRows.join('\n'));
return csvPath;
}
async writeParquet(filepath, records) {
try {
if (records.length === 0) {
this.app?.debug('No records to write to Parquet file');
return filepath;
}
// Check if ParquetJS is available
if (!parquet) {
this.app?.debug('ParquetJS not available, falling back to JSON');
return await this.writeJSON(filepath, records);
}
// Extract path from records for intelligent schema detection
const currentPath = records.length > 0 ? records[0].path : undefined;
// Extract output directory from filepath (go up to find the base data directory)
// const outputDirectory = this.extractOutputDirectory(filepath);
// Extract filename prefix from filepath (everything before the date part)
// const filename = path.basename(filepath, '.parquet');
// const match = filename.match(/^(.+)_\d{4}-\d{2}-\d{2}/);
// const filenamePrefix = match ? match[1] : 'signalk_data';
// Check if parquet library is available
if (!parquet) {
throw new Error('ParquetJS not available');
}
// Use intelligent schema detection for optimal data types
const schema = await this.createParquetSchema(records, currentPath);
// Create Parquet writer
const writer = await parquet.ParquetWriter.openFile(schema, filepath);
// Write records to Parquet file
for (let i = 0; i < records.length; i++) {
const record = records[i];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const cleanRecord = {};
// Prepare record for typed Parquet schema
const preparedRecord = this.prepareRecordForParquet(record, schema);
Object.assign(cleanRecord, preparedRecord);
await writer.appendRow(cleanRecord);
}
// Close the writer
await writer.close();
// Validate the written file
const isValid = await this.validateParquetFile(filepath);
if (!isValid) {
// Move invalid file to quarantine and log
const quarantineDir = path.join(path.dirname(filepath), 'quarantine');
await fs.ensureDir(quarantineDir);
const quarantineFile = path.join(quarantineDir, path.basename(filepath));
await fs.move(filepath, quarantineFile, { overwrite: true });
await this.logQuarantine(quarantineFile, 'write', 'File failed validation after write');
throw new Error(`Parquet file failed validation after write, moved to quarantine: ${quarantineFile}`);
}
return filepath;
}
catch (error) {
this.app?.debug(`❌ Parquet writing failed: ${error.message}`);
this.app?.debug(`Error stack: ${error.stack}`);
// Save to failed directory to maintain schema consistency
const failedDir = path.join(path.dirname(filepath), 'failed');
await fs.ensureDir(failedDir);
const failedPath = path.join(failedDir, path.basename(filepath).replace('.parquet', '_FAILED.json'));
this.app?.debug(`💾 Saving failed Parquet data as JSON to: ${failedPath}`);
this.app?.debug('⚠️ This data will need manual conversion to maintain DuckDB schema consistency');
await this.writeJSON(failedPath, records);
// Throw error to alert system that Parquet writing is broken
throw new Error(`Parquet writing failed for ${filepath}. Data saved to ${failedPath} for recovery.`);
}
}
/**
* Write Parquet file in batches to avoid loading all records into memory.
* Uses firstBatch for schema detection, then pulls subsequent batches via callback.
*/
async writeParquetBatched(filepath, firstBatch, nextBatch, currentPath) {
try {
if (firstBatch.length === 0) {
this.app?.debug('No records to write to Parquet file');
return filepath;
}
if (!parquet) {
throw new Error('ParquetJS not available');
}
const schema = await this.createParquetSchema(firstBatch, currentPath || firstBatch[0].path);
const writer = await parquet.ParquetWriter.openFile(schema, filepath);
// Write first batch
for (const record of firstBatch) {
const preparedRecord = this.prepareRecordForParquet(record, schema);
await writer.appendRow({ ...preparedRecord });
}
// Pull and write subsequent batches
let batch = nextBatch();
while (batch.length > 0) {
for (const record of batch) {
const preparedRecord = this.prepareRecordForParquet(record, schema);
await writer.appendRow({ ...preparedRecord });
}
batch = nextBatch();
}
await writer.close();
const isValid = await this.validateParquetFile(filepath);
if (!isValid) {
const quarantineDir = path.join(path.dirname(filepath), 'quarantine');
await fs.ensureDir(quarantineDir);
const quarantineFile = path.join(quarantineDir, path.basename(filepath));
await fs.move(filepath, quarantineFile, { overwrite: true });
await this.logQuarantine(quarantineFile, 'write', 'File failed validation after write');
throw new Error(`Parquet file failed validation after write, moved to quarantine: ${quarantineFile}`);
}
return filepath;
}
catch (error) {
this.app?.debug(`Parquet batched writing failed: ${error.message}`);
throw error;
}
}
// Create Parquet schema based on sample records
// Now uses consolidated SchemaService
// eslint-disable-next-line @typescript-eslint/no-explicit-any
async createParquetSchema(records, currentPath) {
if (!this.schemaService) {
throw new Error('SchemaService not available');
}
const result = await this.schemaService.detectOptimalSchema(records, currentPath);
return result.schema;
}
// Guideline 3: Get type for empty columns using SignalK metadata and other files
async getTypeForEmptyColumn(colName, currentPath, outputDirectory, metadataCache, _filenamePrefix) {
this.app?.debug(` 🔍 Empty column fallback for: ${colName} (path: ${currentPath || 'unknown'})`);
// For non-value columns, default to UTF8
if (colName !== 'value') {
this.app?.debug(` ↪️ Non-value column, defaulting to UTF8`);
return 'UTF8';
}
// Try SignalK metadata first
if (currentPath && this.app && metadataCache) {
this.app?.debug(` 🔎 Checking SignalK metadata for path: ${currentPath}`);
if (!metadataCache.has(currentPath)) {
try {
this.app?.debug(` 🔍 Trying metadata lookup for path: "${currentPath}"`);
// Use app's getMetadata method
let metadata = null;
try {
metadata = this.app.getMetadata(currentPath);
this.app?.debug(` 📡 Metadata result: ${metadata ? JSON.stringify(metadata) : 'null'}`);
}
catch (error) {
this.app?.debug(` ❌ Metadata lookup error: ${error.message}`);
}
metadataCache.set(currentPath, metadata);
this.app?.debug(` 📡 Retrieved metadata: ${metadata ? JSON.stringify(metadata) : 'null'}`);
}
catch (error) {
this.app?.debug(` ❌ Metadata API call failed: ${error.message}`);
metadataCache.set(currentPath, null);
}
}
else {
this.app?.debug(` 💾 Using cached metadata for ${currentPath}`);
}
const metadata = metadataCache.get(currentPath);
if (metadata && metadata.units) {
// If metadata suggests numeric units (m/s, degrees, etc.), assume numeric
const numericUnits = [
'm/s',
'm',
'deg',
'rad',
'Pa',
'K',
'Hz',
'V',
'A',
'W',
];
const matchedUnit = numericUnits.find(unit => metadata.units.includes(unit));
if (matchedUnit) {
this.app?.debug(` ✅ Metadata indicates numeric unit '${matchedUnit}', using DOUBLE`);
return 'DOUBLE';
}
else {
this.app?.debug(` ↪️ Metadata has units '${metadata.units}' but not recognized as numeric`);
}
}
else {
this.app?.debug(` ↪️ No useful metadata found (metadata: ${!!metadata}, units: ${metadata?.units})`);
}
}
// Fallback to other consolidated files for the same path
// Disabled to prevent errors from corrupted parquet files
// if (currentPath && outputDirectory) {
// this.app?.debug(` 🔎 Searching other files for path: ${currentPath}`);
// const typeFromOtherFiles = this.getTypeFromOtherFiles(currentPath, outputDirectory, undefined, filenamePrefix);
// if (typeFromOtherFiles) {
// this.app?.debug(` ✅ Found type ${typeFromOtherFiles} from other files`);
// return typeFromOtherFiles;
// } else {
// this.app?.debug(` ↪️ No type information found in other files`);
// }
// }
// Final fallback to UTF8
this.app?.debug(` ✅ Final fallback to UTF8`);
return 'UTF8';
}
// Guideline 4: Get type for exploded value_ fields by parsing actual values
async getTypeForExplodedField(colName, currentPath, outputDirectory, values, metadataCache, filenamePrefix) {
this.app?.debug(` 🧩 Exploded field analysis for: ${colName} (path: ${currentPath || 'unknown'})`);
// Always keep value_json as VARCHAR
if (colName === 'value_json') {
this.app?.debug(` ✅ ${colName}: UTF8 (JSON field always string)`);
return 'UTF8';
}
// If we have values, parse them to detect actual data types
if (values && values.length > 0) {
this.app?.debug(` 🧮 Parsing ${values.length} values for ${colName}`);
let parsedNumbers = 0;
let parsedBooleans = 0;
let actualStrings = 0;
let unparseable = 0;
const stringValues = values.filter(v => typeof v === 'string');
for (const str of stringValues) {
const trimmed = str.trim();
if (trimmed === 'true' || trimmed === 'false') {
parsedBooleans++;
}
else if (!isNaN(Number(trimmed)) && trimmed !== '') {
parsedNumbers++;
}
else if (trimmed === '') {
unparseable++;
}
else {
actualStrings++;
}
}
const hasNumbers = values.some(v => typeof v === 'number') || parsedNumbers > 0;
const hasStrings = values.some(v => typeof v === 'string' &&
v.trim() !== '' &&
isNaN(Number(v.trim())) &&
v.trim() !== 'true' &&
v.trim() !== 'false') || actualStrings > 0;
const hasBooleans = values.some(v => typeof v === 'boolean') || parsedBooleans > 0;
this.app?.debug(` 🧮 ${colName}: Parsed - numbers:${parsedNumbers}, booleans:${parsedBooleans}, strings:${actualStrings}, unparseable:${unparseable}`);
this.app?.debug(` 🧮 ${colName}: Final - hasNumbers:${hasNumbers}, hasStrings:${hasStrings}, hasBooleans:${hasBooleans}`);
if (hasNumbers && !hasStrings && !hasBooleans) {
// All numbers - check if integers or floats
// Always use DOUBLE for numeric maritime data (never INT64/BIGINT)
const finalType = 'DOUBLE';
this.app?.debug(` ✅ ${colName}: ${finalType} (parsed numbers, always DOUBLE for maritime data)`);
return finalType;
}
else if (hasBooleans && !hasNumbers && !hasStrings) {
this.app?.debug(` ✅ ${colName}: BOOLEAN (parsed booleans)`);
return 'BOOLEAN';
}
else if (unparseable > 0 &&
!hasNumbers &&
!hasStrings &&
!hasBooleans) {
// Only unparseable (empty) values - use HTTP metadata
if (currentPath && metadataCache) {
this.app?.debug(` 🔍 ${colName}: Only empty values, using HTTP metadata fallback`);
const fallbackType = await this.getTypeForEmptyColumn(colName, currentPath, outputDirectory, metadataCache, filenamePrefix);
this.app?.debug(` ✅ ${colName}: ${fallbackType} (from HTTP metadata for empty values)`);
return fallbackType;
}
}
else if (hasStrings || actualStrings > 0) {
this.app?.debug(` ✅ ${colName}: UTF8 (parsed strings)`);
return 'UTF8';
}
}
// Fallback to field name inference if no values or unclear parsing
this.app?.debug(` ↪️ No clear type from value parsing, using field name inference`);
return this.inferTypeFromFieldName(colName);
}
// Helper: Search other consolidated files for type information
getTypeFromOtherFiles(currentPath, outputDirectory, specificColumn, filenamePrefix) {
const targetColumn = specificColumn || 'value';
this.app?.debug(` 🔍 Searching files for column '${targetColumn}' in path '${currentPath}'`);
try {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const glob = require('glob');
const prefix = filenamePrefix || 'signalk_data';
const pathPattern = path.join(outputDirectory, 'vessels', '*', currentPath.replace(/\./g, '/'), `${prefix}_*.parquet`);
this.app?.debug(` 📁 Search pattern: ${pathPattern}`);
const allFiles = glob.sync(pathPattern);
// Filter out consolidated files
const files = allFiles.filter((file) => !file.includes('_consolidated.parquet'));
this.app?.debug(` 📄 Found ${files.length} regular files to check (excluding consolidated)`);
for (const filePath of files) {
try {
this.app?.debug(` 🔎 Checking file: ${path.basename(filePath)}`);
if (!parquet) {
this.app?.debug(` ❌ Parquet library not available`);
continue;
}
// Skip corrupted parquet files to prevent crashes
if (path.basename(filePath).includes('corrupted') ||
path.basename(filePath).includes('quarantine')) {
this.app?.debug(` ⚠️ Skipping quarantined file: ${path.basename(filePath)}`);
continue;
}
try {
const reader = parquet.ParquetReader.openFile(filePath);
const schema = reader.schema;
if (schema && schema.schema && schema.schema[targetColumn]) {
const columnType = schema.schema[targetColumn].type;
this.app?.debug(` ✅ Found type ${columnType} for column '${targetColumn}' in ${path.basename(filePath)}`);
if (typeof reader.close === 'function')
reader.close();
return columnType;
}
else {
this.app?.debug(` ↪️ Column '${targetColumn}' not found in ${path.basename(filePath)}`);
}
if (typeof reader.close === 'function')
reader.close();
}
catch (fileError) {
this.app?.debug(` ⚠️ Corrupted file, skipping: ${path.basename(filePath)} - ${fileError.message}`);
continue;
}
}
catch (error) {
this.app?.debug(` ❌ Error reading file ${path.basename(filePath)}: ${error.message}`);
continue;
}
}
}
catch (error) {
this.app?.debug(` ❌ File search error: ${error.message}`);
}
this.app?.debug(` ❌ No type information found in any files`);
return null;
}
// Helper: Infer type from field name patterns
inferTypeFromFieldName(fieldName) {
this.app?.debug(` 🏷️ Inferring type from field name: ${fieldName}`);
const field = fieldName.toLowerCase();
// Coordinate fields
if (field.includes('latitude') ||
field.includes('longitude') ||
field.includes('lat') ||
field.includes('lon')) {
this.app?.debug(` ✅ Coordinate field detected, using DOUBLE`);
return 'DOUBLE';
}
// Numeric measurements
if (field.includes('speed') ||
field.includes('distance') ||
field.includes('depth') ||
field.includes('temperature') ||
field.includes('pressure') ||
field.includes('angle') ||
field.includes('bearing') ||
field.includes('course') ||
field.includes('heading')) {
this.app?.debug(` ✅ Numeric measurement field detected, using DOUBLE`);
return 'DOUBLE';
}
// Time/duration fields
if (field.includes('time') ||
field.includes('duration') ||
field.includes('age')) {
const isTimestamp = field.includes('timestamp');
const resultType = isTimestamp ? 'UTF8' : 'DOUBLE';
this.app?.debug(` ✅ Time field detected, using ${resultType} (timestamp: ${isTimestamp})`);
return resultType;
}
// Default to UTF8 for unknown patterns
this.app?.debug(` ✅ Unknown pattern, defaulting to UTF8`);
return 'UTF8';
}
// Helper: Extract output directory from filepath
extractOutputDirectory(filepath) {
// filepath format: /path/to/outputDir/vessels/context/path/filename.parquet
// We want to extract up to the outputDir part
const parts = filepath.split(path.sep);
const vesselIndex = parts.findIndex(part => part === 'vessels');
if (vesselIndex > 0) {
// Return everything up to but not including 'vessels'
return parts.slice(0, vesselIndex).join(path.sep);
}
// Fallback: assume current directory structure
return path.dirname(path.dirname(path.dirname(filepath)));
}
// Prepare a record for typed Parquet writing
prepareRecordForParquet(record,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
schema
// eslint-disable-next-line @typescript-eslint/no-explicit-any
) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const cleanRecord = {};
// Serialize object fields to JSON strings (deferred from delta processing)
// This improves performance by avoiding JSON.stringify() on every delta message
const recordWithSerializedFields = { ...record };
if (recordWithSerializedFields.source &&
typeof recordWithSerializedFields.source === 'object') {
recordWithSerializedFields.source = JSON.stringify(recordWithSerializedFields.source);
}
if (recordWithSerializedFields.value_json &&
typeof recordWithSerializedFields.value_json === 'object') {
recordWithSerializedFields.value_json = JSON.stringify(recordWithSerializedFields.value_json);
}
if (recordWithSerializedFields.meta &&
typeof recordWithSerializedFields.meta === 'object') {
recordWithSerializedFields.meta = JSON.stringify(recordWithSerializedFields.meta);
}
const schemaFields = schema.schema;
Object.keys(schemaFields).forEach(fieldName => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const value = recordWithSerializedFields[fieldName];
const fieldType = schemaFields[fieldName].type;
if (value === null || value === undefined) {
// Use undefined instead of null - parquet handles undefined (omit field) better than null
cleanRecord[fieldName] = undefined;
}
else if (typeof value === 'bigint') {
// Handle BigInt values by converting to appropriate type
switch (fieldType) {
case 'DOUBLE':
case 'FLOAT':
cleanRecord[fieldName] = Number(value);
break;
case 'INT64':
case 'INT32':
// Convert BigInt to number if it fits in safe integer range
if (value <= Number.MAX_SAFE_INTEGER &&
value >= Number.MIN_SAFE_INTEGER) {
cleanRecord[fieldName] = Number(value);
}
else {
cleanRecord[fieldName] = value.toString();
}
break;
case 'UTF8':
default:
cleanRecord[fieldName] = value.toString();
break;
}
}
else {
switch (fieldType) {
case 'DOUBLE':
case 'FLOAT':
cleanRecord[fieldName] =
typeof value === 'number' ? value : parseFloat(String(value));
break;
case 'INT64':
case 'INT32':
cleanRecord[fieldName] =
typeof value === 'number'
? Math.round(value)
: parseInt(String(value));
break;
case 'BOOLEAN':
cleanRecord[fieldName] =
typeof value === 'boolean' ? value : Boolean(value);
break;
case 'UTF8':
default:
if (typeof value === 'object') {
cleanRecord[fieldName] = JSON.stringify(value);
}
else {
cleanRecord[fieldName] = String(value);
}
break;
}
}
});
return cleanRecord;
}
// Validate parquet file for corruption
async validateParquetFile(filepath) {
try {
if (!parquet || !(await fs.pathExists(filepath))) {
return false;
}
// Check file size (must be > 100 bytes as per existing logic)
const stats = await fs.stat(filepath);
const fileSize = stats.size;
if (fileSize < 100) {
this.app?.debug(`❌ Parquet file too small: ${filepath} (${fileSize} bytes)`);
return false;
}
// Try to open and read the parquet file
try {
const reader = await parquet.ParquetReader.openFile(filepath);
const cursor = reader.getCursor();
// Try to read first record to verify file structure
const firstRecord = await cursor.next();
await reader.close();
// Log file size for debugging (matches your stat command format)
this.app?.debug(`✅ Valid parquet file: ${fileSize.toString().padStart(12, ' ')} ${filepath}`);
return firstRecord !== null;
}
catch (readError) {
this.app?.debug(`❌ Parquet file read failed: ${filepath} - ${readError.message}`);
return false;
}
}
catch (error) {
this.app?.debug(`❌ Parquet validation error: ${filepath} - ${error.message}`);
return false;
}
}
// Log quarantined files
async logQuarantine(filepath, operation, reason) {
try {
const stats = await fs.stat(filepath);
const logEntry = {
timestamp: new Date().toISOString(),
filepath,
fileSize: stats.size,
operation,
reason,
formattedSize: `${stats.size.toString().padStart(12, ' ')} ${filepath}`,
};
const quarantineDir = path.dirname(filepath);
const logFile = path.join(quarantineDir, 'quarantine.log');
// Append to log file
const logLine = `${logEntry.timestamp} | ${logEntry.operation} | ${logEntry.fileSize} bytes | ${logEntry.reason} | ${filepath}\n`;
await fs.appendFile(logFile, logLine);
this.app?.debug(`📋 Quarantine logged: ${logEntry.formattedSize}`);
}
catch (error) {
this.app?.debug(`Failed to log quarantine entry: ${error.message}`);
}
}
}
exports.ParquetWriter = ParquetWriter;
/**
* Move undersized (`< 100B`) parquet files left behind by a crash between
* ParquetWriter.openFile() — which creates a 0-byte stub on disk — and the
* first appendRow()/close() that would populate it. The per-write catch
* block can't run if the process is killed in that window, so we sweep
* on plugin start. Files are moved to a sibling `quarantine/` dir, matching
* the in-flight quarantine layout; the History API already excludes those
* paths from queries.
*
* The sweep is incremental: a stub is a newly *created* file, which bumps its
* parent directory's mtime, so directories unchanged since the last sweep are
* skipped. The last-sweep time is persisted in `<baseDirectory>/.last-empty-
* sweep`; the first run (no watermark) scans everything once to catch any
* pre-existing stub, then later runs cost O(directories changed since the
* previous start) and still catch a stub of any age after long downtime.
*/
async function quarantineEmptyParquetFiles(app, baseDirectory) {
if (!(await fs.pathExists(baseDirectory))) {
return { quarantined: 0, failed: 0 };
}
// Persisted last-sweep timestamp. Missing/unparseable (first run after this
// change) leaves the watermark at 0, so everything is scanned once to catch
// pre-existing stubs; subsequent runs only inspect directories changed since.
const stateFile = path.join(baseDirectory, '.last-empty-sweep');
let watermark = 0;
try {
const parsed = Number((await fs.readFile(stateFile, 'utf8')).trim());
if (Number.isFinite(parsed))
watermark = parsed;
}
catch {
// No prior sweep recorded — proceed with watermark = 0 (full scan once).
}
const sweepStart = Date.now();
const ignoredDir = (name) => name === 'quarantine' ||
name === 'failed' ||
name === 'repaired' ||
name.startsWith('.compaction-trash-');
let quarantined = 0;
let failed = 0;
const walk = async (dir) => {
let entries;
try {
entries = await fs.readdir(dir, { withFileTypes: true });
}
catch {
return; // unreadable directory — skip
}
// A new stub bumps the mtime of the directory it was created in, so a
// directory unchanged since the last sweep has no new files to check.
// Subdirectories are always descended: an ancestor's mtime does NOT change
// when a file is added deep in the tree.
let inspectFiles = true;
if (watermark > 0) {
try {
inspectFiles = (await fs.stat(dir)).mtimeMs > watermark;
}
catch {
inspectFiles = true;
}
}
for (const entry of entries) {
if (entry.isDirectory()) {
if (!ignoredDir(entry.name)) {
await walk(path.join(dir, entry.name));
}
continue;
}
if (!inspectFiles || !entry.name.endsWith('.parquet'))
continue;
const file = path.join(dir, entry.name);
try {
const stats = await fs.stat(file);
if (stats.size >= 100)
continue;
const quarantineDir = path.join(dir, 'quarantine');
await fs.ensureDir(quarantineDir);
const quarantinePath = path.join(quarantineDir, entry.name);
await fs.move(file, quarantinePath, { overwrite: true });
const logLine = `${new Date().toISOString()} | startup-sweep | ${stats.size} bytes | Undersized parquet (likely crash between openFile and close) | ${quarantinePath}\n`;
await fs.appendFile(path.join(quarantineDir, 'quarantine.log'), logLine);
quarantined++;
app.debug(`Quarantined undersized parquet (${stats.size}B): ${file}`);
}
catch (err) {
failed++;
app.error(`Failed to quarantine undersized parquet ${file}: ${err.message}`);
}
}
};
await walk(baseDirectory);
// Record the next run's cutoff: the sweep's start (not end, so files written
// while it ran are re-checked), minus a slack margin so a coarse-resolution
// filesystem (1s mtime) or a same-second write can't round a just-created
// directory below the watermark and skip it. The overlap only re-inspects
// directories touched in the ~2s around this sweep.
const WATERMARK_SLACK_MS = 2000;
try {
await fs.writeFile(stateFile, String(sweepStart - WATERMARK_SLACK_MS), 'utf8');
}
catch (err) {
app.error(`Failed to persist empty-sweep watermark ${stateFile}: ${err.message}`);
}
if (quarantined > 0) {
app.debug(`Parquet startup sweep: quarantined ${quarantined} undersized file(s)`);
}
return { quarantined, failed };
}
//# sourceMappingURL=parquet-writer.js.map