signalk-parquet
Version:
SignalK plugin to save marine data directly to Parquet files with regimen-based control
523 lines • 22.3 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.MigrationService = void 0;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const glob_1 = require("glob");
const events_1 = require("events");
// Try to import ParquetJS, fall back if not available
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let parquet;
try {
parquet = require('@dsnp/parquetjs');
}
catch (error) {
parquet = null;
}
class MigrationService extends events_1.EventEmitter {
constructor(app) {
super();
this.app = app;
}
emitProgress(progress) {
super.emit('progress', progress);
this.app?.debug(`Migration: ${progress.message}`);
}
async scanForProblematicFiles(dataDir) {
this.emitProgress({
type: 'log',
message: `🔍 Starting scan of ${dataDir}...`,
});
try {
// Find all parquet files (excluding processed directories)
const parquetFiles = await (0, glob_1.glob)(`${dataDir}/**/*.parquet`, {
ignore: '**/processed/**',
});
this.emitProgress({
type: 'log',
message: `📁 Found ${parquetFiles.length} parquet files to check`,
});
const problematicFiles = [];
let checkedCount = 0;
// Check each file for schema issues
for (const filePath of parquetFiles) {
try {
const needsMigration = await this.checkFileNeedsMigration(filePath);
if (needsMigration) {
problematicFiles.push(filePath);
}
checkedCount++;
const progress = Math.round((checkedCount / parquetFiles.length) * 100);
if (checkedCount % 50 === 0 || checkedCount === parquetFiles.length) {
this.emitProgress({
type: 'progress',
message: `Checking schemas... (${checkedCount}/${parquetFiles.length})`,
progress,
});
}
}
catch (error) {
this.emitProgress({
type: 'log',
message: `⚠️ Could not check ${filePath}: ${error.message}`,
});
}
}
// Group problematic files by path
const pathGroups = this.groupFilesByPath(problematicFiles);
const result = {
totalFiles: parquetFiles.length,
problematicPaths: pathGroups,
summary: {
pathsNeedingMigration: pathGroups.length,
filesNeedingMigration: problematicFiles.length,
},
};
this.emitProgress({
type: 'complete',
message: `✅ Scan complete: ${result.summary.filesNeedingMigration} files in ${result.summary.pathsNeedingMigration} paths need migration`,
data: result,
});
return result;
}
catch (error) {
this.emitProgress({
type: 'error',
message: `❌ Scan failed: ${error.message}`,
});
throw error;
}
}
async restoreFromBackups(dataDir) {
this.emitProgress({
type: 'log',
message: `🔄 Restoring corrupted files from backups...`,
});
try {
// Find all backup files
const backupFiles = await (0, glob_1.glob)(`${dataDir}/**/*.backup-utf8`, {
ignore: '**/processed/**',
});
this.emitProgress({
type: 'log',
message: `📁 Found ${backupFiles.length} backup files to restore`,
});
let restored = 0;
let errors = 0;
for (const backupPath of backupFiles) {
try {
const originalPath = backupPath.replace('.backup-utf8', '');
// Check if corrupted file exists
if (await fs.pathExists(originalPath)) {
await fs.remove(originalPath);
}
// Restore from backup
await fs.move(backupPath, originalPath);
restored++;
this.emitProgress({
type: 'log',
message: `✅ Restored: ${path.relative(dataDir, originalPath)}`,
});
}
catch (error) {
errors++;
this.emitProgress({
type: 'log',
message: `❌ Error restoring ${backupPath}: ${error.message}`,
});
}
}
this.emitProgress({
type: 'complete',
message: `🎉 Restore complete: ${restored} files restored, ${errors} errors`,
data: { restored, errors },
});
}
catch (error) {
this.emitProgress({
type: 'error',
message: `❌ Restore failed: ${error.message}`,
});
throw error;
}
}
async repairSelectedPaths(dataDir, selectedPaths) {
this.emitProgress({
type: 'log',
message: `🔧 Starting repair of ${selectedPaths.length} selected paths...`,
});
let totalRepaired = 0;
let totalErrors = 0;
for (let i = 0; i < selectedPaths.length; i++) {
const selectedPath = selectedPaths[i];
try {
this.emitProgress({
type: 'progress',
message: `Repairing path: ${selectedPath}`,
progress: Math.round((i / selectedPaths.length) * 100),
});
// Find all problematic files in this path
const pathPattern = path.join(dataDir, selectedPath, '**/*.parquet');
const pathFiles = await (0, glob_1.glob)(pathPattern, {
ignore: '**/processed/**',
});
// Repair each file in this path
for (const filePath of pathFiles) {
const needsMigration = await this.checkFileNeedsMigration(filePath);
if (needsMigration) {
await this.repairSingleFile(filePath);
totalRepaired++;
this.emitProgress({
type: 'log',
message: `✅ Repaired: ${path.relative(dataDir, filePath)}`,
});
}
}
}
catch (error) {
totalErrors++;
this.emitProgress({
type: 'log',
message: `❌ Error repairing ${selectedPath}: ${error.message}`,
});
}
}
this.emitProgress({
type: 'complete',
message: `🎉 Repair complete: ${totalRepaired} files repaired, ${totalErrors} errors`,
data: { repaired: totalRepaired, errors: totalErrors },
});
}
async checkFileNeedsMigration(filePath) {
if (!parquet) {
return false;
}
try {
const reader = await parquet.ParquetReader.openFile(filePath);
const schema = reader.schema;
// Check all value-related columns (value, value_latitude, value_longitude, etc.)
// But exclude metadata fields like value_json which should stay as strings
const schemaFields = schema.schema || {};
const valueFieldNames = Object.keys(schemaFields).filter(name => name === 'value' ||
(name.startsWith('value_') && name !== 'value_json'));
if (valueFieldNames.length === 0) {
await reader.close();
return false;
}
// Check if any value field is UTF8/BYTE_ARRAY (string type)
for (const fieldName of valueFieldNames) {
const field = schemaFields[fieldName];
if (field) {
const needsMigration = field.primitiveType === 'UTF8' ||
field.type === 'UTF8' ||
field.primitiveType === 'BYTE_ARRAY' ||
field.type === 'BYTE_ARRAY' ||
(field.logicalType && field.logicalType.type === 'UTF8') ||
(field.logicalType && field.logicalType.type === 'STRING');
if (needsMigration) {
await reader.close();
return true;
}
}
}
await reader.close();
return false;
}
catch (error) {
return false;
}
}
groupFilesByPath(files) {
const pathMap = new Map();
// Group files by their parent path structure
files.forEach(filePath => {
// Extract the logical path (remove data/ prefix and filename)
const relativePath = path.dirname(filePath);
const parts = relativePath.split(path.sep);
// Find the vessels/* pattern and build logical path
const vesselsIndex = parts.findIndex(p => p === 'vessels');
if (vesselsIndex >= 0) {
const logicalPath = parts.slice(vesselsIndex).join('/');
if (!pathMap.has(logicalPath)) {
pathMap.set(logicalPath, []);
}
pathMap.get(logicalPath).push(filePath);
}
});
// Convert to PathInfo array
return Array.from(pathMap.entries())
.map(([path, files]) => ({
path,
fileCount: files.length,
files,
needsMigration: true,
}))
.sort((a, b) => a.path.localeCompare(b.path));
}
async repairSingleFile(filePath) {
if (!parquet) {
throw new Error('ParquetJS not available');
}
try {
// Read all records from the file
this.emitProgress({
type: 'log',
message: `📖 Reading ${filePath}...`,
});
const records = await this.readParquetFile(filePath);
if (records.length === 0) {
this.emitProgress({
type: 'log',
message: `⏭️ Skipping empty file ${filePath}`,
});
return; // Skip empty files
}
// Create new file with intelligent schema first
const tempPath = `${filePath}.migrating`;
// Remove temp file if it already exists
if (await fs.pathExists(tempPath)) {
this.emitProgress({
type: 'log',
message: `🗑️ Removing existing temp file: ${tempPath}`,
});
await fs.remove(tempPath);
}
this.emitProgress({
type: 'log',
message: `🔄 Creating migrated file with ${records.length} records...`,
});
await this.writeParquetWithIntelligentSchema(tempPath, records);
// Backup original and replace
const backupPath = `${filePath}.backup-utf8`;
// If backup already exists, remove it first (from previous migration attempt)
if (await fs.pathExists(backupPath)) {
this.emitProgress({
type: 'log',
message: `🗑️ Removing existing backup file: ${backupPath}`,
});
await fs.remove(backupPath);
}
this.emitProgress({
type: 'log',
message: `💾 Creating backup: ${filePath} → ${backupPath}`,
});
await fs.move(filePath, backupPath);
this.emitProgress({
type: 'log',
message: `🔄 Replacing original: ${tempPath} → ${filePath}`,
});
await fs.move(tempPath, filePath);
}
catch (error) {
// Clean up temp file if it exists
const tempPath = `${filePath}.migrating`;
if (await fs.pathExists(tempPath)) {
await fs.remove(tempPath);
}
throw error; // Re-throw to be caught by the calling function
}
}
async readParquetFile(filePath) {
const reader = await parquet.ParquetReader.openFile(filePath);
const cursor = reader.getCursor();
const records = [];
let record = null;
while ((record = await cursor.next())) {
records.push(record);
}
await reader.close();
return records;
}
async writeParquetWithIntelligentSchema(filepath, records) {
if (records.length === 0) {
throw new Error('No records to write');
}
// Create intelligent schema (reuse logic from ParquetWriter)
const schema = this.createIntelligentSchema(records);
// Create Parquet writer
const writer = await parquet.ParquetWriter.openFile(schema, filepath);
// Write records with proper type conversion
for (const record of records) {
const preparedRecord = this.prepareRecordForParquet(record, schema);
await writer.appendRow(preparedRecord);
}
// Close the writer
await writer.close();
}
// Replicate the intelligent schema logic from ParquetWriter
// eslint-disable-next-line @typescript-eslint/no-explicit-any
createIntelligentSchema(records) {
if (!parquet || records.length === 0) {
throw new Error('Cannot create Parquet schema');
}
// Get all unique column names from all records
const allColumns = new Set();
records.forEach(record => {
Object.keys(record).forEach(key => allColumns.add(key));
});
const columns = Array.from(allColumns).sort();
const schemaFields = {};
// Analyze each column to determine the best Parquet type
columns.forEach(colName => {
const values = records
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.map(r => r[colName])
.filter(v => v !== null && v !== undefined);
if (values.length === 0) {
// All null values, default to string
schemaFields[colName] = { type: 'UTF8', optional: true };
return;
}
// For migration: attempt to parse string values as numbers
const parsedValues = values.map(v => {
// Skip null, undefined, or empty string values
if (v === null || v === undefined || v === '') {
return v;
}
if (typeof v === 'string') {
const trimmed = v.trim();
if (trimmed === '') {
return null; // Treat whitespace-only as null
}
// Don't parse timestamp strings as numbers - check for ISO date patterns
if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3})?Z?$/.test(trimmed) ||
/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/.test(trimmed)) {
return v; // Keep as string
}
// Try to parse as number
const parsed = parseFloat(trimmed);
return !isNaN(parsed) ? parsed : v;
}
return v;
});
// Filter out null/undefined/empty values for type analysis
const nonNullValues = parsedValues.filter(v => v !== null && v !== undefined && v !== '');
// Use non-null values for type detection to avoid null skewing results
const hasNumbers = nonNullValues.some(v => typeof v === 'number');
const hasStrings = nonNullValues.some(v => typeof v === 'string');
const hasBooleans = nonNullValues.some(v => typeof v === 'boolean');
// If all non-null values are the same type, use that type
if (nonNullValues.length === 0) {
// All values are null - default to UTF8 but this is fine
schemaFields[colName] = { type: 'UTF8', optional: true };
return;
}
if (hasNumbers && !hasStrings && !hasBooleans) {
// All non-null values are numbers - check if integers or floats
const allIntegers = nonNullValues.every(v => typeof v === 'number' && Number.isInteger(v));
const detectedType = allIntegers ? 'INT64' : 'DOUBLE';
schemaFields[colName] = {
type: detectedType,
optional: true,
};
// Debug log for value-related columns (exclude metadata fields)
if (colName === 'value' ||
(colName.startsWith('value_') && colName !== 'value_json')) {
this.emitProgress({
type: 'log',
message: `🔍 ${colName} detected as ${detectedType} (${nonNullValues.length}/${values.length} non-null, sample: ${nonNullValues.slice(0, 3).join(', ')})`,
});
}
}
else if (hasBooleans && !hasNumbers && !hasStrings) {
schemaFields[colName] = { type: 'BOOLEAN', optional: true };
}
else {
// Mixed types or strings - use UTF8
schemaFields[colName] = { type: 'UTF8', optional: true };
// Debug log for value-related columns that stay as string (exclude metadata fields)
if (colName === 'value' ||
(colName.startsWith('value_') && colName !== 'value_json')) {
this.emitProgress({
type: 'log',
message: `⚠️ ${colName} staying as UTF8 - hasNumbers:${hasNumbers}, hasStrings:${hasStrings}, hasBooleans:${hasBooleans} (${nonNullValues.length}/${values.length} non-null, sample: ${nonNullValues.slice(0, 3).join(', ')})`,
});
}
}
});
return new parquet.ParquetSchema(schemaFields);
}
// Replicate the record preparation logic from ParquetWriter
prepareRecordForParquet(record,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
schema
// eslint-disable-next-line @typescript-eslint/no-explicit-any
) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const cleanRecord = {};
const schemaFields = schema.schema;
Object.keys(schemaFields).forEach(fieldName => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const value = record[fieldName];
const fieldType = schemaFields[fieldName].type;
if (value === null || value === undefined) {
cleanRecord[fieldName] = null;
}
else {
switch (fieldType) {
case 'DOUBLE':
case 'FLOAT':
cleanRecord[fieldName] =
typeof value === 'number' ? value : parseFloat(String(value));
break;
case 'INT64':
case 'INT32':
cleanRecord[fieldName] =
typeof value === 'number'
? Math.round(value)
: parseInt(String(value));
break;
case 'BOOLEAN':
cleanRecord[fieldName] =
typeof value === 'boolean' ? value : Boolean(value);
break;
case 'UTF8':
default:
if (typeof value === 'object') {
cleanRecord[fieldName] = JSON.stringify(value);
}
else {
cleanRecord[fieldName] = String(value);
}
break;
}
}
});
return cleanRecord;
}
}
exports.MigrationService = MigrationService;
//# sourceMappingURL=migration-service.js.map