signalk-parquet
Version:
SignalK plugin to save marine data directly to Parquet files with regimen-based control
288 lines ā¢ 11.7 kB
JavaScript
;
/**
* Schema Migration Script
*
* Migrates old parquet files with UTF8-only schemas to new intelligent schemas.
* Backs up originals and recreates files with proper data types.
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.migrateSchemas = migrateSchemas;
const fs = __importStar(require("fs-extra"));
const glob_1 = require("glob");
// Try to import ParquetJS, fall back if not available
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let parquet;
try {
parquet = require('@dsnp/parquetjs');
}
catch (error) {
console.error('ParquetJS not available:', error);
process.exit(1);
}
async function migrateSchemas(dataDir = 'data') {
console.log(`š Scanning for parquet files in ${dataDir}...`);
// Find all parquet files (excluding processed directories)
const parquetFiles = await (0, glob_1.glob)(`${dataDir}/**/*.parquet`, {
ignore: '**/processed/**',
});
console.log(`š Found ${parquetFiles.length} parquet files`);
const stats = {
migrated: 0,
skipped: 0,
errors: 0,
};
for (const filePath of parquetFiles) {
try {
console.log(`\nš Checking: ${filePath}`);
// Check if file needs migration (has UTF8 value column)
const needsMigration = await checkNeedsMigration(filePath);
if (!needsMigration) {
console.log(`ā
Already has proper schema, skipping`);
stats.skipped++;
continue;
}
console.log(`š Migrating schema...`);
// Read all records from the file
const records = await readParquetFile(filePath);
if (records.length === 0) {
console.log(`ā ļø Empty file, skipping`);
stats.skipped++;
continue;
}
// Create new file with intelligent schema first
const tempPath = `${filePath}.migrated`;
await writeParquetWithIntelligentSchema(tempPath, records);
// Backup original and replace
const backupPath = `${filePath}.backup-utf8`;
// If backup already exists, remove it first (from previous migration attempt)
if (await fs.pathExists(backupPath)) {
await fs.remove(backupPath);
}
await fs.move(filePath, backupPath);
await fs.move(tempPath, filePath);
console.log(`š¾ Original backed up to: ${backupPath}`);
console.log(`ā
Migrated: ${records.length} records`);
stats.migrated++;
}
catch (error) {
console.error(`ā Error migrating ${filePath}:`, error.message);
stats.errors++;
}
}
console.log(`\nš Migration Summary:`);
console.log(` ā
Migrated: ${stats.migrated} files`);
console.log(` āļø Skipped: ${stats.skipped} files`);
console.log(` ā Errors: ${stats.errors} files`);
return stats;
}
async function checkNeedsMigration(filePath) {
try {
const reader = await parquet.ParquetReader.openFile(filePath);
const schema = reader.schema;
// Check all value-related columns (value, value_latitude, value_longitude, etc.)
// But exclude metadata fields like value_json which should stay as strings
const schemaFields = schema.schema || {};
const valueFieldNames = Object.keys(schemaFields).filter(name => name === 'value' || (name.startsWith('value_') && name !== 'value_json'));
if (valueFieldNames.length === 0) {
await reader.close();
return false;
}
// Check if any value field is UTF8/BYTE_ARRAY (string type)
for (const fieldName of valueFieldNames) {
const field = schemaFields[fieldName];
if (field) {
const needsMigration = field.primitiveType === 'UTF8' ||
field.type === 'UTF8' ||
field.primitiveType === 'BYTE_ARRAY' ||
field.type === 'BYTE_ARRAY' ||
(field.logicalType && field.logicalType.type === 'UTF8') ||
(field.logicalType && field.logicalType.type === 'STRING');
if (needsMigration) {
await reader.close();
return true;
}
}
}
await reader.close();
return false;
}
catch (error) {
console.warn(`Warning: Could not read schema from ${filePath}`);
return false;
}
}
async function readParquetFile(filePath) {
const reader = await parquet.ParquetReader.openFile(filePath);
const cursor = reader.getCursor();
const records = [];
let record = null;
while ((record = await cursor.next())) {
records.push(record);
}
await reader.close();
return records;
}
// Replicate the intelligent schema logic from ParquetWriter
function createIntelligentSchema(records) {
if (!parquet || records.length === 0) {
throw new Error('Cannot create Parquet schema');
}
// Get all unique column names from all records
const allColumns = new Set();
records.forEach(record => {
Object.keys(record).forEach(key => allColumns.add(key));
});
const columns = Array.from(allColumns).sort();
console.log(`š Schema columns: ${columns.join(', ')}`);
const schemaFields = {};
// Analyze each column to determine the best Parquet type
columns.forEach(colName => {
const values = records
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.map(r => r[colName])
.filter(v => v !== null && v !== undefined);
if (values.length === 0) {
// All null values, default to string
schemaFields[colName] = { type: 'UTF8', optional: true };
return;
}
const hasNumbers = values.some(v => typeof v === 'number');
const hasStrings = values.some(v => typeof v === 'string');
const hasBooleans = values.some(v => typeof v === 'boolean');
// Only log details for the value column that we care about
if (colName === 'value') {
console.log(`šÆ Value column analysis - numbers: ${hasNumbers}, strings: ${hasStrings}, booleans: ${hasBooleans}`);
console.log(`š Value samples: ${JSON.stringify(values.slice(0, 3))}`);
}
if (hasNumbers && !hasStrings && !hasBooleans) {
// All numbers - check if integers or floats
const allIntegers = values.every(v => Number.isInteger(v));
schemaFields[colName] = {
type: allIntegers ? 'INT64' : 'DOUBLE',
optional: true,
};
if (colName === 'value') {
console.log(`š¢ Value column -> ${allIntegers ? 'INT64' : 'DOUBLE'}`);
}
}
else if (hasBooleans && !hasNumbers && !hasStrings) {
schemaFields[colName] = { type: 'BOOLEAN', optional: true };
if (colName === 'value') {
console.log(`ā
Value column -> BOOLEAN`);
}
}
else {
// Mixed types or strings - use UTF8
schemaFields[colName] = { type: 'UTF8', optional: true };
if (colName === 'value') {
console.log(`š Value column -> UTF8 (mixed/strings)`);
}
}
});
return new parquet.ParquetSchema(schemaFields);
}
// Replicate the record preparation logic from ParquetWriter
function prepareRecordForParquet(record,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
schema
// eslint-disable-next-line @typescript-eslint/no-explicit-any
) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const cleanRecord = {};
const schemaFields = schema.schema;
Object.keys(schemaFields).forEach(fieldName => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const value = record[fieldName];
const fieldType = schemaFields[fieldName].type;
if (value === null || value === undefined) {
cleanRecord[fieldName] = null;
}
else {
switch (fieldType) {
case 'DOUBLE':
case 'FLOAT':
cleanRecord[fieldName] =
typeof value === 'number' ? value : parseFloat(String(value));
break;
case 'INT64':
case 'INT32':
cleanRecord[fieldName] =
typeof value === 'number'
? Math.round(value)
: parseInt(String(value));
break;
case 'BOOLEAN':
cleanRecord[fieldName] =
typeof value === 'boolean' ? value : Boolean(value);
break;
case 'UTF8':
default:
if (typeof value === 'object') {
cleanRecord[fieldName] = JSON.stringify(value);
}
else {
cleanRecord[fieldName] = String(value);
}
break;
}
}
});
return cleanRecord;
}
async function writeParquetWithIntelligentSchema(filepath, records) {
if (records.length === 0) {
throw new Error('No records to write');
}
// Create intelligent schema
const schema = createIntelligentSchema(records);
console.log(`š Created schema with ${Object.keys(schema.schema).length} fields`);
// Create Parquet writer
const writer = await parquet.ParquetWriter.openFile(schema, filepath);
// Write records with proper type conversion
for (const record of records) {
const preparedRecord = prepareRecordForParquet(record, schema);
await writer.appendRow(preparedRecord);
}
// Close the writer
await writer.close();
}
// Run if called directly
if (require.main === module) {
const dataDir = process.argv[2] || 'data';
migrateSchemas(dataDir).catch(console.error);
}
//# sourceMappingURL=migrate-schemas.js.map