signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
407 lines • 16.8 kB
JavaScript
"use strict";
/**
* Parquet Export Service
*
* Handles periodic export of data from SQLite buffer to Parquet files.
* Provides crash recovery by checking for pending records on startup.
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.ParquetExportService = void 0;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const hive_path_builder_1 = require("../utils/hive-path-builder");
class ParquetExportService {
constructor(sqliteBuffer, parquetWriter, config, app) {
this.exportInterval = null;
this.isExporting = false;
this.lastExportTime = null;
this.totalExported = 0;
this.lastBatchExported = 0;
this.lastExportTrigger = null;
this.sqliteBuffer = sqliteBuffer;
this.parquetWriter = parquetWriter;
this.config = config;
this.app = app;
this.hivePathBuilder = new hive_path_builder_1.HivePathBuilder();
}
/**
* Start the export service
*
* No startup export — catchup for completed days is handled by
* exportAllUnexported() called from index.ts after a 10s delay.
* Today's data stays in SQLite for the History API to read live.
*/
start() {
this.app.debug(`ParquetExportService started (daily export mode, no startup blast)`);
}
/**
* Stop the export service
*/
stop() {
if (this.exportInterval) {
clearInterval(this.exportInterval);
this.exportInterval = null;
}
this.app.debug('ParquetExportService stopped');
}
/**
* Force an immediate export of completed days (excludes today)
*/
async forceExport() {
this.lastExportTrigger = 'forced';
return this.exportAllUnexported();
}
/**
* Build a flat-structure file path (legacy compatibility)
*/
buildFlatFilePath(context, signalkPath) {
// Clean context for filesystem
let contextPath;
if (context === 'vessels.self') {
contextPath = this.app.selfContext.replace(/\./g, '/').replace(/:/g, '_');
}
else if (context.startsWith('vessels.')) {
const vesselId = context.replace('vessels.', '').replace(/:/g, '_');
contextPath = `vessels/${vesselId}`;
}
else {
contextPath = context.replace(/:/g, '_').replace(/\./g, '/');
}
const dirPath = path.join(this.config.outputDirectory, contextPath, signalkPath.replace(/\./g, '/'));
const timestamp = new Date()
.toISOString()
.replace(/[:.]/g, '')
.slice(0, 15);
return path.join(dirPath, `${this.config.filenamePrefix}_${timestamp}.parquet`);
}
/**
* Generate a unique batch ID
*/
generateBatchId() {
const timestamp = new Date()
.toISOString()
.replace(/[:.]/g, '')
.slice(0, 15);
const random = Math.random().toString(36).substring(2, 8);
return `batch_${timestamp}_${random}`;
}
/**
* Get service status
*/
getStatus() {
return {
isRunning: true, // Always running in daily mode (scheduled from index.ts)
isExporting: this.isExporting,
lastExportTime: this.lastExportTime,
lastBatchExported: this.lastBatchExported,
totalExported: this.totalExported,
pendingRecords: this.sqliteBuffer.getPendingCount(),
dailyExportHour: this.config.dailyExportHour,
lastExportTrigger: this.lastExportTrigger,
mode: 'daily',
};
}
/**
* Get health check information
*/
getHealth() {
const stats = this.sqliteBuffer.getStats();
const healthy = !this.isExporting;
return {
healthy,
lastExportTime: this.lastExportTime,
pendingRecords: stats.pendingRecords,
bufferStats: stats,
};
}
/**
* Export all unexported data from SQLite to Parquet (complete days only)
* Used at startup to catch up on any missed exports.
* Excludes today's data to avoid creating partial day files that would
* conflict with the daily export.
*/
async exportAllUnexported() {
const startTime = Date.now();
const batchId = this.generateBatchId();
let totalRecordsExported = 0;
const allFilesCreated = [];
const allErrors = [];
// Get dates with unexported records (excludes today to avoid partial files)
const dates = this.sqliteBuffer.getDatesWithUnexportedRecords(true);
if (dates.length === 0) {
this.app.debug('[StartupExport] No unexported records found');
this.lastExportTime = new Date();
this.lastBatchExported = 0;
this.lastExportTrigger = 'startup';
return {
batchId,
recordsExported: 0,
filesCreated: [],
duration: Date.now() - startTime,
errors: [],
};
}
this.app.debug(`[StartupExport] Found unexported records for ${dates.length} dates: ${dates.join(', ')}`);
// Export each date
for (const dateStr of dates) {
const targetDate = new Date(dateStr + 'T00:00:00.000Z');
try {
const result = await this.exportDayToParquet(targetDate);
totalRecordsExported += result.recordsExported;
allFilesCreated.push(...result.filesCreated);
allErrors.push(...result.errors);
if (result.recordsExported > 0) {
this.app.debug(`[StartupExport] Exported ${result.recordsExported} records for ${dateStr}`);
}
}
catch (error) {
const errorMsg = `[StartupExport] Failed to export ${dateStr}: ${error.message}`;
this.app.error(errorMsg);
allErrors.push(errorMsg);
}
}
this.lastExportTime = new Date();
this.lastBatchExported = totalRecordsExported;
this.totalExported += totalRecordsExported;
this.lastExportTrigger = 'startup';
this.app.debug(`[StartupExport] Complete: ${totalRecordsExported} records to ${allFilesCreated.length} files in ${Date.now() - startTime}ms`);
return {
batchId,
recordsExported: totalRecordsExported,
filesCreated: allFilesCreated,
duration: Date.now() - startTime,
errors: allErrors,
};
}
/**
* Export a full day's data to Parquet files (one file per context/path)
* This creates consolidated daily files directly, without needing a separate
* consolidation step.
*
* @param targetDate The date to export (UTC). Typically yesterday.
* @returns Export result with details about files created
*/
async exportDayToParquet(targetDate) {
if (this.isExporting) {
this.app.debug('Export already in progress, skipping daily export');
return {
batchId: '',
recordsExported: 0,
filesCreated: [],
duration: 0,
errors: ['Export already in progress'],
};
}
this.isExporting = true;
const startTime = Date.now();
const batchId = this.generateBatchId();
const filesCreated = [];
const errors = [];
let recordsExported = 0;
const dateStr = targetDate.toISOString().slice(0, 10);
this.app.debug(`[DailyExport] Starting daily export for ${dateStr}`);
try {
// Get all distinct context/path combinations for this date
const pathsForDate = this.sqliteBuffer.getPathsForDate(targetDate);
if (pathsForDate.length === 0) {
this.app.debug(`[DailyExport] No data found for ${dateStr}`);
this.lastExportTime = new Date();
this.lastBatchExported = 0;
this.lastExportTrigger = 'daily';
return {
batchId,
recordsExported: 0,
filesCreated: [],
duration: Date.now() - startTime,
errors: [],
};
}
this.app.debug(`[DailyExport] Found ${pathsForDate.length} paths with data for ${dateStr}`);
// Export each context/path to its own file (batched to limit memory)
const BATCH_SIZE = 5000;
for (const { context, path: signalkPath } of pathsForDate) {
try {
const count = this.sqliteBuffer.getRecordCountForPathAndDate(context, signalkPath, targetDate);
if (count === 0) {
continue;
}
const firstBatch = this.sqliteBuffer.getRecordsForPathAndDateBatched(context, signalkPath, targetDate, BATCH_SIZE, 0);
let offset = BATCH_SIZE;
const filePath = await this.exportDailyGroupBatched(context, signalkPath, firstBatch, () => {
const batch = this.sqliteBuffer.getRecordsForPathAndDateBatched(context, signalkPath, targetDate, BATCH_SIZE, offset);
offset += BATCH_SIZE;
return batch;
}, targetDate);
if (filePath) {
filesCreated.push(filePath);
recordsExported += count;
// Mark records as exported by date range
this.sqliteBuffer.markDateExported(context, signalkPath, targetDate, batchId);
this.app.debug(`[DailyExport] Exported ${count} records for ${context}:${signalkPath}`);
}
}
catch (error) {
const errorMsg = `[DailyExport] Failed to export ${context}:${signalkPath}: ${error.message}`;
this.app.error(errorMsg);
errors.push(errorMsg);
}
}
// Cleanup old exported records
const cleaned = this.sqliteBuffer.cleanup();
if (cleaned > 0) {
this.app.debug(`[DailyExport] Cleaned up ${cleaned} old exported records`);
}
// Truncate WAL after heavy export+cleanup batch
try {
this.sqliteBuffer.checkpoint();
}
catch {
// Non-critical — WAL will be checkpointed eventually
}
this.lastExportTime = new Date();
this.lastBatchExported = recordsExported;
this.totalExported += recordsExported;
this.lastExportTrigger = 'daily';
this.app.debug(`[DailyExport] Complete: ${recordsExported} records to ${filesCreated.length} files in ${Date.now() - startTime}ms`);
return {
batchId,
recordsExported,
filesCreated,
duration: Date.now() - startTime,
errors,
};
}
finally {
this.isExporting = false;
}
}
/**
* Export a group of records as a timestamped file
* Uses consistent timestamped naming (same as startup exports) for simplicity
*/
async exportDailyGroup(context, signalkPath, records, targetDate, _batchId) {
if (records.length === 0)
return null;
// Build file path with timestamp (consistent with exportGroup)
let filePath;
if (this.config.useHivePartitioning) {
// Resolve vessels.self to actual vessel context
const resolvedContext = context === 'vessels.self' ? this.app.selfContext : context;
// Use Hive-style partitioning
// Directory is based on targetDate (for correct day partition)
// Filename uses CURRENT time for uniqueness (avoid overwrites)
const dirPath = this.hivePathBuilder.buildPath(this.config.outputDirectory, 'raw', resolvedContext, signalkPath, targetDate);
const timestampStr = new Date()
.toISOString()
.replace(/[:.]/g, '')
.slice(0, 15);
filePath = path.join(dirPath, `${this.config.filenamePrefix}_${timestampStr}.parquet`);
}
else {
// Use legacy flat structure with timestamp
filePath = this.buildFlatFilePath(context, signalkPath);
}
// Ensure directory exists
await fs.ensureDir(path.dirname(filePath));
// Write to temp file first for atomic operation
const tempFilePath = filePath + '.tmp';
try {
// Write records to temp file
await this.parquetWriter.writeRecords(tempFilePath, records);
// Validate the written file
const stats = await fs.stat(tempFilePath);
if (stats.size < 100) {
throw new Error('Written file is too small, likely corrupt');
}
// Atomic rename
await fs.rename(tempFilePath, filePath);
return filePath;
}
catch (error) {
// Clean up temp file on failure
try {
await fs.remove(tempFilePath);
}
catch {
// Ignore cleanup errors
}
throw error;
}
}
/**
* Export records in batches to avoid loading all rows into memory at once.
* firstBatch is used for schema detection; nextBatch callback pulls subsequent chunks.
*/
async exportDailyGroupBatched(context, signalkPath, firstBatch, nextBatch, targetDate) {
if (firstBatch.length === 0)
return null;
let filePath;
if (this.config.useHivePartitioning) {
const resolvedContext = context === 'vessels.self' ? this.app.selfContext : context;
const dirPath = this.hivePathBuilder.buildPath(this.config.outputDirectory, 'raw', resolvedContext, signalkPath, targetDate);
const timestampStr = new Date()
.toISOString()
.replace(/[:.]/g, '')
.slice(0, 15);
filePath = path.join(dirPath, `${this.config.filenamePrefix}_${timestampStr}.parquet`);
}
else {
filePath = this.buildFlatFilePath(context, signalkPath);
}
await fs.ensureDir(path.dirname(filePath));
const tempFilePath = filePath + '.tmp';
try {
await this.parquetWriter.writeParquetBatched(tempFilePath, firstBatch, nextBatch, signalkPath);
const stats = await fs.stat(tempFilePath);
if (stats.size < 100) {
throw new Error('Written file is too small, likely corrupt');
}
await fs.rename(tempFilePath, filePath);
return filePath;
}
catch (error) {
try {
await fs.remove(tempFilePath);
}
catch {
// Ignore cleanup errors
}
throw error;
}
}
}
exports.ParquetExportService = ParquetExportService;
//# sourceMappingURL=parquet-export-service.js.map