signalk-parquet
Version:
SignalK plugin and webapp that archives SK data to Parquet files with a regimen control system, advanced querying, Claude integrated AI analysis, spatial capabilities, and REST API.
614 lines • 28.4 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.toParquetFilePath = exports.toContextFilePath = void 0;
exports.default = default_1;
exports.initializeStreamingService = initializeStreamingService;
exports.shutdownStreamingService = shutdownStreamingService;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const parquet_writer_1 = require("./parquet-writer");
const HistoryAPI_1 = require("./HistoryAPI");
const api_routes_1 = require("./api-routes");
const historical_streaming_1 = require("./historical-streaming");
const commands_1 = require("./commands");
const claude_models_1 = require("./claude-models");
const data_handler_1 = require("./data-handler");
function default_1(app) {
const plugin = {
id: 'signalk-parquet',
name: 'SignalK to Parquet',
description: 'Save SignalK marine data directly to Parquet files with regimen-based control',
schema: {},
start: () => { },
stop: () => { },
registerWithRouter: undefined,
};
// Plugin state
const state = {
unsubscribes: [],
dataBuffers: new Map(),
activeRegimens: new Set(),
subscribedPaths: new Set(),
saveInterval: undefined,
consolidationInterval: undefined,
parquetWriter: undefined,
s3Client: undefined,
currentConfig: undefined,
commandState: {
registeredCommands: new Map(),
putHandlers: new Map(),
},
};
let currentPaths = [];
plugin.start = async function (options) {
// Get vessel MMSI from SignalK
const vesselMMSI = app.getSelfPath('mmsi') || app.getSelfPath('name') || 'unknown_vessel';
// Use SignalK's application data directory
const defaultOutputDir = path.join(app.getDataDirPath(), 'signalk-parquet');
// Validate and migrate Claude model if needed
let claudeConfig = options?.claudeIntegration || { enabled: false };
if (claudeConfig.model) {
const validatedModel = (0, claude_models_1.getValidClaudeModel)(claudeConfig.model);
if (validatedModel !== claudeConfig.model) {
app.debug(`⚠️ Saved Claude model "${claudeConfig.model}" is no longer supported. Migrating to ${validatedModel}`);
claudeConfig = { ...claudeConfig, model: validatedModel };
// Save migrated config
app.savePluginOptions({ ...options, claudeIntegration: claudeConfig }, (err) => {
if (err) {
app.error(`Failed to save migrated Claude model: ${err}`);
}
else {
app.debug(`✅ Successfully migrated Claude model to ${validatedModel}`);
}
});
}
}
state.currentConfig = {
bufferSize: options?.bufferSize || 1000,
saveIntervalSeconds: options?.saveIntervalSeconds || 30,
outputDirectory: options?.outputDirectory || defaultOutputDir,
filenamePrefix: options?.filenamePrefix || 'signalk_data',
retentionDays: options?.retentionDays || 7,
fileFormat: options?.fileFormat || 'parquet',
vesselMMSI: vesselMMSI,
s3Upload: options?.s3Upload || { enabled: false },
claudeIntegration: claudeConfig,
homePortLatitude: options?.homePortLatitude || 0,
homePortLongitude: options?.homePortLongitude || 0,
setCurrentLocationAction: options?.setCurrentLocationAction || {
setCurrentLocation: false,
},
// enableStreaming: options?.enableStreaming ?? false,
};
// Load webapp configuration including commands
const webAppConfig = (0, commands_1.loadWebAppConfig)(app);
currentPaths = webAppConfig.paths;
(0, commands_1.setCurrentCommands)(webAppConfig.commands);
// Initialize ParquetWriter
state.parquetWriter = new parquet_writer_1.ParquetWriter({
format: state.currentConfig.fileFormat,
app: app,
});
// Initialize S3 client if enabled
await (0, data_handler_1.initializeS3)(state.currentConfig, app);
state.s3Client = (0, data_handler_1.createS3Client)(state.currentConfig, app);
// Ensure output directory exists
fs.ensureDirSync(state.currentConfig.outputDirectory);
// Subscribe to command paths first (these control regimens)
(0, data_handler_1.subscribeToCommandPaths)(currentPaths, state, state.currentConfig, app);
// Check current command values at startup
(0, data_handler_1.initializeRegimenStates)(currentPaths, state, app);
// Initialize command state
(0, commands_1.initializeCommandState)(currentPaths, app);
// Start threshold monitoring system
(0, commands_1.startThresholdMonitoring)(app, state.currentConfig);
// Subscribe to data paths based on initial regimen states
(0, data_handler_1.updateDataSubscriptions)(currentPaths, state, state.currentConfig, app);
// Set up periodic save
state.saveInterval = setInterval(() => {
(0, data_handler_1.saveAllBuffers)(state.currentConfig, state, app);
}, state.currentConfig.saveIntervalSeconds * 1000);
// Set up daily consolidation (run at midnight UTC)
const now = new Date();
const nextMidnightUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1, 0, 0, 0, 0));
const msUntilMidnightUTC = nextMidnightUTC.getTime() - now.getTime();
setTimeout(() => {
(0, data_handler_1.consolidateYesterday)(state.currentConfig, state, app);
// Then run daily consolidation every 24 hours
state.consolidationInterval = setInterval(() => {
(0, data_handler_1.consolidateYesterday)(state.currentConfig, state, app);
}, 24 * 60 * 60 * 1000);
}, msUntilMidnightUTC);
// Run startup consolidation for missed previous days
setTimeout(() => {
(0, data_handler_1.consolidateMissedDays)(state.currentConfig, state, app);
}, 5000); // Wait 5 seconds after startup to avoid conflicts
// Upload all existing consolidated files to S3 (for catching up after BigInt fix)
if (state.currentConfig.s3Upload.enabled) {
setTimeout(() => {
(0, data_handler_1.uploadAllConsolidatedFilesToS3)(state.currentConfig, state, app);
}, 10000); // Wait 10 seconds after startup to avoid conflicts
}
// Register History API routes directly with the main app
try {
(0, HistoryAPI_1.registerHistoryApiRoute)(app, app.selfId, state.currentConfig.outputDirectory, app.debug, app, state.currentConfig.unitConversionCacheMinutes || 5 // Default to 5 minutes
);
}
catch (error) {
app.error(`Failed to register History API routes with main server: ${error}`);
}
// Initialize historical streaming service (for history API endpoints)
try {
state.historicalStreamingService = new historical_streaming_1.HistoricalStreamingService(app, state.currentConfig.outputDirectory);
}
catch (error) {
app.error(`Failed to initialize historical streaming service: ${error}`);
}
// Initialize runtime streaming service if enabled in configuration
// if (state.currentConfig.enableStreaming) {
// try {
// const result = await initializeStreamingService(state, app);
// if (!result.success) {
// app.error(`Failed to initialize runtime streaming service: ${result.error}`);
// }
// } catch (error) {
// app.error(`Error initializing runtime streaming service: ${error}`);
// }
// }
// Handle "Set Current Location" action
handleSetCurrentLocationAction(state.currentConfig).catch(err => {
app.error(`Error handling set current location action: ${err}`);
});
// Publish home port position to SignalK if configured
if (state.currentConfig.homePortLatitude && state.currentConfig.homePortLongitude &&
state.currentConfig.homePortLatitude !== 0 && state.currentConfig.homePortLongitude !== 0) {
publishHomePortToSignalK(state.currentConfig.homePortLatitude, state.currentConfig.homePortLongitude);
}
};
plugin.stop = function () {
// Stop threshold monitoring system
(0, commands_1.stopThresholdMonitoring)();
// Clear intervals
if (state.saveInterval) {
clearInterval(state.saveInterval);
}
if (state.consolidationInterval) {
clearInterval(state.consolidationInterval);
}
// Save any remaining buffered data
if (state.currentConfig) {
(0, data_handler_1.saveAllBuffers)(state.currentConfig, state, app);
}
// Unsubscribe from all paths
state.unsubscribes.forEach(unsubscribe => {
if (typeof unsubscribe === 'function') {
unsubscribe();
}
});
state.unsubscribes = [];
// Clean up stream subscriptions (new streambundle approach)
if (state.streamSubscriptions) {
state.streamSubscriptions.forEach(stream => {
if (stream && typeof stream.end === 'function') {
stream.end();
}
});
state.streamSubscriptions = [];
}
// Shutdown runtime streaming service
if (state.streamingService) {
try {
shutdownStreamingService(state, app);
}
catch (error) {
app.error(`Error shutting down runtime streaming service: ${error}`);
}
}
// Shutdown historical streaming service
if (state.historicalStreamingService) {
try {
state.historicalStreamingService.shutdown();
state.historicalStreamingService = undefined;
}
catch (error) {
app.error(`Error shutting down historical streaming service: ${error}`);
}
}
// Clear data structures
state.dataBuffers.clear();
state.activeRegimens.clear();
state.subscribedPaths.clear();
};
plugin.schema = {
type: 'object',
title: 'SignalK to Parquet Data Store',
description: "The archiving commands, paths and other processes are managed in the companion 'SignalK to Parquet Data Store' in the Webapp section.\n\nThese settings here underpin the system.",
properties: {
bufferSize: {
type: 'number',
title: 'Buffer Size',
description: 'Number of records to buffer before writing to file',
default: 1000,
minimum: 10,
maximum: 10000,
},
saveIntervalSeconds: {
type: 'number',
title: 'Save Interval (seconds)',
description: 'How often to save buffered data to files',
default: 30,
minimum: 5,
maximum: 300,
},
outputDirectory: {
type: 'string',
title: 'Output Directory',
description: 'Directory to save data files (defaults to application_data/{vessel}/signalk-parquet)',
default: '',
},
filenamePrefix: {
type: 'string',
title: 'Filename Prefix',
description: 'Prefix for generated filenames',
default: 'signalk_data',
},
fileFormat: {
type: 'string',
title: 'File Format',
description: 'Format for saved data files',
enum: ['json', 'csv', 'parquet'],
default: 'parquet',
},
retentionDays: {
type: 'number',
title: 'Retention Days',
description: 'Days to keep processed files',
default: 7,
minimum: 1,
maximum: 365,
},
unitConversionCacheMinutes: {
type: 'number',
title: 'Unit Conversion Cache Duration (minutes)',
description: 'How long to cache unit conversions from signalk-units-preference plugin before reloading. Lower values reflect preference changes faster but use more resources.',
default: 5,
minimum: 1,
maximum: 60,
},
s3Upload: {
type: 'object',
title: 'S3 Upload Configuration',
description: 'Optional S3 backup/archive functionality',
properties: {
enabled: {
type: 'boolean',
title: 'Enable S3 Upload',
description: 'Enable uploading files to Amazon S3',
default: false,
},
timing: {
type: 'string',
title: 'Upload Timing',
description: 'When to upload files to S3',
enum: ['realtime', 'consolidation'],
enumNames: [
'Real-time (after each file save)',
'At consolidation (daily)',
],
default: 'consolidation',
},
bucket: {
type: 'string',
title: 'S3 Bucket Name',
description: 'Name of the S3 bucket to upload to',
default: '',
},
region: {
type: 'string',
title: 'AWS Region',
description: 'AWS region where the S3 bucket is located',
default: 'us-east-1',
},
keyPrefix: {
type: 'string',
title: 'S3 Key Prefix',
description: 'Optional prefix for S3 object keys (e.g., "marine-data/")',
default: '',
},
accessKeyId: {
type: 'string',
title: 'AWS Access Key ID',
description: 'AWS Access Key ID (leave empty to use IAM role or environment variables)',
default: '',
},
secretAccessKey: {
type: 'string',
title: 'AWS Secret Access Key',
description: 'AWS Secret Access Key (leave empty to use IAM role or environment variables)',
default: '',
},
deleteAfterUpload: {
type: 'boolean',
title: 'Delete Local Files After Upload',
description: 'Delete local files after successful upload to S3',
default: false,
},
},
},
claudeIntegration: {
type: 'object',
title: 'Claude AI Analysis',
description: 'Configure Claude AI for intelligent data analysis and insights',
properties: {
enabled: {
type: 'boolean',
title: 'Enable Claude AI Analysis',
description: 'Enable AI-powered analysis of your maritime data',
default: false,
},
apiKey: {
type: 'string',
title: 'Claude API Key',
description: 'Your Anthropic Claude API key (get one at console.anthropic.com)',
default: '',
},
model: {
type: 'string',
title: 'Default Claude Model (Optional)',
description: 'Default Claude model for analysis. Can be overridden in the web interface.',
enum: [...claude_models_1.SUPPORTED_CLAUDE_MODELS],
enumNames: claude_models_1.SUPPORTED_CLAUDE_MODELS.map(m => claude_models_1.CLAUDE_MODEL_DESCRIPTIONS[m]),
default: claude_models_1.DEFAULT_CLAUDE_MODEL,
},
maxTokens: {
type: 'number',
title: 'Max Tokens',
description: 'Maximum tokens for Claude responses (higher = more detailed analysis)',
default: 4000,
minimum: 1000,
maximum: 8192,
},
temperature: {
type: 'number',
title: 'Temperature',
description: 'Analysis creativity (0.0 = focused, 1.0 = creative)',
default: 0.3,
minimum: 0.0,
maximum: 1.0,
},
autoAnalysis: {
type: 'object',
title: 'Automated Analysis',
description: 'Configure automated analysis features',
properties: {
daily: {
type: 'boolean',
title: 'Daily Summary Reports',
description: 'Generate daily analysis reports automatically',
default: false,
},
anomaly: {
type: 'boolean',
title: 'Anomaly Detection',
description: 'Automatically detect unusual patterns in real-time',
default: false,
},
threshold: {
type: 'number',
title: 'Anomaly Threshold',
description: 'Statistical threshold for anomaly detection (higher = fewer alerts)',
default: 2.5,
minimum: 1.0,
maximum: 5.0,
},
},
},
cacheEnabled: {
type: 'boolean',
title: 'Enable Result Caching',
description: 'Cache analysis results to reduce API calls and costs',
default: true,
},
},
},
homePortLatitude: {
type: 'number',
title: 'Home Port Latitude (Optional)',
description: 'Home port latitude for vessel context. If not set (0), will use vessel position from navigation.position',
default: 0,
},
homePortLongitude: {
type: 'number',
title: 'Home Port Longitude (Optional)',
description: 'Home port longitude for vessel context. If not set (0), will use vessel position from navigation.position',
default: 0,
},
setCurrentLocationAction: {
type: 'object',
title: 'Home Port Location Actions',
description: 'Actions for setting the home port location',
properties: {
setCurrentLocation: {
type: 'boolean',
title: 'Set Current Location as Home Port',
description: "Check this box and save to use the vessel's current position as the home port coordinates",
default: false,
},
},
},
// enableStreaming: {
// type: 'boolean',
// title: 'Enable WebSocket Streaming',
// description: 'Enable real-time streaming of historical data via WebSocket connections',
// default: false,
// },
},
};
// Webapp static files and API routes
plugin.registerWithRouter = function (router) {
(0, api_routes_1.registerApiRoutes)(router, state, app);
};
// Handle "Set Current Location" action
async function handleSetCurrentLocationAction(config) {
app.debug(`handleSetCurrentLocationAction called with setCurrentLocation: ${config.setCurrentLocationAction?.setCurrentLocation}`);
if (config.setCurrentLocationAction?.setCurrentLocation) {
// Get current position from SignalK
const currentPosition = getCurrentVesselPosition();
app.debug(`Current position: ${currentPosition ? `${currentPosition.latitude}, ${currentPosition.longitude}` : 'null'}`);
if (currentPosition) {
// Update the configuration with current position
const updatedConfig = {
...config,
homePortLatitude: currentPosition.latitude,
homePortLongitude: currentPosition.longitude,
setCurrentLocationAction: {
setCurrentLocation: false, // Reset the checkbox
},
};
// Save the updated configuration
app.savePluginOptions(updatedConfig, (err) => {
if (err) {
app.error(`Failed to save current location as home port: ${err}`);
}
else {
app.debug(`Set home port location to: ${currentPosition.latitude}, ${currentPosition.longitude}`);
// Update current config
state.currentConfig = updatedConfig;
// Publish home port position to SignalK
publishHomePortToSignalK(currentPosition.latitude, currentPosition.longitude);
}
});
}
else {
app.error('No current vessel position available. Ensure navigation.position is being published to SignalK.');
}
}
}
// Get current vessel position from SignalK
function getCurrentVesselPosition() {
try {
const position = app.getSelfPath('navigation.position');
if (position && position.value && position.value.latitude && position.value.longitude) {
return {
latitude: position.value.latitude,
longitude: position.value.longitude,
timestamp: new Date(position.timestamp || Date.now()),
};
}
}
catch (error) {
app.debug(`Error getting vessel position: ${error}`);
}
return null;
}
// Publish home port position to SignalK
function publishHomePortToSignalK(latitude, longitude) {
const homePortPosition = {
latitude: latitude,
longitude: longitude,
};
const delta = {
context: app.selfContext,
updates: [
{
$source: 'signalk-parquet.homePort',
timestamp: new Date().toISOString(),
values: [
{
path: 'navigation.homePort.position',
value: homePortPosition,
},
],
},
],
};
app.handleMessage(plugin.id, delta);
app.debug(`Published home port position to SignalK: ${latitude}, ${longitude}`);
}
return plugin;
}
// Streaming service lifecycle functions for runtime control
async function initializeStreamingService(state, app) {
try {
if (state.streamingService) {
return { success: true, error: 'Streaming service is already running' };
}
if (!state.currentConfig?.enableStreaming) {
return { success: false, error: 'Streaming is disabled in plugin configuration. Enable it in settings first.' };
}
// Reuse the existing historical streaming service instead of creating a new one
state.streamingService = state.historicalStreamingService;
state.streamingEnabled = true;
// Restore any previous subscriptions if available
// The historical streaming service will automatically handle incoming subscriptions
return { success: true };
}
catch (error) {
app.error(`Failed to initialize streaming service: ${error}`);
return { success: false, error: error.message };
}
}
function shutdownStreamingService(state, app) {
try {
if (!state.streamingService) {
return { success: true, error: 'Streaming service is not running' };
}
// Store active subscriptions for potential restoration
if (state.streamingService.getActiveSubscriptions) {
const activeSubscriptions = state.streamingService.getActiveSubscriptions();
if (activeSubscriptions.length > 0) {
state.restoredSubscriptions = new Map();
activeSubscriptions.forEach((sub, index) => {
state.restoredSubscriptions.set(`sub_${index}`, sub);
});
}
}
// Shutdown the streaming service
state.streamingService.shutdown();
state.streamingService = undefined;
state.streamingEnabled = false;
return { success: true };
}
catch (error) {
app.error(`Error shutting down streaming service: ${error}`);
return { success: false, error: error.message };
}
}
// Re-export utility functions for backward compatibility
var path_helpers_1 = require("./utils/path-helpers");
Object.defineProperty(exports, "toContextFilePath", { enumerable: true, get: function () { return path_helpers_1.toContextFilePath; } });
Object.defineProperty(exports, "toParquetFilePath", { enumerable: true, get: function () { return path_helpers_1.toParquetFilePath; } });
//# sourceMappingURL=index.js.map