signalk-parquet
Version:
SignalK plugin to save marine data directly to Parquet files with regimen-based control
1,084 lines • 98.8 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = default_1;
exports.toContextFilePath = toContextFilePath;
exports.toParquetFilePath = toParquetFilePath;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const express_1 = __importDefault(require("express"));
const parquet_writer_1 = require("./parquet-writer");
const glob_1 = require("glob");
const server_api_1 = require("@signalk/server-api");
// AWS S3 for file upload
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let S3Client,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
PutObjectCommand,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
ListObjectsV2Command,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
HeadObjectCommand;
Promise.resolve().then(() => __importStar(require('@aws-sdk/client-s3'))).then(s3 => {
S3Client = s3.S3Client;
PutObjectCommand = s3.PutObjectCommand;
ListObjectsV2Command = s3.ListObjectsV2Command;
HeadObjectCommand = s3.HeadObjectCommand;
})
.catch(() => {
// eslint-disable-next-line no-console
console.warn('AWS S3 SDK not available for file uploads');
});
// DuckDB for webapp queries
const node_api_1 = require("@duckdb/node-api");
const HistoryAPI_1 = require("./HistoryAPI");
// Global variables for path and command management
let currentPaths = [];
let currentCommands = [];
let commandHistory = [];
let appInstance;
// Command state management
const commandState = {
registeredCommands: new Map(),
putHandlers: new Map(),
};
// Enhanced function to load webapp configuration with backward compatibility
function loadWebAppConfig() {
const webAppConfigPath = path.join(appInstance.getDataDirPath(), 'signalk-parquet', 'webapp-config.json');
try {
if (fs.existsSync(webAppConfigPath)) {
const configData = fs.readFileSync(webAppConfigPath, 'utf8');
const rawConfig = JSON.parse(configData);
// Migrate old config format to new format with backward compatibility
const migratedPaths = (rawConfig.paths || []).map((path) => ({
path: path.path,
name: path.name,
enabled: path.enabled,
regimen: path.regimen,
source: path.source || undefined, // Add missing source field for streamer branch
context: path.context,
excludeMMSI: path.excludeMMSI,
}));
const migratedCommands = rawConfig.commands || [];
const migratedConfig = {
paths: migratedPaths,
commands: migratedCommands,
};
appInstance.debug(`Loaded and migrated ${migratedPaths.length} paths and ${migratedCommands.length} commands from existing config`);
// Save the migrated config back to preserve the new format
try {
fs.writeFileSync(webAppConfigPath, JSON.stringify(migratedConfig, null, 2));
appInstance.debug('Saved migrated configuration with source field compatibility');
}
catch (saveError) {
appInstance.debug(`Warning: Could not save migrated config: ${saveError}`);
}
return migratedConfig;
}
}
catch (error) {
appInstance.error(`Failed to load webapp configuration: ${error}`);
// BACKUP the broken file instead of destroying it
try {
const backupPath = webAppConfigPath + '.backup.' + Date.now();
if (fs.existsSync(webAppConfigPath)) {
fs.copyFileSync(webAppConfigPath, backupPath);
appInstance.debug(`Backed up broken config to: ${backupPath}`);
}
}
catch (backupError) {
appInstance.debug(`Could not backup broken config: ${backupError}`);
}
}
// Only create defaults if NO config file exists
if (!fs.existsSync(webAppConfigPath)) {
const defaultConfig = getDefaultWebAppConfig();
appInstance.debug('No existing configuration found, using default installation');
// Save the default configuration for future use
try {
const configDir = path.dirname(webAppConfigPath);
if (!fs.existsSync(configDir)) {
fs.mkdirSync(configDir, { recursive: true });
}
fs.writeFileSync(webAppConfigPath, JSON.stringify(defaultConfig, null, 2));
appInstance.debug('Saved default installation configuration');
}
catch (error) {
appInstance.debug(`Failed to save default configuration: ${error}`);
}
return defaultConfig;
}
// If file exists but couldn't be parsed, return empty config to avoid data loss
appInstance.debug('Config file exists but could not be parsed, returning empty config to avoid data loss');
return {
paths: [],
commands: [],
};
}
// Get default configuration for first-time installation
function getDefaultWebAppConfig() {
const defaultCommands = [
{
command: 'captureMoored',
path: 'commands.captureMoored',
registered: 'COMPLETED',
description: 'Capture data when moored (position and wind)',
active: false,
},
];
const defaultPaths = [
{
path: 'commands.captureMoored',
name: 'Command: captureMoored',
enabled: true,
regimen: 'commands',
source: undefined,
context: 'vessels.self',
},
{
path: 'navigation.position',
name: 'Navigation Position',
enabled: true,
regimen: 'captureMoored',
source: undefined,
context: 'vessels.self',
},
{
path: 'environment.wind.speedApparent',
name: 'Apparent Wind Speed',
enabled: true,
regimen: 'captureMoored',
source: undefined,
context: 'vessels.self',
},
];
return {
commands: defaultCommands,
paths: defaultPaths,
};
}
// Enhanced function to save webapp configuration
function saveWebAppConfig(paths, commands) {
const webAppConfigPath = path.join(appInstance.getDataDirPath(), 'signalk-parquet', 'webapp-config.json');
try {
const configDir = path.dirname(webAppConfigPath);
if (!fs.existsSync(configDir)) {
fs.mkdirSync(configDir, { recursive: true });
}
const webAppConfig = { paths, commands };
fs.writeFileSync(webAppConfigPath, JSON.stringify(webAppConfig, null, 2));
appInstance.debug(`Saved webapp configuration: ${paths.length} paths, ${commands.length} commands`);
}
catch (error) {
appInstance.error(`Failed to save webapp configuration: ${error}`);
}
}
// Legacy function for backward compatibility
function loadWebAppPaths() {
return loadWebAppConfig().paths;
}
// Legacy function for backward compatibility
function saveWebAppPaths(paths) {
saveWebAppConfig(paths, currentCommands);
}
function default_1(app) {
// Store app instance for global access
appInstance = app;
const plugin = {
id: 'signalk-parquet',
name: 'SignalK to Parquet',
description: 'Save SignalK marine data directly to Parquet files with regimen-based control',
schema: {},
start: () => { },
stop: () => { },
registerWithRouter: undefined,
};
// Plugin state
const state = {
unsubscribes: [],
dataBuffers: new Map(),
activeRegimens: new Set(),
subscribedPaths: new Set(),
saveInterval: undefined,
consolidationInterval: undefined,
parquetWriter: undefined,
s3Client: undefined,
currentConfig: undefined,
commandState: commandState,
};
plugin.start = async function (options) {
app.debug('Starting...');
// Get vessel MMSI from SignalK
const vesselMMSI = app.getSelfPath('mmsi') || app.getSelfPath('name') || 'unknown_vessel';
// Use SignalK's application data directory
const defaultOutputDir = path.join(app.getDataDirPath(), 'signalk-parquet');
state.currentConfig = {
bufferSize: options?.bufferSize || 1000,
saveIntervalSeconds: options?.saveIntervalSeconds || 30,
outputDirectory: options?.outputDirectory || defaultOutputDir,
filenamePrefix: options?.filenamePrefix || 'signalk_data',
retentionDays: options?.retentionDays || 7,
fileFormat: options?.fileFormat || 'parquet',
vesselMMSI: vesselMMSI,
s3Upload: options?.s3Upload || { enabled: false },
};
// Load webapp configuration including commands
const webAppConfig = loadWebAppConfig();
currentPaths = webAppConfig.paths;
currentCommands = webAppConfig.commands;
// Initialize ParquetWriter
state.parquetWriter = new parquet_writer_1.ParquetWriter({
format: state.currentConfig.fileFormat,
app: app,
});
// Initialize S3 client if enabled
app.debug(`DEBUG: S3 setup - enabled: ${state.currentConfig.s3Upload.enabled}, S3Client available: ${!!S3Client}`);
if (state.currentConfig.s3Upload.enabled) {
// Wait for AWS SDK import to complete
try {
if (!S3Client) {
app.debug(`DEBUG: Waiting for AWS SDK import...`);
const awsS3 = await Promise.resolve().then(() => __importStar(require('@aws-sdk/client-s3')));
S3Client = awsS3.S3Client;
PutObjectCommand = awsS3.PutObjectCommand;
ListObjectsV2Command = awsS3.ListObjectsV2Command;
HeadObjectCommand = awsS3.HeadObjectCommand;
app.debug(`DEBUG: AWS SDK imported successfully`);
}
}
catch (importError) {
app.debug(`DEBUG: Failed to import AWS SDK: ${importError}`);
S3Client = undefined;
}
}
if (state.currentConfig.s3Upload.enabled && S3Client) {
try {
const s3Config = {
region: state.currentConfig.s3Upload.region || 'us-east-1',
};
// Add credentials if provided
if (state.currentConfig.s3Upload.accessKeyId &&
state.currentConfig.s3Upload.secretAccessKey) {
s3Config.credentials = {
accessKeyId: state.currentConfig.s3Upload.accessKeyId,
secretAccessKey: state.currentConfig.s3Upload.secretAccessKey,
};
app.debug(`DEBUG: S3 credentials provided`);
}
else {
app.debug(`DEBUG: No S3 credentials provided, using default AWS profile`);
}
app.debug(`DEBUG: Creating S3 client with region: ${s3Config.region}`);
state.s3Client = new S3Client(s3Config);
app.debug(`S3 client initialized for bucket: ${state.currentConfig.s3Upload.bucket}`);
}
catch (error) {
app.debug(`Error initializing S3 client: ${error}`);
state.s3Client = undefined;
}
}
else {
app.debug(`DEBUG: S3 client not initialized - enabled: ${state.currentConfig.s3Upload.enabled}, S3Client: ${!!S3Client}`);
}
// Ensure output directory exists
fs.ensureDirSync(state.currentConfig.outputDirectory);
// Subscribe to command paths first (these control regimens)
subscribeToCommandPaths(state.currentConfig);
// Check current command values at startup
initializeRegimenStates(state.currentConfig);
// Initialize command state
initializeCommandState();
// Subscribe to data paths based on initial regimen states
updateDataSubscriptions(state.currentConfig);
// Set up periodic save
state.saveInterval = setInterval(() => {
saveAllBuffers(state.currentConfig);
}, state.currentConfig.saveIntervalSeconds * 1000);
// Set up daily consolidation (run at midnight UTC)
const now = new Date();
const nextMidnightUTC = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1, 0, 0, 0, 0));
const msUntilMidnightUTC = nextMidnightUTC.getTime() - now.getTime();
app.debug(`Next consolidation at ${nextMidnightUTC.toISOString()} (in ${Math.round(msUntilMidnightUTC / 1000 / 60)} minutes)`);
setTimeout(() => {
consolidateYesterday(state.currentConfig);
// Then run daily consolidation every 24 hours
state.consolidationInterval = setInterval(() => {
consolidateYesterday(state.currentConfig);
}, 24 * 60 * 60 * 1000);
}, msUntilMidnightUTC);
// Run startup consolidation for missed previous days
setTimeout(() => {
consolidateMissedDays(state.currentConfig);
}, 5000); // Wait 5 seconds after startup to avoid conflicts
// Upload all existing consolidated files to S3 (for catching up after BigInt fix)
if (state.currentConfig.s3Upload.enabled) {
setTimeout(() => {
uploadAllConsolidatedFilesToS3(state.currentConfig);
}, 10000); // Wait 10 seconds after startup to avoid conflicts
}
(0, HistoryAPI_1.registerHistoryApiRoute)(app, app.selfId, state.currentConfig?.outputDirectory || 'data', app.debug);
app.debug('Started');
};
plugin.stop = function () {
app.debug('Stopping...');
// Clear intervals
if (state.saveInterval) {
clearInterval(state.saveInterval);
}
if (state.consolidationInterval) {
clearInterval(state.consolidationInterval);
}
// Save any remaining buffered data
saveAllBuffers();
// Unsubscribe from all paths
state.unsubscribes.forEach(unsubscribe => {
if (typeof unsubscribe === 'function') {
unsubscribe();
}
});
state.unsubscribes = [];
// Clean up stream subscriptions (new streambundle approach)
if (state.streamSubscriptions) {
state.streamSubscriptions.forEach(stream => {
if (stream && typeof stream.end === 'function') {
stream.end();
}
});
state.streamSubscriptions = [];
}
// Clear data structures
state.dataBuffers.clear();
state.activeRegimens.clear();
state.subscribedPaths.clear();
};
// Command Management Functions
// Initialize command state on plugin start
function initializeCommandState() {
// Clear existing command state
commandState.registeredCommands.clear();
commandState.putHandlers.clear();
// Re-register commands from configuration
currentCommands.forEach((commandConfig) => {
const result = registerCommand(commandConfig.command, commandConfig.description);
if (result.state === 'COMPLETED') {
app.debug(`✅ Restored command: ${commandConfig.command}`);
}
else {
app.error(`❌ Failed to restore command: ${commandConfig.command} - ${result.message}`);
}
});
// Ensure all commands have path configurations (for backwards compatibility)
let addedMissingPaths = false;
currentCommands.forEach((commandConfig) => {
const commandPath = `commands.${commandConfig.command}`;
const existingCommandPath = currentPaths.find(p => p.path === commandPath);
if (!existingCommandPath) {
const commandPathConfig = {
path: commandPath,
name: `Command: ${commandConfig.command}`,
enabled: true,
regimen: undefined,
source: undefined,
context: 'vessels.self',
excludeMMSI: undefined,
};
currentPaths.push(commandPathConfig);
addedMissingPaths = true;
app.debug(`✅ Added missing path configuration for existing command: ${commandConfig.command}`);
}
});
// Save the updated configuration if we added missing paths
if (addedMissingPaths) {
saveWebAppConfig(currentPaths, currentCommands);
}
// Reset all commands to false on startup
currentCommands.forEach((commandConfig) => {
initializeCommandValue(commandConfig.command, false);
});
app.debug(`🎮 Command state initialized with ${currentCommands.length} commands`);
}
// Command registration with full type safety
function registerCommand(commandName, description) {
try {
// Validate command name
if (!isValidCommandName(commandName)) {
return {
state: 'FAILED',
statusCode: 400,
message: 'Invalid command name. Use alphanumeric characters and underscores only.',
timestamp: new Date().toISOString(),
};
}
// Check if command already exists
if (commandState.registeredCommands.has(commandName)) {
return {
state: 'FAILED',
statusCode: 409,
message: `Command '${commandName}' already registered`,
timestamp: new Date().toISOString(),
};
}
const commandPath = `commands.${commandName}`;
const fullPath = `vessels.self.${commandPath}`;
// Create command configuration
const commandConfig = {
command: commandName,
path: fullPath,
registered: new Date().toISOString(),
description: description || `Command: ${commandName}`,
active: false,
lastExecuted: undefined,
};
// Create PUT handler with proper typing
const putHandler = (context, path,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
value, _callback) => {
app.debug(`Handling PUT for commands.${commandName} with value: ${JSON.stringify(value)}`);
return executeCommand(commandName, Boolean(value));
};
// Register PUT handler with SignalK
app.registerPutHandler('vessels.self', commandPath, putHandler, //FIXME server api registerPutHandler is incorrectly typed https://github.com/SignalK/signalk-server/pull/2043
'zennora-parquet-commands');
// Store command and handler
commandState.registeredCommands.set(commandName, commandConfig);
commandState.putHandlers.set(commandName, putHandler);
// Initialize command value to false
initializeCommandValue(commandName, false);
// Update current commands and save
currentCommands = Array.from(commandState.registeredCommands.values());
// Automatically create a path configuration for this command
const commandPathConfig = {
path: commandPath,
name: `Command: ${commandName}`,
enabled: true, // Always enabled so it gets subscribed to
regimen: undefined, // Commands don't need regimen control
source: undefined,
context: 'vessels.self',
excludeMMSI: undefined,
};
// Check if this command path already exists
const existingCommandPath = currentPaths.find(p => p.path === commandPath);
if (!existingCommandPath) {
currentPaths.push(commandPathConfig);
app.debug(`✅ Auto-created path configuration for command: ${commandName}`);
}
saveWebAppConfig(currentPaths, currentCommands);
// Log command registration
addCommandHistoryEntry(commandName, 'REGISTER', undefined, true);
app.debug(`✅ Registered command: ${commandName} at ${fullPath}`);
return {
state: 'COMPLETED',
statusCode: 200,
message: `Command '${commandName}' registered successfully with automatic path configuration`,
timestamp: new Date().toISOString(),
};
}
catch (error) {
const errorMessage = `Failed to register command '${commandName}': ${error}`;
app.error(errorMessage);
return {
state: 'FAILED',
statusCode: 500,
message: errorMessage,
timestamp: new Date().toISOString(),
};
}
}
// Command unregistration with type safety
function unregisterCommand(commandName) {
try {
const commandConfig = commandState.registeredCommands.get(commandName);
if (!commandConfig) {
return {
state: 'FAILED',
statusCode: 404,
message: `Command '${commandName}' not found`,
timestamp: new Date().toISOString(),
};
}
// Remove PUT handler (SignalK API doesn't have unregister, but we can track it)
commandState.putHandlers.delete(commandName);
commandState.registeredCommands.delete(commandName);
// Remove the auto-created path configuration for this command
const commandPath = `commands.${commandName}`;
const commandPathIndex = currentPaths.findIndex(p => p.path === commandPath);
if (commandPathIndex !== -1) {
currentPaths.splice(commandPathIndex, 1);
app.debug(`🗑️ Removed auto-created path configuration for command: ${commandName}`);
}
// Update current commands and save
currentCommands = Array.from(commandState.registeredCommands.values());
saveWebAppConfig(currentPaths, currentCommands);
// Log command unregistration
addCommandHistoryEntry(commandName, 'UNREGISTER', undefined, true);
app.debug(`🗑️ Unregistered command: ${commandName}`);
return {
state: 'COMPLETED',
statusCode: 200,
message: `Command '${commandName}' unregistered successfully with path cleanup`,
timestamp: new Date().toISOString(),
};
}
catch (error) {
const errorMessage = `Failed to unregister command '${commandName}': ${error}`;
app.error(errorMessage);
return {
state: 'FAILED',
statusCode: 500,
message: errorMessage,
timestamp: new Date().toISOString(),
};
}
}
// Command execution with full type safety
function executeCommand(commandName, value) {
try {
const commandConfig = commandState.registeredCommands.get(commandName);
if (!commandConfig) {
return {
state: 'FAILED',
statusCode: 404,
message: `Command '${commandName}' not found`,
timestamp: new Date().toISOString(),
};
}
// Execute the command by sending delta
const timestamp = new Date().toISOString();
const delta = {
context: 'vessels.self',
updates: [
{
$source: 'signalk-parquet-commands',
timestamp: timestamp,
values: [
{
path: `commands.${commandName}`,
value: value,
},
],
},
],
};
// Send delta message
//FIXME see if delta can be Delta from the beginning
app.handleMessage('signalk-parquet', delta);
// Update command state
commandConfig.active = value;
commandConfig.lastExecuted = timestamp;
commandState.registeredCommands.set(commandName, commandConfig);
// Update current commands and save
currentCommands = Array.from(commandState.registeredCommands.values());
saveWebAppConfig(currentPaths, currentCommands);
// Log command execution
addCommandHistoryEntry(commandName, 'EXECUTE', value, true);
app.debug(`🎮 Executed command: ${commandName} = ${value}`);
return {
state: 'COMPLETED',
statusCode: 200,
message: `Command '${commandName}' executed: ${value}`,
timestamp: timestamp,
};
}
catch (error) {
const errorMessage = `Failed to execute command '${commandName}': ${error}`;
app.error(errorMessage);
addCommandHistoryEntry(commandName, 'EXECUTE', value, false, errorMessage);
return {
state: 'FAILED',
statusCode: 500,
message: errorMessage,
timestamp: new Date().toISOString(),
};
}
}
// Helper functions with type safety
function isValidCommandName(commandName) {
const validPattern = /^[a-zA-Z0-9_]+$/;
return (validPattern.test(commandName) &&
commandName.length > 0 &&
commandName.length <= 50);
}
function initializeCommandValue(commandName, value) {
const timestamp = new Date().toISOString();
const delta = {
context: 'vessels.self',
updates: [
{
$source: 'signalk-parquet-commands',
timestamp,
values: [
{
path: `commands.${commandName}`,
value,
},
],
},
],
};
//FIXME
app.handleMessage('signalk-parquet', delta);
}
function addCommandHistoryEntry(command, action, value, success = true, error) {
const entry = {
command,
action,
value,
timestamp: new Date().toISOString(),
success,
error,
};
commandHistory.push(entry);
// Keep only last 100 entries
if (commandHistory.length > 100) {
commandHistory = commandHistory.slice(-100);
}
}
// Subscribe to command paths that control regimens using proper subscription manager
function subscribeToCommandPaths(config) {
const commandPaths = currentPaths.filter((pathConfig) => pathConfig &&
pathConfig.path &&
pathConfig.path.startsWith('commands.') &&
pathConfig.enabled);
if (commandPaths.length === 0)
return;
const commandSubscription = {
context: 'vessels.self',
subscribe: commandPaths.map((pathConfig) => ({
path: pathConfig.path,
period: 1000, // Check commands every second
policy: 'fixed',
})),
};
app.debug(`Subscribing to ${commandPaths.length} command paths via subscription manager`);
app.subscriptionmanager.subscribe(commandSubscription, state.unsubscribes, (subscriptionError) => {
app.debug(`Command subscription error: ${subscriptionError}`);
}, (delta) => {
// Process each update in the delta
if (delta.updates) {
delta.updates.forEach((update) => {
if ((0, server_api_1.hasValues)(update)) {
update.values.forEach((valueUpdate) => {
const pathConfig = commandPaths.find(p => p.path === valueUpdate.path);
if (pathConfig) {
handleCommandMessage(valueUpdate, pathConfig, config, update);
}
});
}
});
}
});
commandPaths.forEach(pathConfig => {
state.subscribedPaths.add(pathConfig.path);
});
}
// Handle command messages (regimen control) - now receives complete delta structure
function handleCommandMessage(valueUpdate, pathConfig, config, update) {
try {
app.debug(`📦 Received command update for ${pathConfig.path}: ${JSON.stringify(valueUpdate, null, 2)}`);
// Check source filter if specified for commands too
if (pathConfig.source && pathConfig.source.trim() !== '') {
const messageSource = update.$source || (update.source ? update.source.label : null);
if (messageSource !== pathConfig.source.trim()) {
app.debug(`🚫 Command from source "${messageSource}" filtered out (expecting "${pathConfig.source.trim()}")`);
return;
}
}
if (valueUpdate.value !== undefined) {
const commandName = extractCommandName(pathConfig.path);
const isActive = Boolean(valueUpdate.value);
app.debug(`Command ${commandName}: ${isActive ? 'ACTIVE' : 'INACTIVE'}`);
if (isActive) {
state.activeRegimens.add(commandName);
}
else {
state.activeRegimens.delete(commandName);
}
// Debug active regimens state
app.debug(`🎯 Active regimens: [${Array.from(state.activeRegimens).join(', ')}]`);
// Update data subscriptions based on new regimen state
updateDataSubscriptions(config);
// Buffer this command change with complete metadata
const bufferKey = `${pathConfig.context || 'vessels.self'}:${pathConfig.path}`;
bufferData(bufferKey, {
received_timestamp: new Date().toISOString(),
signalk_timestamp: update.timestamp || new Date().toISOString(),
context: 'vessels.self',
path: valueUpdate.path,
value: valueUpdate.value,
source: update.source ? JSON.stringify(update.source) : undefined,
source_label: update.$source ||
(update.source ? update.source.label : undefined),
source_type: update.source ? update.source.type : undefined,
source_pgn: update.source ? update.source.pgn : undefined,
source_src: update.source ? update.source.src : undefined,
//FIXME
// meta: valueUpdate.meta
// ? JSON.stringify(valueUpdate.meta)
// : undefined,
}, config);
}
}
catch (error) {
app.debug(`Error handling command message: ${error}`);
}
}
// Helper function to handle wildcard contexts
function handleWildcardContext(pathConfig) {
const context = pathConfig.context || 'vessels.self';
if (context === 'vessels.*') {
// For vessels.*, we create a subscription that will receive deltas from any vessel
// The actual filtering by MMSI will happen in the delta handler
return {
...pathConfig,
context: 'vessels.*', // Keep the wildcard for the subscription
};
}
// Not a wildcard, return as-is
return pathConfig;
}
// Helper function to check if a vessel should be excluded based on MMSI
function shouldExcludeVessel(vesselContext, pathConfig) {
if (!pathConfig.excludeMMSI || pathConfig.excludeMMSI.length === 0) {
return false; // No exclusions specified
}
try {
// For vessels.self, use getSelfPath
if (vesselContext === 'vessels.self') {
const mmsiData = app.getSelfPath('mmsi');
if (mmsiData && mmsiData.value) {
const mmsi = String(mmsiData.value);
return pathConfig.excludeMMSI.includes(mmsi);
}
}
else {
// For other vessels, we would need to get their MMSI from the delta or other means
// For now, we'll skip MMSI filtering for other vessels
app.debug(`MMSI filtering not implemented for vessel context: ${vesselContext}`);
}
}
catch (error) {
app.debug(`Error checking MMSI for vessel ${vesselContext}: ${error}`);
}
return false; // Don't exclude if we can't determine MMSI
}
// Update data path subscriptions based on active regimens
function updateDataSubscriptions(config) {
// First, unsubscribe from all existing subscriptions
state.unsubscribes.forEach(unsubscribe => {
if (typeof unsubscribe === 'function') {
unsubscribe();
}
});
state.unsubscribes = [];
state.subscribedPaths.clear();
app.debug('Cleared all existing subscriptions');
// Re-subscribe to command paths
subscribeToCommandPaths(config);
// Now subscribe to data paths using currentPaths
const dataPaths = currentPaths.filter((pathConfig) => pathConfig &&
pathConfig.path &&
!pathConfig.path.startsWith('commands.'));
const shouldSubscribePaths = dataPaths.filter((pathConfig) => shouldSubscribeToPath(pathConfig));
// Handle wildcard contexts (like vessels.*)
const processedPaths = shouldSubscribePaths.map(pathConfig => handleWildcardContext(pathConfig));
if (processedPaths.length === 0) {
app.debug('No data paths need subscription currently');
return;
}
// Group paths by context for separate subscriptions
const contextGroups = new Map();
processedPaths.forEach((pathConfig) => {
const context = (pathConfig.context || 'vessels.self');
if (!contextGroups.has(context)) {
contextGroups.set(context, []);
}
contextGroups.get(context).push(pathConfig);
});
// Use app.streambundle approach as recommended by SignalK developer
// This avoids server arbitration and provides true source filtering
contextGroups.forEach((pathConfigs, context) => {
app.debug(`Creating ${pathConfigs.length} streambundle subscriptions for context ${context}`);
pathConfigs.forEach((pathConfig) => {
// Show MMSI exclusion config for troubleshooting
if (pathConfig.excludeMMSI && pathConfig.excludeMMSI.length > 0) {
app.debug(`🔧 Path ${pathConfig.path} has MMSI exclusions: [${pathConfig.excludeMMSI.join(', ')}]`);
}
// Create individual stream for each path (developer's recommended approach)
const stream = app.streambundle
.getBus(pathConfig.path)
.filter((normalizedDelta) => {
// Filter by source if specified
if (pathConfig.source && pathConfig.source.trim() !== '') {
const expectedSource = pathConfig.source.trim();
const actualSource = normalizedDelta.$source;
if (actualSource !== expectedSource) {
return false;
}
}
// Filter by context
const targetContext = pathConfig.context || 'vessels.self';
if (targetContext === 'vessels.*') {
// For wildcard, accept any vessel context
if (!normalizedDelta.context.startsWith('vessels.')) {
return false;
}
}
else if (targetContext === 'vessels.self') {
// For vessels.self, check if this is the server's own vessel
const selfContext = app.selfContext;
const selfVessel = app.getSelfPath('') || {};
const selfMMSI = selfVessel.mmsi;
const selfUuid = app.getSelfPath('uuid');
// Check if the context matches the server's self vessel
let isSelfVessel = false;
if (normalizedDelta.context === 'vessels.self') {
isSelfVessel = true;
}
else if (normalizedDelta.context === selfContext) {
isSelfVessel = true;
}
else if (selfMMSI &&
normalizedDelta.context.includes(selfMMSI)) {
isSelfVessel = true;
}
else if (selfUuid &&
normalizedDelta.context.includes(selfUuid)) {
isSelfVessel = true;
}
if (!isSelfVessel) {
return false;
}
}
else {
// For specific context, match exactly
if (normalizedDelta.context !== targetContext) {
return false;
}
}
// MMSI exclusion filtering
if (pathConfig.excludeMMSI && pathConfig.excludeMMSI.length > 0) {
const contextHasExcludedMMSI = pathConfig.excludeMMSI.some(mmsi => normalizedDelta.context.includes(mmsi));
if (contextHasExcludedMMSI) {
return false;
}
}
return true;
})
.debounceImmediate(1000) // Built-in debouncing as recommended
.onValue((normalizedDelta) => {
handleStreamData(normalizedDelta, pathConfig, config);
});
// Store stream reference for cleanup (instead of unsubscribe functions)
state.streamSubscriptions = state.streamSubscriptions || [];
state.streamSubscriptions.push(stream);
state.subscribedPaths.add(pathConfig.path);
});
});
}
// Determine if we should subscribe to a path based on regimens
function shouldSubscribeToPath(pathConfig) {
// Always subscribe if explicitly enabled
if (pathConfig.enabled) {
app.debug(`✅ Path ${pathConfig.path} enabled (always on)`);
return true;
}
// Check if any required regimens are active
if (pathConfig.regimen) {
const requiredRegimens = pathConfig.regimen.split(',').map(r => r.trim());
const hasActiveRegimen = requiredRegimens.some(regimen => state.activeRegimens.has(regimen));
app.debug(`🔍 Path ${pathConfig.path} requires regimens [${requiredRegimens.join(', ')}], active: [${Array.from(state.activeRegimens).join(', ')}] → ${hasActiveRegimen ? 'SUBSCRIBE' : 'SKIP'}`);
return hasActiveRegimen;
}
app.debug(`❌ Path ${pathConfig.path} has no regimen control and not enabled`);
return false;
}
// Handle data messages from SignalK - now receives complete delta structure
// Legacy handler - kept for rollback capability
// eslint-disable-next-line @typescript-eslint/no-unused-vars
function handleDataMessage(valueUpdate, pathConfig, config, update, delta) {
try {
// Check if this vessel should be excluded based on MMSI
const vesselContext = delta.context || 'vessels.self';
if (shouldExcludeVessel(vesselContext, pathConfig)) {
app.debug(`Excluding data from vessel ${vesselContext} due to MMSI filter`);
return;
}
// Check source filter if specified
if (pathConfig.source && pathConfig.source.trim() !== '') {
const messageSource = update.$source || (update.source ? update.source.label : null);
if (messageSource !== pathConfig.source.trim()) {
// Source doesn't match filter, skip this message
return;
}
}
// Retrieve metadata for this path
let metadata;
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const pathMetadata = app.getMetadata?.(valueUpdate.path);
if (pathMetadata) {
metadata = JSON.stringify(pathMetadata);
}
}
catch (error) {
// Metadata retrieval failed, continue without it
app.debug(`Failed to retrieve metadata for ${valueUpdate.path}: ${error}`);
}
const record = {
received_timestamp: new Date().toISOString(),
signalk_timestamp: update.timestamp || new Date().toISOString(),
context: delta.context || pathConfig.context || 'vessels.self', // Use actual context from delta message
path: valueUpdate.path,
value: null,
value_json: undefined,
source: update.source ? JSON.stringify(update.source) : undefined,
source_label: update.$source || (update.source ? update.source.label : undefined),
source_type: update.source ? update.source.type : undefined,
source_pgn: update.source ? update.source.pgn : undefined,
source_src: update.source ? update.source.src : undefined,
meta: metadata ||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(valueUpdate.meta
? // eslint-disable-next-line @typescript-eslint/no-explicit-any
JSON.stringify(valueUpdate.meta)
: undefined),
};
// Handle different value types (matching Python logic)
if (typeof valueUpdate.value === 'object' && valueUpdate.value !== null) {
record.value_json = JSON.stringify(valueUpdate.value);
// Flatten object properties for easier querying
for (const [key, val] of Object.entries(valueUpdate.value)) {
if (typeof val === 'string' ||
typeof val === 'number' ||
typeof val === 'boolean') {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
record[`value_${key}`] = val;
}
}
}
else {
record.value = valueUpdate.value;
}
// Use actual context + path as buffer key to separate data from different vessels
const actualContext = delta.context || pathConfig.context || 'vessels.self';
const bufferKey = `${actualContext}:${pathConfig.path}`;
bufferData(bufferKey, record, config);
}
catch (error) {
app.debug(`Error handling data message: ${error}`);
}
}
// New handler for streambundle data (developer's recommended approach)
function handleStreamData(normalizedDelta, pathConfig, config) {
try {
// Retrieve metadata for this path
let metadata;
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const pathMetadata = app.getMetadata?.(normalizedDelta.path);
if (pathMetadata) {
metadata = JSON.stringify(pathMetadata);
}
}
catch (error) {
// Metadata retrieval failed, continue without it
app.debug(`Failed to retrieve metadata for ${normalizedDelta.path}: ${error}`);
}
const record = {
received_timestamp: new Date().toISOString(),
signalk_timestamp: normalizedDelta.timestamp || new Date().toISOString(),
context: normalizedDelta.context || pathConfig.context || 'vessels.self',
path: normalizedDelta.path,
value: null,
value_json: undefined,
source: normalizedDelta.source
? JSON.stringify(normalizedDelta.source)
: undefined,
source_label: normalizedDelta.$source || undefined,
source_type: normalizedDelta.source
? normalizedDelta.source.type
: undefined,
source_pgn: normalizedDelta.source
? normalizedDelta.source.pgn
: undefined,
source_src: normalizedDelta.source
? normalizedDelta.source.src
: undefined,
meta: metadata,
};
// Handle complex values
if (typeof normalizedDelta.value === 'object' &&
normalizedDelta.value !== null) {
record.value_json = JSON.stringify(normalizedDelta.value);
// Extract key properties as columns for easier querying
Object.entries(normalizedDelta.value).forEach(([key, val]) => {
if (typeof val === 'string' ||
typeof val === 'number' ||
typeof val === 'boolean') {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
record[`value_${key}`] = val;
}
});
}
else {
record.value = normalizedDelta.value;
}
// Use actual context + path as buffer key to separate data from different vessels
const bufferKey = `${normalizedDelta.context}:${pathConfig.path}`;
bufferData(bufferKey, record, config);
}
catch (error) {
app.debug(`Error handling stream data: ${error}`);
}
}
// Buffer data and trigger save if buffer is full
function bufferData(signalkPath, record, config) {
if (!state.dataBuffers.has(signalkPath)) {
state.dataBuffers.set(signalkPath, []);
}
const buffer = state.dataBuffers.get(signalkPath);
buffer.push(record);
if (buffer.length >= config.bufferSize) {
// Extract the actual SignalK path from the buffer key (context:path format)
// Find the separator between context and path - look for the last colon followed by a valid SignalK path
const pathMatch = signalkPath.match(/^.*:([a-zA-Z][a-zA-Z0-9._]*)$/);
const actualPath = pathMatch ? pathMatch[1] : signalkPath;
const urnMatch = signalkPath.match(/^([^:]+):/);
const urn = urnMatch ? urnMatch[1] : 'vessels.self';
app.debug(`💾 Buffer full: ${buffer.length} rows | ${urn} | ${actualPath} | trigger=buffer_full`);
saveBuffe