signalk-parquet
Version:
Vessel data Parquet file archive with automated value and geospatial triggers. History API compliant with cloud backups and queries.
888 lines • 39.2 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.createR2Client = exports.createS3Client = exports.initializeS3 = void 0;
exports.initializeCloudSDK = initializeCloudSDK;
exports.createCloudClient = createCloudClient;
exports.getCloudTarget = getCloudTarget;
exports.subscribeToCommandPaths = subscribeToCommandPaths;
exports.updateDataSubscriptions = updateDataSubscriptions;
exports.saveAllBuffers = saveAllBuffers;
exports.initializeRegimenStates = initializeRegimenStates;
exports.uploadAllConsolidatedFilesToS3 = uploadAllConsolidatedFilesToS3;
exports.uploadConsolidatedFilesToS3 = uploadConsolidatedFilesToS3;
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const glob_1 = require("glob");
const util_1 = require("util");
// Wrap glob for compatibility with both glob@7.x (callbacks) and glob@11.x (promises)
const glob = async (pattern, options) => {
// If glob returns a Promise (glob@11.x), use it directly
const result = (0, glob_1.glob)(pattern, options || {});
if (result && typeof result.then === 'function') {
return result;
}
// Otherwise use promisify for glob@7.x callback style
const globPromise = (0, util_1.promisify)(glob_1.glob);
return globPromise(pattern, options || {});
};
const commands_1 = require("./commands");
const cloud_endpoint_1 = require("./utils/cloud-endpoint");
const server_api_1 = require("@signalk/server-api");
// AWS S3 for file upload
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let S3Client,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
PutObjectCommand,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
ListObjectsV2Command;
let _appInstance;
async function initializeCloudSDK(config, app) {
_appInstance = app;
if (config.cloudUpload.provider !== 'none') {
try {
if (!S3Client) {
const awsS3 = await Promise.resolve().then(() => __importStar(require('@aws-sdk/client-s3')));
S3Client = awsS3.S3Client;
PutObjectCommand = awsS3.PutObjectCommand;
ListObjectsV2Command = awsS3.ListObjectsV2Command;
}
}
catch (importError) {
S3Client = undefined;
}
}
}
// Legacy alias
exports.initializeS3 = initializeCloudSDK;
function createCloudClient(config, app) {
const cloud = config.cloudUpload;
if (cloud.provider === 'none' || !S3Client) {
return undefined;
}
try {
if (cloud.provider === 'r2') {
if (!cloud.accountId) {
app.error('R2 upload enabled but no account ID configured');
return undefined;
}
return new S3Client({
region: 'auto',
endpoint: `https://${cloud.accountId}.r2.cloudflarestorage.com`,
credentials: cloud.accessKeyId && cloud.secretAccessKey
? {
accessKeyId: cloud.accessKeyId,
secretAccessKey: cloud.secretAccessKey,
}
: undefined,
});
}
// S3 provider
const s3Config = {
region: cloud.region || 'us-east-1',
};
if (cloud.endpoint) {
const custom = (0, cloud_endpoint_1.resolveCustomS3Endpoint)(cloud.endpoint, cloud.forcePathStyle);
s3Config.endpoint = custom.url;
s3Config.forcePathStyle = custom.forcePathStyle;
}
else {
s3Config.forcePathStyle =
cloud.forcePathStyle !== undefined ? cloud.forcePathStyle : false;
}
if (cloud.accessKeyId && cloud.secretAccessKey) {
s3Config.credentials = {
accessKeyId: cloud.accessKeyId,
secretAccessKey: cloud.secretAccessKey,
};
}
return new S3Client(s3Config);
}
catch (error) {
app.error(`Failed to create ${cloud.provider.toUpperCase()} client: ${error.message}`);
return undefined;
}
}
// Legacy aliases
exports.createS3Client = createCloudClient;
exports.createR2Client = createCloudClient;
// Build cloud target from config and state
function getCloudTarget(config, state) {
const cloud = config.cloudUpload;
if (cloud.provider === 'none' || !state.cloudClient) {
return null;
}
return {
client: state.cloudClient,
bucket: cloud.bucket || '',
keyPrefix: cloud.keyPrefix || '',
deleteAfterUpload: cloud.deleteAfterUpload || false,
label: cloud.provider.toUpperCase(),
};
}
// Subscribe to command paths that control regimens using proper subscription manager
function subscribeToCommandPaths(currentPaths, state, config, app) {
const commandPaths = currentPaths.filter((pathConfig) => pathConfig &&
pathConfig.path &&
pathConfig.path.startsWith('commands.') &&
pathConfig.enabled);
if (commandPaths.length === 0)
return;
// Create Map for O(1) lookup instead of O(n) .find()
// This eliminates O(n²) nested loop on every delta message
const commandPathsMap = new Map(commandPaths.map(pathConfig => [pathConfig.path, pathConfig]));
const commandSubscription = {
context: 'vessels.self',
subscribe: commandPaths.map((pathConfig) => ({
path: pathConfig.path,
period: 1000, // Check commands every second
policy: 'fixed',
})),
};
app.subscriptionmanager.subscribe(commandSubscription, state.unsubscribes, (_subscriptionError) => { }, (delta) => {
// Process each update in the delta
const prevRegimens = new Set(state.activeRegimens);
if (delta.updates) {
delta.updates.forEach((update) => {
if ((0, server_api_1.hasValues)(update)) {
update.values.forEach((valueUpdate) => {
// O(1) Map lookup instead of O(n) array.find()
const pathConfig = commandPathsMap.get(valueUpdate.path);
if (pathConfig) {
handleCommandMessage(valueUpdate, pathConfig, config, update, state, app);
}
});
}
});
}
// If active regimens changed, re-evaluate data subscriptions
if (state.activeRegimens.size !== prevRegimens.size ||
[...state.activeRegimens].some(r => !prevRegimens.has(r))) {
const streamSubscriptions = state.streamSubscriptions;
const disposeStreamSubscription = (subscription) => {
if (typeof subscription === 'function') {
subscription();
return;
}
if (subscription && typeof subscription === 'object') {
const candidate = subscription;
if (typeof candidate.unsubscribe === 'function') {
candidate.unsubscribe();
}
else if (typeof candidate.dispose === 'function') {
candidate.dispose();
}
else if (typeof candidate.off === 'function') {
candidate.off();
}
}
};
if (Array.isArray(streamSubscriptions)) {
streamSubscriptions.forEach(disposeStreamSubscription);
streamSubscriptions.length = 0;
}
else if (streamSubscriptions instanceof Map) {
streamSubscriptions.forEach(disposeStreamSubscription);
streamSubscriptions.clear();
}
else if (streamSubscriptions instanceof Set) {
streamSubscriptions.forEach(disposeStreamSubscription);
streamSubscriptions.clear();
}
else if (streamSubscriptions &&
typeof streamSubscriptions === 'object') {
Object.values(streamSubscriptions).forEach(disposeStreamSubscription);
Object.keys(streamSubscriptions).forEach(key => {
delete streamSubscriptions[key];
});
}
updateDataSubscriptions(currentPaths, state, config, app);
}
});
commandPaths.forEach(pathConfig => {
state.subscribedPaths.add(pathConfig.path);
});
}
// Handle command messages (regimen control) - now receives complete delta structure
function handleCommandMessage(valueUpdate, pathConfig, config, update, state, app) {
try {
// Check source filter if specified for commands too
if (pathConfig.source && pathConfig.source.trim() !== '') {
const messageSource = update.$source || (update.source ? update.source.label : null);
if (messageSource !== pathConfig.source.trim()) {
return;
}
}
if (valueUpdate.value !== undefined) {
const commandName = (0, commands_1.extractCommandName)(pathConfig.path);
const isActive = Boolean(valueUpdate.value);
if (isActive) {
state.activeRegimens.add(commandName);
}
else {
state.activeRegimens.delete(commandName);
}
// Debug active regimens state
// Buffer this command change with complete metadata
const bufferKey = `${pathConfig.context || 'vessels.self'}:${pathConfig.path}`;
bufferData(bufferKey, {
received_timestamp: new Date().toISOString(),
signalk_timestamp: update.timestamp || new Date().toISOString(),
context: 'vessels.self',
path: valueUpdate.path,
value: valueUpdate.value,
source: update.source || undefined, // Store as object, serialize at write time
source_label: update.$source || (update.source ? update.source.label : undefined),
source_type: update.source ? update.source.type : undefined,
source_pgn: update.source ? update.source.pgn : undefined,
source_src: update.source ? update.source.src : undefined,
}, config, state, app);
}
}
catch (error) { }
}
// Helper function to handle wildcard contexts
function handleWildcardContext(pathConfig) {
const context = pathConfig.context || 'vessels.self';
if (context === 'vessels.*') {
// For vessels.*, we create a subscription that will receive deltas from any vessel
// The actual filtering by MMSI will happen in the delta handler
return {
...pathConfig,
context: 'vessels.*', // Keep the wildcard for the subscription
};
}
// Not a wildcard, return as-is
return pathConfig;
}
// Helper function to check if a vessel should be excluded based on MMSI
function _shouldExcludeVessel(vesselContext, pathConfig, app) {
if (!pathConfig.excludeMMSI || pathConfig.excludeMMSI.length === 0) {
return false; // No exclusions specified
}
try {
// For vessels.self, use getSelfPath
// Cast to any for compatibility with different @signalk/server-api versions
if (vesselContext === 'vessels.self') {
const mmsiData = app.getSelfPath('mmsi');
if (mmsiData && mmsiData.value) {
const mmsi = String(mmsiData.value);
return pathConfig.excludeMMSI.includes(mmsi);
}
}
else {
// For other vessels, we would need to get their MMSI from the delta or other means
// For now, we'll skip MMSI filtering for other vessels
}
}
catch (error) { }
return false; // Don't exclude if we can't determine MMSI
}
// Dispose a single stream subscription returned by streambundle .onValue()
function disposeStreamSubscription(subscription) {
if (typeof subscription === 'function') {
subscription();
return;
}
if (subscription && typeof subscription === 'object') {
const candidate = subscription;
if (typeof candidate.unsubscribe === 'function') {
candidate.unsubscribe();
}
else if (typeof candidate.dispose === 'function') {
candidate.dispose();
}
else if (typeof candidate.end === 'function') {
candidate.end();
}
}
}
// Update data path subscriptions based on active regimens
function updateDataSubscriptions(currentPaths, state, config, app) {
// First, unsubscribe from all existing subscriptions
state.unsubscribes.forEach(unsubscribe => {
if (typeof unsubscribe === 'function') {
unsubscribe();
}
});
state.unsubscribes = [];
// Dispose existing streambundle subscriptions to prevent duplicate handlers
if (state.streamSubscriptions) {
state.streamSubscriptions.forEach(disposeStreamSubscription);
state.streamSubscriptions = [];
}
state.subscribedPaths.clear();
// Re-subscribe to command paths
subscribeToCommandPaths(currentPaths, state, config, app);
// Now subscribe to data paths using currentPaths
const dataPaths = currentPaths.filter((pathConfig) => pathConfig && pathConfig.path && !pathConfig.path.startsWith('commands.'));
const shouldSubscribePaths = dataPaths.filter((pathConfig) => shouldSubscribeToPath(pathConfig, state, app));
// Handle wildcard contexts (like vessels.*)
const processedPaths = shouldSubscribePaths.map(pathConfig => handleWildcardContext(pathConfig));
if (processedPaths.length === 0) {
return;
}
// Group paths by context for separate subscriptions
const contextGroups = new Map();
processedPaths.forEach((pathConfig) => {
const context = (pathConfig.context || 'vessels.self');
if (!contextGroups.has(context)) {
contextGroups.set(context, []);
}
contextGroups.get(context).push(pathConfig);
});
// Context filter helper — reused for both normal and root-level paths
function passesContextFilter(normalizedDelta, pathConfig) {
// Filter by source if specified
if (pathConfig.source && pathConfig.source.trim() !== '') {
if (normalizedDelta.$source !== pathConfig.source.trim())
return false;
}
// Filter by context
const targetContext = pathConfig.context || 'vessels.self';
if (targetContext === 'vessels.*') {
if (!normalizedDelta.context.startsWith('vessels.'))
return false;
}
else if (targetContext === 'vessels.self') {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const selfContext = app.selfContext;
const selfVessel = app.getSelfPath('') || {};
const selfMMSI = selfVessel.mmsi;
const selfUuid = app.getSelfPath('uuid');
const isSelfVessel = normalizedDelta.context === 'vessels.self' ||
normalizedDelta.context === selfContext ||
(selfMMSI && normalizedDelta.context.includes(selfMMSI)) ||
(selfUuid && normalizedDelta.context.includes(selfUuid));
if (!isSelfVessel)
return false;
}
else {
if (normalizedDelta.context !== targetContext)
return false;
}
// MMSI exclusion filtering
if (pathConfig.excludeMMSI && pathConfig.excludeMMSI.length > 0) {
if (pathConfig.excludeMMSI.some(mmsi => normalizedDelta.context.includes(mmsi)))
return false;
}
// Skip meta deltas
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if (normalizedDelta.isMeta)
return false;
return true;
}
// Use app.streambundle approach as recommended by SignalK developer
// This avoids server arbitration and provides true source filtering
contextGroups.forEach((pathConfigs, _context) => {
// Root-level paths (no dots, e.g., "name", "mmsi") arrive on the root bus ("")
// as part of a bundled object update, not as individual path deltas
const rootPaths = pathConfigs.filter((pc) => !pc.path.includes('.'));
const normalPaths = pathConfigs.filter((pc) => pc.path.includes('.'));
// Subscribe to root bus for root-level paths
if (rootPaths.length > 0) {
const rootPathMap = new Map();
rootPaths.forEach((pc) => rootPathMap.set(pc.path, pc));
const rootStream = app.streambundle
.getBus('')
.filter((normalizedDelta) => {
if (!normalizedDelta.value ||
typeof normalizedDelta.value !== 'object')
return false;
const valueObj = normalizedDelta.value;
return rootPaths.some((pc) => valueObj[pc.path] !== undefined);
})
.debounceImmediate(5000) // Root properties are static, debounce aggressively
.onValue((normalizedDelta) => {
const valueObj = normalizedDelta.value;
for (const [pathName, pathConfig] of rootPathMap) {
if (valueObj[pathName] === undefined)
continue;
if (!passesContextFilter(normalizedDelta, pathConfig))
continue;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const syntheticDelta = {
...normalizedDelta,
path: pathName,
value: valueObj[pathName],
};
handleStreamData(syntheticDelta, pathConfig, config, state, app);
}
});
state.streamSubscriptions = state.streamSubscriptions || [];
state.streamSubscriptions.push(rootStream);
rootPaths.forEach((pc) => state.subscribedPaths.add(pc.path));
}
// Normal dotted paths — individual stream per path
normalPaths.forEach((pathConfig) => {
const stream = app.streambundle
.getBus(pathConfig.path)
.filter((normalizedDelta) => passesContextFilter(normalizedDelta, pathConfig))
.debounceImmediate(1000)
.onValue((normalizedDelta) => {
handleStreamData(normalizedDelta, pathConfig, config, state, app);
});
state.streamSubscriptions = state.streamSubscriptions || [];
state.streamSubscriptions.push(stream);
state.subscribedPaths.add(pathConfig.path);
});
});
}
// Determine if we should subscribe to a path based on regimens
function shouldSubscribeToPath(pathConfig, state, _app) {
// Always subscribe if explicitly enabled
if (pathConfig.enabled) {
return true;
}
// Check if any required regimens are active
if (pathConfig.regimen) {
const requiredRegimens = pathConfig.regimen.split(',').map(r => r.trim());
const hasActiveRegimen = requiredRegimens.some(regimen => state.activeRegimens.has(regimen));
return hasActiveRegimen;
}
return false;
}
// New handler for streambundle data (developer's recommended approach)
function handleStreamData(normalizedDelta, pathConfig, config, state, app) {
try {
// Retrieve metadata for this path
let metadata;
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const pathMetadata = app.getMetadata?.(normalizedDelta.path);
if (pathMetadata) {
metadata = pathMetadata; // Store as object, serialize at write time
}
}
catch (error) {
// Metadata retrieval failed, continue without it
}
const record = {
received_timestamp: new Date().toISOString(),
signalk_timestamp: normalizedDelta.timestamp || new Date().toISOString(),
context: normalizedDelta.context || pathConfig.context || 'vessels.self',
path: normalizedDelta.path,
value: null,
value_json: undefined,
source: normalizedDelta.source || undefined, // Store as object, serialize at write time
source_label: normalizedDelta.$source || undefined,
source_type: normalizedDelta.source
? normalizedDelta.source.type
: undefined,
source_pgn: normalizedDelta.source
? normalizedDelta.source.pgn
: undefined,
source_src: normalizedDelta.source
? normalizedDelta.source.src
: undefined,
meta: metadata,
};
// Handle complex values
if (typeof normalizedDelta.value === 'object' &&
normalizedDelta.value !== null) {
const valueObj = normalizedDelta.value;
const objKeys = Object.keys(valueObj);
// Skip if this looks like a meta-only update (only has units, meta, description keys)
// These are metadata updates, not actual data values
const metaOnlyKeys = [
'units',
'meta',
'description',
'displayUnits',
'zones',
'timeout',
];
const isMetaOnly = objKeys.length > 0 && objKeys.every(k => metaOnlyKeys.includes(k));
if (isMetaOnly) {
// This is a metadata update, not real data - skip it
return;
}
record.value_json = normalizedDelta.value; // Store as object, serialize at write time
// Extract key properties as columns for easier querying
Object.entries(normalizedDelta.value).forEach(([key, val]) => {
if (typeof val === 'string' ||
typeof val === 'number' ||
typeof val === 'boolean') {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
record[`value_${key}`] = val;
}
});
}
else {
record.value = normalizedDelta.value;
}
// Use actual context + path as buffer key to separate data from different vessels
const bufferKey = `${normalizedDelta.context}:${pathConfig.path}`;
bufferData(bufferKey, record, config, state, app);
}
catch (error) { }
}
// Buffer data and trigger save if buffer is full
function bufferData(signalkPath, record, config, state, app) {
// Use SQLite buffer if enabled
if (config.useSqliteBuffer) {
if (!state.sqliteBuffer) {
app.error(`[DataHandler] SQLite buffer is enabled but not initialized! Data for ${signalkPath} will be lost.`);
return;
}
if (!state.sqliteBuffer.isOpen()) {
app.error(`[DataHandler] SQLite buffer is closed! Data for ${signalkPath} will be lost.`);
return;
}
state.sqliteBuffer.insert(record);
return;
}
// LRU cache only used when SQLite is disabled
if (!state.dataBuffers.has(signalkPath)) {
state.dataBuffers.set(signalkPath, []);
}
const buffer = state.dataBuffers.get(signalkPath);
buffer.push(record);
if (buffer.length >= config.bufferSize) {
// Extract the actual SignalK path from the buffer key (context:path format)
// Find the separator between context and path - look for the last colon followed by a valid SignalK path
const pathMatch = signalkPath.match(/^.*:([a-zA-Z][a-zA-Z0-9._]*)$/);
const actualPath = pathMatch ? pathMatch[1] : signalkPath;
saveBufferToParquet(actualPath, buffer, config, state, app);
state.dataBuffers.set(signalkPath, []); // Clear buffer
}
}
// Save all buffers (called periodically and on shutdown)
function saveAllBuffers(config, state, app) {
state.dataBuffers.forEach((buffer, signalkPath) => {
if (buffer.length > 0) {
// Extract the actual SignalK path from the buffer key (context:path format)
// Find the separator between context and path - look for the last colon followed by a valid SignalK path
const pathMatch = signalkPath.match(/^.*:([a-zA-Z][a-zA-Z0-9._]*)$/);
const actualPath = pathMatch ? pathMatch[1] : signalkPath;
saveBufferToParquet(actualPath, buffer, config, state, app);
state.dataBuffers.delete(signalkPath); // Delete buffer to free memory
}
});
}
// Save buffer to Parquet file
async function saveBufferToParquet(signalkPath, buffer, config, state, app) {
try {
// Get context from first record in buffer (all records in buffer have same path/context)
const context = buffer.length > 0 ? buffer[0].context : 'vessels.self';
// Create proper directory structure
let contextPath;
if (context === 'vessels.self') {
// Clean the self context for filesystem usage (replace dots with slashes, colons with underscores)
contextPath = app.selfContext.replace(/\./g, '/').replace(/:/g, '_');
}
else if (context.startsWith('vessels.')) {
// Extract vessel identifier and clean it for filesystem
const vesselId = context.replace('vessels.', '').replace(/:/g, '_');
contextPath = `vessels/${vesselId}`;
}
else if (context.startsWith('meteo.')) {
// Extract meteo station identifier and clean it for filesystem
const meteoId = context.replace('meteo.', '').replace(/:/g, '_');
contextPath = `meteo/${meteoId}`;
}
else {
// Fallback: clean the entire context
contextPath = context.replace(/:/g, '_').replace(/\./g, '/');
}
const dirPath = path.join(config.outputDirectory, contextPath, signalkPath.replace(/\./g, '/'));
await fs.ensureDir(dirPath);
// Generate filename with timestamp
const timestamp = new Date()
.toISOString()
.replace(/[:.]/g, '')
.slice(0, 15);
const fileExt = config.fileFormat === 'csv'
? 'csv'
: config.fileFormat === 'parquet'
? 'parquet'
: 'json';
const filename = `${config.filenamePrefix}_${timestamp}.${fileExt}`;
const filepath = path.join(dirPath, filename);
// Use ParquetWriter to save in the configured format
await state.parquetWriter.writeRecords(filepath, buffer);
}
catch (error) { }
}
// Initialize regimen states from current API values at startup
function initializeRegimenStates(currentPaths, state, app) {
const commandPaths = currentPaths.filter((pathConfig) => pathConfig &&
pathConfig.path &&
pathConfig.path.startsWith('commands.') &&
pathConfig.enabled);
commandPaths.forEach((pathConfig) => {
try {
// Get current value from SignalK API
// Cast to any for compatibility with different @signalk/server-api versions
const currentData = app.getSelfPath(pathConfig.path);
if (currentData !== undefined && currentData !== null) {
// Check if there's source information
const shouldProcess = true;
// If source filter is specified, check it
if (pathConfig.source && pathConfig.source.trim() !== '') {
// For startup, we need to check the API source info
// This is a simplified check - in real deltas we get more source info
// For now, we'll process the value if it exists and log a warning
// In practice, you might want to check the source here too
}
if (shouldProcess && currentData.value !== undefined) {
const commandName = (0, commands_1.extractCommandName)(pathConfig.path);
const isActive = Boolean(currentData.value);
if (isActive) {
state.activeRegimens.add(commandName);
}
else {
state.activeRegimens.delete(commandName);
}
}
}
else {
/* no-op: no matching command for this regimen path */
}
}
catch (error) { }
});
}
// REMOVED: consolidateMissedDays() and consolidateYesterday()
// These functions have been replaced by the daily export system in parquet-export-service.ts
// The new exportDayToParquet() creates consolidated daily files directly
// List all existing keys in cloud bucket (paginated)
async function listCloudKeys(client, bucket, prefix) {
const keys = new Set();
if (!ListObjectsV2Command)
return keys;
let continuationToken;
do {
const response = await client.send(new ListObjectsV2Command({
Bucket: bucket,
Prefix: prefix || undefined,
ContinuationToken: continuationToken,
}));
if (response.Contents) {
for (const obj of response.Contents) {
if (obj.Key)
keys.add(obj.Key);
}
}
continuationToken = response.IsTruncated
? response.NextContinuationToken
: undefined;
} while (continuationToken);
return keys;
}
// Upload a single file to cloud (no HEAD check — caller handles dedup)
async function putToCloud(filePath, target, config) {
if (!target.client || !PutObjectCommand)
return false;
const relativePath = path.relative(config.outputDirectory, filePath);
let cloudKey = relativePath;
if (target.keyPrefix) {
const prefix = target.keyPrefix.endsWith('/')
? target.keyPrefix
: `${target.keyPrefix}/`;
cloudKey = `${prefix}${relativePath}`;
}
try {
const fileContent = await fs.readFile(filePath);
await target.client.send(new PutObjectCommand({
Bucket: target.bucket,
Key: cloudKey,
Body: fileContent,
ContentType: 'application/octet-stream',
}));
if (target.deleteAfterUpload) {
await fs.unlink(filePath);
}
return true;
}
catch (err) {
_appInstance?.error(`[CloudSync] Upload failed for ${cloudKey}: ${err.message}`);
return false;
}
}
// Get cloud key for a local file path
function getCloudKey(filePath, target, config) {
const relativePath = path.relative(config.outputDirectory, filePath);
if (target.keyPrefix) {
const prefix = target.keyPrefix.endsWith('/')
? target.keyPrefix
: `${target.keyPrefix}/`;
return `${prefix}${relativePath}`;
}
return relativePath;
}
// Upload missing hive files to cloud with batch concurrency
async function uploadMissingFiles(localFiles, existingKeys, target, config, app, concurrency = 3) {
const excludedDirs = [
'/processed/',
'/repaired/',
'/failed/',
'/quarantine/',
];
const filtered = localFiles.filter(f => !excludedDirs.some(dir => f.includes(dir)));
const missing = filtered.filter(f => !existingKeys.has(getCloudKey(f, target, config)));
if (missing.length === 0)
return 0;
app.debug(`[CloudSync] ${missing.length} files to upload (${concurrency} concurrent)`);
let uploaded = 0;
for (let i = 0; i < missing.length; i += concurrency) {
const batch = missing.slice(i, i + concurrency);
const results = await Promise.all(batch.map(f => putToCloud(f, target, config)));
uploaded += results.filter(Boolean).length;
// Yield to event loop between batches
await new Promise(resolve => setTimeout(resolve, 10));
}
return uploaded;
}
// Upload all hive-partitioned parquet files to cloud (7-day lookback)
async function uploadAllConsolidatedFilesToS3(config, state, app) {
const target = getCloudTarget(config, state);
if (!target)
return;
try {
const daysToCheck = 7;
const today = new Date();
// Gather local files for the lookback window first (fast, local I/O)
const allLocalFiles = [];
for (let daysAgo = 1; daysAgo <= daysToCheck; daysAgo++) {
const targetDate = new Date(today);
targetDate.setUTCDate(today.getUTCDate() - daysAgo);
const year = targetDate.getUTCFullYear();
const dayOfYear = String(Math.floor((targetDate.getTime() - Date.UTC(year, 0, 1)) / 86400000) + 1).padStart(3, '0');
const files = await glob(`tier=*/**/year=${year}/day=${dayOfYear}/*.parquet`, {
cwd: config.outputDirectory,
absolute: true,
nodir: true,
});
allLocalFiles.push(...files);
}
app.debug(`[StartupSync] Found ${allLocalFiles.length} local files in last ${daysToCheck} days`);
if (allLocalFiles.length === 0)
return;
// List only raw-tier prefixes in R2 to find which context/path/year/day combos are synced
const prefixSet = new Set();
const basePrefix = target.keyPrefix
? target.keyPrefix.endsWith('/')
? target.keyPrefix
: `${target.keyPrefix}/`
: '';
for (const file of allLocalFiles) {
const rel = path.relative(config.outputDirectory, file);
if (!rel.startsWith('tier=raw'))
continue;
const dirPart = path.dirname(rel);
prefixSet.add(`${basePrefix}${dirPart}/`);
}
app.debug(`[StartupSync] Listing ${target.label} objects for ${prefixSet.size} raw-tier prefixes...`);
// Build set of synced directories (context/path/year/day) from raw tier
// e.g. "context=X/path=Y/year=2026/day=073"
const syncedDirs = new Set();
for (const prefix of prefixSet) {
const keys = await listCloudKeys(target.client, target.bucket, prefix);
if (keys.size > 0) {
// Extract context/path/year/day from the prefix (strip basePrefix and tier=raw/)
const withoutBase = prefix.startsWith(basePrefix)
? prefix.slice(basePrefix.length)
: prefix;
// withoutBase = "tier=raw/context=X/path=Y/year=YYYY/day=DDD/"
const withoutTier = withoutBase.replace(/^tier=[^/]+\//, '');
syncedDirs.add(withoutTier);
}
}
app.debug(`[StartupSync] Found ${syncedDirs.size} synced directories in ${target.label}`);
// Filter local files: skip any file whose context/path/year/day is already synced
const excludedDirs = [
'/processed/',
'/repaired/',
'/failed/',
'/quarantine/',
];
const filesToUpload = allLocalFiles.filter(f => {
if (excludedDirs.some(dir => f.includes(dir)))
return false;
const rel = path.relative(config.outputDirectory, f);
// Strip tier segment to get context/path/year/day/
const withoutTier = rel.replace(/^tier=[^/]+\//, '');
const dirPart = path.dirname(withoutTier) + '/';
return !syncedDirs.has(dirPart);
});
if (filesToUpload.length === 0) {
app.debug(`[StartupSync] All files already synced`);
return;
}
app.debug(`[CloudSync] ${filesToUpload.length} files to upload (3 concurrent)`);
let uploaded = 0;
for (let i = 0; i < filesToUpload.length; i += 3) {
const batch = filesToUpload.slice(i, i + 3);
const results = await Promise.all(batch.map(f => putToCloud(f, target, config)));
uploaded += results.filter(Boolean).length;
await new Promise(resolve => setTimeout(resolve, 10));
}
if (uploaded > 0) {
app.debug(`[StartupSync] Uploaded ${uploaded} files to ${target.label}`);
}
}
catch (error) {
app.error(`[StartupSync] Failed: ${error.message}`);
}
}
// Upload hive-partitioned parquet files for a specific date to cloud
async function uploadConsolidatedFilesToS3(config, date, state, app) {
const target = getCloudTarget(config, state);
if (!target)
return;
try {
const year = date.getUTCFullYear();
const dayOfYear = String(Math.floor((date.getTime() - Date.UTC(year, 0, 1)) / 86400000) + 1).padStart(3, '0');
const dateStr = date.toISOString().slice(0, 10);
const localFiles = await glob(`tier=*/**/year=${year}/day=${dayOfYear}/*.parquet`, {
cwd: config.outputDirectory,
absolute: true,
nodir: true,
});
if (localFiles.length === 0)
return;
// List existing keys and only upload missing
const existingKeys = await listCloudKeys(target.client, target.bucket, target.keyPrefix || undefined);
const uploaded = await uploadMissingFiles(localFiles, existingKeys, target, config, app);
if (uploaded > 0) {
app.debug(`${target.label}: Uploaded ${uploaded} hive files for ${dateStr}`);
}
}
catch (error) {
app.error(`Cloud upload failed for date ${date.toISOString().slice(0, 10)}: ${error.message}`);
}
}
//# sourceMappingURL=data-handler.js.map