@cumulus/message
Version:
Utilities for building and parsing Cumulus messages
218 lines • 8.98 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.generateGranuleApiRecord = exports.getGranuleCmrTemporalInfo = exports.getGranuleProcessingTimeInfo = exports.getGranuleTimeToArchive = exports.getGranuleTimeToPreprocess = exports.getGranuleProductVolume = exports.getGranuleQueryFields = exports.getGranuleStatus = exports.messageHasGranules = exports.getMessageGranules = void 0;
/**
* Utility functions for parsing granule information from a Cumulus message
* @module Granules
*
* @example
* const Granules = require('@cumulus/message/Granules');
*/
const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
const isNil_1 = __importDefault(require("lodash/isNil"));
const isInteger_1 = __importDefault(require("lodash/isInteger"));
const isUndefined_1 = __importDefault(require("lodash/isUndefined"));
const mapValues_1 = __importDefault(require("lodash/mapValues"));
const omitBy_1 = __importDefault(require("lodash/omitBy"));
const pick_1 = __importDefault(require("lodash/pick"));
const errors_1 = require("@cumulus/errors");
/**
* Get granules from payload?.granules of a workflow message.
*
* @param {Message.CumulusMessage} message - A workflow message
* @returns {Array<object>|undefined} An array of granule objects, or
* undefined if `message.payload.granules` is not set
* @alias module:Granules
*/
const getMessageGranules = (message) => {
const granules = message.payload?.granules;
return Array.isArray(granules) ? granules : [];
};
exports.getMessageGranules = getMessageGranules;
/**
* Determine if message has a granules object.
*
* @param {Message.CumulusMessage} message - A workflow message object
* @returns {boolean} true if message has a granules object
* @alias module:Granules
*/
function messageHasGranules(message) {
return (0, exports.getMessageGranules)(message).length > 0;
}
exports.messageHasGranules = messageHasGranules;
/**
* Determine the status of a granule.
*
* @param {string} workflowStatus - The workflow status
* @param {MessageGranule} granule - A granule record conforming to the 'api' schema
* @returns {string} The granule status
*
* @alias module:Granules
*/
const getGranuleStatus = (workflowStatus, granule) => workflowStatus || granule.status;
exports.getGranuleStatus = getGranuleStatus;
/**
* Get the query fields of a granule, if any
*
* @param {MessageWithGranules} message - A workflow message
* @returns {unknown|undefined} The granule query fields, if any
*
* @alias module:Granules
*/
const getGranuleQueryFields = (message) => message.meta?.granule?.queryFields;
exports.getGranuleQueryFields = getGranuleQueryFields;
/**
* Calculate granule product volume, which is the sum of the file
* sizes in bytes
*
* @param {Array<Object>} granuleFiles - array of granule file objects that conform to the
* Cumulus 'api' schema
* @returns {string} - sum of granule file sizes in bytes as a string
*/
const getGranuleProductVolume = (granuleFiles = []) => {
if (granuleFiles.length === 0)
return '0';
return String(granuleFiles
.map((f) => f.size ?? 0)
.filter(isInteger_1.default)
.reduce((x, y) => x + BigInt(y), BigInt(0)));
};
exports.getGranuleProductVolume = getGranuleProductVolume;
const getGranuleTimeToPreprocess = ({ sync_granule_duration = 0, } = {}) => sync_granule_duration / 1000;
exports.getGranuleTimeToPreprocess = getGranuleTimeToPreprocess;
const getGranuleTimeToArchive = ({ post_to_cmr_duration = 0, } = {}) => post_to_cmr_duration / 1000;
exports.getGranuleTimeToArchive = getGranuleTimeToArchive;
/**
* Convert date string to standard ISO format.
*
* @param {string} date - Date string, possibly in multiple formats
* @returns {string} Standardized ISO date string
*/
const convertDateToISOString = (date) => new Date(date).toISOString();
function isProcessingTimeInfo(info = {}) {
return info?.processingStartDateTime !== undefined
&& info?.processingEndDateTime !== undefined;
}
/**
* ** Private **
* Convert date string to standard ISO format, retaining null/undefined values if they exist
* and converting '' to null
*
* @param {string} date - Date string, possibly in multiple formats
* @returns {string} Standardized ISO date string
*/
const convertDateToISOStringSettingNull = (date) => {
if ((0, isNil_1.default)(date))
return date;
if (date === '')
return null;
return convertDateToISOString(date);
};
/**
* Convert granule processing timestamps to a standardized ISO string
* format for compatibility across database systems.
*
* @param {ExecutionProcessingTimes} [processingTimeInfo]
* Granule processing time info, if any
* @returns {Promise<ExecutionProcessingTimes | undefined>}
*/
const getGranuleProcessingTimeInfo = (processingTimeInfo) => {
const updatedProcessingTimeInfo = isProcessingTimeInfo(processingTimeInfo)
? { ...processingTimeInfo }
: {};
return (0, mapValues_1.default)(updatedProcessingTimeInfo, convertDateToISOStringSettingNull);
};
exports.getGranuleProcessingTimeInfo = getGranuleProcessingTimeInfo;
function isGranuleTemporalInfo(info = {}) {
return info?.beginningDateTime !== undefined
&& info?.endingDateTime !== undefined
&& info?.productionDateTime !== undefined
&& info?.lastUpdateDateTime !== undefined;
}
/**
* Get granule temporal information from argument, directly from CMR
* file or from granule object.
*
* Converts temporal information timestamps to a standardized ISO string
* format for compatibility across database systems.
*
* @param {Object} params
* @param {MessageGranule} params.granule - Granule from workflow message
* @param {Object} [params.cmrTemporalInfo] - CMR temporal info, if any
* @param {CmrUtilsClass} params.cmrUtils - CMR utilities object
* @returns {Promise<GranuleTemporalInfo | undefined>}
*/
const getGranuleCmrTemporalInfo = async ({ granule, cmrTemporalInfo, cmrUtils, }) => {
// Get CMR temporalInfo (beginningDateTime, endingDateTime,
// productionDateTime, lastUpdateDateTime)
const temporalInfo = isGranuleTemporalInfo(cmrTemporalInfo)
? { ...cmrTemporalInfo }
: await cmrUtils.getGranuleTemporalInfo(granule);
if ((0, isEmpty_1.default)(temporalInfo)) {
return (0, pick_1.default)(granule, ['beginningDateTime', 'endingDateTime', 'productionDateTime', 'lastUpdateDateTime']);
}
return (0, mapValues_1.default)(temporalInfo, convertDateToISOStringSettingNull);
};
exports.getGranuleCmrTemporalInfo = getGranuleCmrTemporalInfo;
/**
* Generate an API granule record
*
* @param {MessageWithGranules} message - A workflow message
* @returns {Promise<ApiGranule>} The granule API record
*
* @alias module:Granules
*/
const generateGranuleApiRecord = async ({ granule, executionUrl, collectionId, provider, error, pdrName, status, queryFields, updatedAt, files, processingTimeInfo, cmrUtils, timestamp, duration, productVolume, timeToPreprocess, timeToArchive, cmrTemporalInfo, }) => {
if (!granule.granuleId)
throw new errors_1.CumulusMessageError(`Could not create granule record, invalid granuleId: ${granule.granuleId}`);
if (!collectionId) {
throw new errors_1.CumulusMessageError('collectionId required to generate a granule record');
}
// null should not be supported in generated API records
if (files === null) {
throw new errors_1.CumulusMessageError('granule.files must not be null');
}
if (error === null) {
throw new errors_1.CumulusMessageError('granule.error must not be null');
}
const { granuleId, cmrLink, producerGranuleId, published, createdAt, } = granule;
const now = Date.now();
const recordUpdatedAt = updatedAt ?? now;
const recordTimestamp = timestamp ?? now;
// Get CMR temporalInfo
const temporalInfo = await (0, exports.getGranuleCmrTemporalInfo)({
granule: { ...granule, status },
cmrTemporalInfo,
cmrUtils,
});
const updatedProcessingTimeInfo = (0, exports.getGranuleProcessingTimeInfo)(processingTimeInfo);
const record = {
granuleId,
pdrName,
collectionId,
status,
provider,
execution: executionUrl,
cmrLink,
files,
error,
published,
createdAt,
timestamp: recordTimestamp,
updatedAt: recordUpdatedAt,
duration,
producerGranuleId,
productVolume,
timeToPreprocess,
timeToArchive,
...updatedProcessingTimeInfo,
...temporalInfo,
queryFields,
};
return (0, omitBy_1.default)(record, isUndefined_1.default);
};
exports.generateGranuleApiRecord = generateGranuleApiRecord;
//# sourceMappingURL=Granules.js.map