@cumulus/ingest/src/queue.js

Ingest utilities

'use strict';

const { getJsonS3Object } = require('@cumulus/aws-client/S3');
const { sendSQSMessage } = require('@cumulus/aws-client/SQS');

const { buildQueueMessageFromTemplate } = require('@cumulus/message/Build');
const { buildExecutionArn } = require('@cumulus/message/Executions');

const {
  getWorkflowFileKey,
  templateKey,
} = require('@cumulus/common/workflows');
const isNil = require('lodash/isNil');

const CmrProviderNotConfiguredMessage = 'all collections must configure a cmr_provider for sf to be scheduled';
class CMRProviderNotConfiguredError extends Error {
  /**
   *
   * @param {string} message
   */
  constructor(message) {
    super(`${message} ${CmrProviderNotConfiguredMessage}`);

    this.name = 'CMRProviderNotConfiguredError';

    Error.captureStackTrace(this, CMRProviderNotConfiguredError);
  }
}
/**
 * @typedef { import('@cumulus/types/api/collections').CollectionRecord } CollectionRecord
 * @param {object | undefined} messageTemplate
 * @param {CollectionRecord} collection
 */
function joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection) {
  const { cmrProvider } = collection;
  if (isNil(cmrProvider)) {
    throw new CMRProviderNotConfiguredError(
      `no cmr_provider found for collection ${collection.name}___${collection.version}`
    );
  }
  const templateCmr = messageTemplate?.meta?.cmr || {};
  return {
    ...templateCmr,
    provider: cmrProvider,
  };
}
exports.joinCollectionProviderToTemplateCmrMeta = joinCollectionProviderToTemplateCmrMeta;

/**
 * Enqueue a PDR to be parsed
 *
 * @param {object} params
 * @param {object} params.collection         - the collection config to be attached to the message
 * @param {string} params.parentExecutionArn - parent workflow execution arn to add to the message
 * @param {string} params.parsePdrWorkflow   - the S3 URI of template for a PDR parse message
 * @param {object} params.pdr                - the PDR to be enqueued for parsing
 * @param {object} params.provider           - the provider config to be attached to the message
 * @param {string} params.stack              - stack name
 * @param {string} params.systemBucket       - system bucket name
 * @param {string} params.queueUrl           - the SQS queue to add the message to
 * @param {string} [params.executionNamePrefix]
 *   - the prefix to apply to the name of the enqueued execution
 * @param {object} [params.additionalCustomMeta]
 *   - additional object to merge into meta object
 * @returns {Promise<string>}
 *   - resolves when the message has been enqueued
 */
async function enqueueParsePdrMessage({
  collection,
  parentExecutionArn,
  parsePdrWorkflow,
  pdr,
  provider,
  stack,
  systemBucket,
  queueUrl,
  executionNamePrefix,
  additionalCustomMeta = {},
}) {
  const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
  let cmrMeta = {};
  if (collection) {
    cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
  }
  const { arn: parsePdrArn } = await getJsonS3Object(
    systemBucket,
    getWorkflowFileKey(stack, parsePdrWorkflow)
  );
  const payload = { pdr };
  const workflow = {
    name: parsePdrWorkflow,
    arn: parsePdrArn,
  };

  const message = buildQueueMessageFromTemplate({
    messageTemplate,
    parentExecutionArn,
    payload,
    workflow,
    customMeta: {
      ...additionalCustomMeta,
      collection,
      provider,
      cmr: cmrMeta,
    },
    executionNamePrefix,
  });

  const arn = buildExecutionArn(
    message.cumulus_meta.state_machine,
    message.cumulus_meta.execution_name
  );

  await sendSQSMessage(queueUrl, message);

  return arn;
}
module.exports.enqueueParsePdrMessage = enqueueParsePdrMessage;

/**
 * Enqueue a granule to be ingested
 *
 * @param {object} params
 * @param {object} params.collection      - the collection config to be attached to the message
 * @param {object} params.granules        - the granules to be enqueued for ingest
 * @param {string} params.parentExecutionArn
 *   - parent workflow execution arn to add to the message
 * @param {object} params.provider        - the provider config to be attached to the message
 * @param {object} params.messageTemplate - Message template for the workflow
 * @param {object} params.workflow        - workflow name & arn object
 * @param {string} params.queueUrl        - the SQS queue to add the message to
 * @param {object | undefined} [params.pdr]
 *   - an optional PDR to be configured in the message payload
 * @param {string} [params.executionNamePrefix]
 *   - the prefix to apply to the name of the enqueued execution
 * @param {object} [params.additionalCustomMeta]
 *   - additional object to merge into meta object
 * @returns {Promise<string>}
 *   - resolves when the message has been enqueued
 */
async function enqueueGranuleIngestMessage({
  collection,
  granules,
  parentExecutionArn,
  pdr,
  provider,
  messageTemplate,
  workflow,
  queueUrl,
  executionNamePrefix,
  additionalCustomMeta = {},
}) {
  let cmrMeta = {};
  if (collection) {
    cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
  }
  const message = buildQueueMessageFromTemplate({
    messageTemplate,
    parentExecutionArn,
    payload: { granules },
    workflow,
    customMeta: {
      ...additionalCustomMeta,
      ...(pdr ? { pdr } : {}),
      collection,
      provider,
      cmr: cmrMeta,
    },
    executionNamePrefix,
  });

  await sendSQSMessage(queueUrl, message);
  return buildExecutionArn(
    message.cumulus_meta.state_machine,
    message.cumulus_meta.execution_name
  );
}
exports.enqueueGranuleIngestMessage = enqueueGranuleIngestMessage;

/**
 * Enqueue a workflow
 *
 * @param {object} params
 * @param {string} params.parentExecutionArn
 *   - parent workflow execution arn to add to the message
 * @param {string} params.stack         - stack name
 * @param {string} params.systemBucket  - system bucket name
 * @param {object} params.collection    - the collection config to be attached to the message
 * @param {object} params.provider      - the provider config to be attached to the message
 * @param {object} params.workflow      - the workflow to be enqueued
 * @param {object} params.workflowInput - the input that should be passed to the queued workflow
 * @param {string} [params.queueUrl]    - an optional SQS queue to add the message to
 * @param {string} [params.executionNamePrefix]
 *   - the prefix to apply to the name of the enqueued execution
 * @returns {Promise<string>}
 *   - resolves when the message has been enqueued
 */
async function enqueueWorkflowMessage({
  parentExecutionArn,
  stack,
  systemBucket,
  collection,
  provider,
  queueUrl,
  workflow,
  workflowInput,
  executionNamePrefix,
  additionalCustomMeta = {},
}) {
  const messageTemplate = await getJsonS3Object(systemBucket, templateKey(stack));
  let cmrMeta = {};
  if (collection) {
    cmrMeta = joinCollectionProviderToTemplateCmrMeta(messageTemplate, collection);
  }
  const { arn: queuedWorkflowArn } = await getJsonS3Object(
    systemBucket,
    getWorkflowFileKey(stack, workflow)
  );

  const payload = {
    ...workflowInput,
  };

  const queuedWorkflowDefinition = {
    name: workflow,
    arn: queuedWorkflowArn,
  };

  const message = buildQueueMessageFromTemplate({
    messageTemplate,
    parentExecutionArn,
    payload,
    queueUrl,
    workflow: queuedWorkflowDefinition,
    executionNamePrefix,
    customMeta: {
      ...additionalCustomMeta,
      collection,
      provider,
      cmr: cmrMeta,
    },
  });

  const arn = buildExecutionArn(
    message.cumulus_meta.state_machine,
    message.cumulus_meta.execution_name
  );

  await sendSQSMessage(queueUrl, message);

  return arn;
}
exports.enqueueWorkflowMessage = enqueueWorkflowMessage;