@cumulus/ingest/consumer.js

Ingest utilities

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Consumer = void 0;
const SQS_1 = require("@cumulus/aws-client/SQS");
const sqs = __importStar(require("@cumulus/aws-client/SQS"));
const StepFunctions_1 = require("@cumulus/aws-client/StepFunctions");
const logger_1 = __importDefault(require("@cumulus/logger"));
const log = new logger_1.default({ sender: '@cumulus/ingest/consumer' });
class Consumer {
    constructor({ queueUrl, messageLimit = 1, timeLimit = 90, visibilityTimeout, deleteProcessedMessage = true, }) {
        this.queueUrl = queueUrl;
        this.messageLimit = messageLimit;
        this.visibilityTimeout = visibilityTimeout;
        this.timeLimit = timeLimit * 1000;
        this.now = Date.now();
        this.timeLapsed = false;
        this.deleteProcessedMessage = deleteProcessedMessage;
    }
    async processMessage(message, fn) {
        try {
            await fn(this.queueUrl, message);
            if (this.deleteProcessedMessage) {
                await sqs.deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
            }
            return 1;
        }
        catch (error) {
            if (error instanceof StepFunctions_1.ExecutionAlreadyExists) {
                log.debug('Deleting message for execution that already exists...');
                await sqs.deleteSQSMessage(this.queueUrl, message.ReceiptHandle);
                log.debug('Completed deleting message.');
                return 1;
            }
            log.error(error);
            return 0;
        }
    }
    async processMessages(fn, messageLimit, visibilityTimeout) {
        if (messageLimit > 10)
            throw new Error(`Cannot process more than 10 messages per function call. Received limit: ${messageLimit}`);
        let counter = 0;
        const messages = await (0, SQS_1.receiveSQSMessages)(this.queueUrl, { numOfMessages: messageLimit, visibilityTimeout });
        if (messages.length > 0) {
            log.info(`processing ${messages.length} messages`);
            const processes = messages.map((message) => this.processMessage(message, fn));
            const results = await Promise.all(processes);
            counter = results.reduce((total, value) => total + value, 0);
        }
        return counter;
    }
    async consume(fn) {
        let messageLimit = this.messageLimit;
        log.info(`Attempting to process up to ${messageLimit} messages...`);
        let sum = 0;
        /* eslint-disable no-await-in-loop */
        // Only request up to the original messageLimit messages on subsequent `processMessages` calls
        while (messageLimit > 0 && !this.timeLapsed) {
            let results = 0;
            if (messageLimit > 10) {
                results = await this.processMessages(fn, 10, this.visibilityTimeout);
                messageLimit -= 10;
            }
            else if (messageLimit > 0) {
                results = await this.processMessages(fn, messageLimit, this.visibilityTimeout);
                messageLimit -= messageLimit;
            }
            sum += results;
            // if the function is running for longer than the timeLimit, stop it
            const timeSpent = (Date.now() - this.now);
            if (timeSpent > this.timeLimit) {
                this.timeLapsed = true;
                log.warn(`${this.timeLimit / 1000}-second time limit reached, exiting...`);
            }
        }
        /* eslint-enable no-await-in-loop */
        log.info(`${sum} messages successfully processed from ${this.queueUrl}`);
        return sum;
    }
}
exports.Consumer = Consumer;
//# sourceMappingURL=consumer.js.map