@cumulus/ingest
Version:
Ingest utilities
148 lines • 7.18 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ConsumerRateLimited = void 0;
const SQS_1 = require("@cumulus/aws-client/SQS");
const sqs = __importStar(require("@cumulus/aws-client/SQS"));
const StepFunctions_1 = require("@cumulus/aws-client/StepFunctions");
const logger_1 = __importDefault(require("@cumulus/logger"));
const common_1 = require("@cumulus/common");
const log = new logger_1.default({ sender: '@cumulus/ingest/consumer' });
class ConsumerRateLimited {
constructor({ queueUrls, timeRemainingFunc, visibilityTimeout, rateLimitPerSecond, deleteProcessedMessage = true, }) {
this.queueUrls = queueUrls;
this.visibilityTimeout = visibilityTimeout;
this.timeRemainingFunc = timeRemainingFunc;
this.deleteProcessedMessage = deleteProcessedMessage;
this.rateLimitPerSecond = rateLimitPerSecond;
// The maximum number of messages to fetch in one request per queue
this.messageLimitPerFetch = 10;
// The amount of time to wait before retrying to fetch messages when none are found
this.waitTime = 5000;
}
async processMessage(message, fn, queueUrl) {
try {
await fn(queueUrl, message);
}
catch (error) {
if (error instanceof StepFunctions_1.ExecutionAlreadyExists) {
log.debug('Deleting message for execution that already exists...');
await sqs.deleteSQSMessage(queueUrl, message.ReceiptHandle);
log.debug('Completed deleting message.');
return true;
}
log.error(error);
return false;
}
if (this.deleteProcessedMessage) {
await sqs.deleteSQSMessage(queueUrl, message.ReceiptHandle);
}
return true;
}
async processMessages(fn, messagesWithQueueUrls) {
let counter = 0;
for (const [message, queueUrl] of messagesWithQueueUrls) {
const waitTime = 1000 / this.rateLimitPerSecond;
log.debug(`Waiting for ${waitTime} ms`);
// We normally don't want to await in a loop due to decreased performance
// from running sequentially, but here we want to enforce rate limiting
// by specifically adding a delay to each loop iteration.
// eslint-disable-next-line no-await-in-loop
await (0, common_1.sleep)(waitTime);
// eslint-disable-next-line no-await-in-loop
if (await this.processMessage(message, fn, queueUrl)) {
counter += 1;
}
}
return counter;
}
async fetchMessages(queueUrl, messageLimit) {
const messages = await (0, SQS_1.receiveSQSMessages)(queueUrl, {
numOfMessages: messageLimit,
visibilityTimeout: this.visibilityTimeout,
});
return messages.map((message) => [message, queueUrl]);
}
async fetchMessagesFromAllQueues() {
return Promise.all(this.queueUrls.map((queueUrl) => this.fetchMessages(queueUrl, this.messageLimitPerFetch))).then((messageArrays) => messageArrays.flat());
}
async consume(fn) {
let messageCounter = 0;
let processingPromise;
let fetchPromise;
let messages;
let processTimeMilliseconds = 0;
let startTime;
// The below block of code attempts to always have a batch of messages
// available for `processMessages` to process, so, after the initial fetch,
// we'll immediately start fetching the next batch while processing the
// current one
// There are several await-in-loop instances below all required for flow control to assure
// we're submitting at a specified rate.
while (this.timeRemainingFunc(processTimeMilliseconds) > 0) {
if (messages === undefined) {
// This will be run in the first iteration, included in the loop in case of a small
// timeRemainingFunc value
// eslint-disable-next-line no-await-in-loop
messages = await this.fetchMessagesFromAllQueues();
}
if (messages.length === 0) {
log.info(`No messages fetched, waiting ${this.waitTime} ms before retrying`);
// eslint-disable-next-line no-await-in-loop
await (0, common_1.sleep)(this.waitTime);
// eslint-disable-next-line no-await-in-loop
messages = await this.fetchMessagesFromAllQueues();
}
else {
// Start processing current batch and immediately fetch next batch
processingPromise = this.processMessages(fn, messages);
fetchPromise = this.fetchMessagesFromAllQueues();
startTime = Date.now();
// Wait for processing to complete and increment counter
// eslint-disable-next-line no-await-in-loop
messageCounter += await processingPromise;
if (processTimeMilliseconds === 0) {
// First processing time measurement, add 50% buffer to account for possible longer
// processing time on the last iteration
processTimeMilliseconds = (Date.now() - startTime) + (Date.now() - startTime) * 0.5;
}
// Get the next batch that was fetched concurrently
// eslint-disable-next-line no-await-in-loop
messages = await fetchPromise;
}
}
// Process any remaining messages after time has expired
if (messages !== undefined && messages.length > 0) {
messageCounter += await this.processMessages(fn, messages);
}
log.info(`${messageCounter} messages successfully processed from ${this.queueUrls}`);
return messageCounter;
}
}
exports.ConsumerRateLimited = ConsumerRateLimited;
//# sourceMappingURL=consumerRateLimited.js.map