sqs-consumer
Version:
Build SQS-based Node applications without the boilerplate
516 lines (515 loc) • 20.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Consumer = void 0;
const client_sqs_1 = require("@aws-sdk/client-sqs");
const emitter_js_1 = require("./emitter.js");
const errors_js_1 = require("./errors.js");
const validation_js_1 = require("./validation.js");
const logger_js_1 = require("./logger.js");
/**
* [Usage](https://bbc.github.io/sqs-consumer/index.html#usage)
*/
class Consumer extends emitter_js_1.TypedEventEmitter {
constructor(options) {
super(options.queueUrl);
this.pollingTimeoutId = undefined;
this.stopped = true;
this.isPolling = false;
(0, validation_js_1.assertOptions)(options);
this.queueUrl = options.queueUrl;
this.isFifoQueue = this.queueUrl.endsWith(".fifo");
this.suppressFifoWarning = options.suppressFifoWarning ?? false;
this.handleMessage = options.handleMessage;
this.handleMessageBatch = options.handleMessageBatch;
this.preReceiveMessageCallback = options.preReceiveMessageCallback;
this.postReceiveMessageCallback = options.postReceiveMessageCallback;
this.handleMessageTimeout = options.handleMessageTimeout;
this.attributeNames = options.attributeNames || [];
this.messageAttributeNames = options.messageAttributeNames || [];
this.messageSystemAttributeNames =
options.messageSystemAttributeNames || [];
this.batchSize = options.batchSize || 1;
this.visibilityTimeout = options.visibilityTimeout;
this.terminateVisibilityTimeout =
options.terminateVisibilityTimeout || false;
this.heartbeatInterval = options.heartbeatInterval;
this.waitTimeSeconds = options.waitTimeSeconds ?? 20;
this.authenticationErrorTimeout =
options.authenticationErrorTimeout ?? 10000;
this.pollingWaitTimeMs = options.pollingWaitTimeMs ?? 0;
this.pollingCompleteWaitTimeMs = options.pollingCompleteWaitTimeMs ?? 0;
this.shouldDeleteMessages = options.shouldDeleteMessages ?? true;
this.alwaysAcknowledge = options.alwaysAcknowledge ?? false;
this.extendedAWSErrors = options.extendedAWSErrors ?? false;
this.strictReturn = options.strictReturn ?? false;
this.sqs =
options.sqs ||
new client_sqs_1.SQSClient({
useQueueUrlAsEndpoint: options.useQueueUrlAsEndpoint ?? true,
region: options.region || process.env.AWS_REGION || "eu-west-1",
});
}
/**
* Creates a new SQS consumer.
*/
static create(options) {
return new Consumer(options);
}
/**
* Start polling the queue for messages.
*/
start() {
if (this.stopped) {
if (this.isFifoQueue && !this.suppressFifoWarning) {
logger_js_1.logger.warn("WARNING: A FIFO queue was detected. SQS Consumer does not guarantee FIFO queues will work as expected. Set 'suppressFifoWarning: true' to disable this warning.");
}
// Create a new abort controller each time the consumer is started
this.abortController = new AbortController();
logger_js_1.logger.debug("starting");
this.stopped = false;
this.emit("started");
this.poll();
}
}
/**
* A reusable options object for sqs.send that's used to avoid duplication.
*/
get sqsSendOptions() {
return {
// return the current abortController signal or a fresh signal that has not been aborted.
// This effectively defaults the signal sent to the AWS SDK to not aborted
abortSignal: this.abortController?.signal || new AbortController().signal,
};
}
/**
* Stop polling the queue for messages (pre existing requests will still be made until concluded).
*/
stop(options) {
if (this.stopped) {
logger_js_1.logger.debug("already_stopped");
return;
}
logger_js_1.logger.debug("stopping");
this.stopped = true;
if (this.pollingTimeoutId) {
clearTimeout(this.pollingTimeoutId);
this.pollingTimeoutId = undefined;
}
if (options?.abort) {
logger_js_1.logger.debug("aborting");
this.abortController.abort();
this.emit("aborted");
}
this.stopRequestedAtTimestamp = Date.now();
this.waitForPollingToComplete();
}
/**
* Wait for final poll and in flight messages to complete.
* @private
*/
waitForPollingToComplete() {
if (!this.isPolling || !(this.pollingCompleteWaitTimeMs > 0)) {
this.emit("stopped");
return;
}
const exceededTimeout = Date.now() - this.stopRequestedAtTimestamp >
this.pollingCompleteWaitTimeMs;
if (exceededTimeout) {
this.emit("waiting_for_polling_to_complete_timeout_exceeded");
this.emit("stopped");
return;
}
this.emit("waiting_for_polling_to_complete");
setTimeout(() => this.waitForPollingToComplete(), 1000);
}
/**
* Returns the current status of the consumer.
* This includes whether it is running or currently polling.
*/
get status() {
return {
isRunning: !this.stopped,
isPolling: this.isPolling,
};
}
/**
* Validates and then updates the provided option to the provided value.
* @param option The option to validate and then update
* @param value The value to set the provided option to
*/
updateOption(option, value) {
(0, validation_js_1.validateOption)(option, value, this, true);
this[option] = value;
this.emit("option_updated", option, value);
}
/**
* Emit one of the consumer's error events depending on the error received.
* @param err The error object to forward on
* @param message The message that the error occurred on
*/
emitError(err, message) {
if (!message) {
this.emit("error", err, undefined);
}
else if (err.name === errors_js_1.SQSError.name) {
this.emit("error", err, message);
}
else if (err instanceof errors_js_1.TimeoutError) {
this.emit("timeout_error", err, message);
}
else {
this.emit("processing_error", err, message);
}
}
/**
* Poll for new messages from SQS
*/
poll() {
if (this.stopped) {
logger_js_1.logger.debug("cancelling_poll", {
detail: "Poll was called while consumer was stopped, cancelling poll...",
});
return;
}
logger_js_1.logger.debug("polling");
this.isPolling = true;
let currentPollingTimeout = this.pollingWaitTimeMs;
this.receiveMessage({
QueueUrl: this.queueUrl,
AttributeNames: this.attributeNames,
MessageAttributeNames: this.messageAttributeNames,
MessageSystemAttributeNames: this.messageSystemAttributeNames,
MaxNumberOfMessages: this.batchSize,
WaitTimeSeconds: this.waitTimeSeconds,
VisibilityTimeout: this.visibilityTimeout,
})
.then((output) => this.handleSqsResponse(output))
.catch((err) => {
this.emitError(err);
if ((0, errors_js_1.isConnectionError)(err)) {
logger_js_1.logger.debug("authentication_error", {
code: err.code || "Unknown",
detail: "There was an authentication error. Pausing before retrying.",
});
currentPollingTimeout = this.authenticationErrorTimeout;
}
return;
})
.then(() => {
if (this.pollingTimeoutId) {
clearTimeout(this.pollingTimeoutId);
}
this.pollingTimeoutId = setTimeout(() => this.poll(), currentPollingTimeout);
})
.catch((err) => {
this.emitError(err);
})
.finally(() => {
this.isPolling = false;
});
}
/**
* Send a request to SQS to retrieve messages
* @param params The required params to receive messages from SQS
*/
async receiveMessage(params) {
try {
if (this.preReceiveMessageCallback) {
await this.preReceiveMessageCallback();
}
const result = await this.sqs.send(new client_sqs_1.ReceiveMessageCommand(params), this.sqsSendOptions);
if (this.postReceiveMessageCallback) {
await this.postReceiveMessageCallback();
}
return result;
}
catch (err) {
throw (0, errors_js_1.toSQSError)(err, `SQS receive message failed: ${err.message}`, this.extendedAWSErrors, this.queueUrl);
}
}
/**
* Handles the response from AWS SQS, determining if we should proceed to
* the message handler.
* @param response The output from AWS SQS
*/
async handleSqsResponse(response) {
if ((0, validation_js_1.hasMessages)(response)) {
if (this.handleMessageBatch) {
await this.processMessageBatch(response.Messages);
}
else {
await Promise.all(response.Messages.map((message) => this.processMessage(message)));
}
this.emit("response_processed");
}
else if (response) {
this.emit("empty");
}
}
/**
* Process a message that has been received from SQS. This will execute the message
* handler and delete the message once complete.
* @param message The message that was delivered from SQS
*/
async processMessage(message) {
let heartbeatTimeoutId = undefined;
try {
this.emit("message_received", message);
if (this.heartbeatInterval) {
heartbeatTimeoutId = this.startHeartbeat(message);
}
const ackedMessage = await this.executeHandler(message);
if (ackedMessage?.MessageId === message.MessageId) {
await this.deleteMessage(message);
this.emit("message_processed", message);
}
}
catch (err) {
this.emitError(err, message);
if (this.terminateVisibilityTimeout !== false) {
if (typeof this.terminateVisibilityTimeout === "function") {
const timeout = this.terminateVisibilityTimeout([message]);
await this.changeVisibilityTimeout(message, timeout);
}
else {
const timeout = this.terminateVisibilityTimeout === true
? 0
: this.terminateVisibilityTimeout;
await this.changeVisibilityTimeout(message, timeout);
}
}
}
finally {
if (this.heartbeatInterval) {
clearInterval(heartbeatTimeoutId);
}
}
}
/**
* Process a batch of messages from the SQS queue.
* @param messages The messages that were delivered from SQS
*/
async processMessageBatch(messages) {
let heartbeatTimeoutId = undefined;
try {
messages.forEach((message) => {
this.emit("message_received", message);
});
if (this.heartbeatInterval) {
heartbeatTimeoutId = this.startHeartbeat(null, messages);
}
const ackedMessages = await this.executeBatchHandler(messages);
if (ackedMessages?.length > 0) {
await this.deleteMessageBatch(ackedMessages);
ackedMessages.forEach((message) => {
this.emit("message_processed", message);
});
}
}
catch (err) {
this.emit("error", err, messages);
if (this.terminateVisibilityTimeout !== false) {
if (typeof this.terminateVisibilityTimeout === "function") {
const timeout = this.terminateVisibilityTimeout(messages);
await this.changeVisibilityTimeoutBatch(messages, timeout);
}
else {
const timeout = this.terminateVisibilityTimeout === true
? 0
: this.terminateVisibilityTimeout;
await this.changeVisibilityTimeoutBatch(messages, timeout);
}
}
}
finally {
clearInterval(heartbeatTimeoutId);
}
}
/**
* Trigger a function on a set interval
* @param heartbeatFn The function that should be triggered
*/
startHeartbeat(message, messages) {
return setInterval(() => {
if (this.handleMessageBatch) {
return this.changeVisibilityTimeoutBatch(messages, this.visibilityTimeout);
}
return this.changeVisibilityTimeout(message, this.visibilityTimeout);
}, this.heartbeatInterval * 1000);
}
/**
* Change the visibility timeout on a message
* @param message The message to change the value of
* @param timeout The new timeout that should be set
*/
async changeVisibilityTimeout(message, timeout) {
try {
const input = {
QueueUrl: this.queueUrl,
ReceiptHandle: message.ReceiptHandle,
VisibilityTimeout: timeout,
};
return await this.sqs.send(new client_sqs_1.ChangeMessageVisibilityCommand(input), this.sqsSendOptions);
}
catch (err) {
this.emit("error", (0, errors_js_1.toSQSError)(err, `Error changing visibility timeout: ${err.message}`, this.extendedAWSErrors, this.queueUrl, message), message);
}
}
/**
* Change the visibility timeout on a batch of messages
* @param messages The messages to change the value of
* @param timeout The new timeout that should be set
*/
async changeVisibilityTimeoutBatch(messages, timeout) {
const params = {
QueueUrl: this.queueUrl,
Entries: messages.map((message) => ({
Id: message.MessageId,
ReceiptHandle: message.ReceiptHandle,
VisibilityTimeout: timeout,
})),
};
try {
return await this.sqs.send(new client_sqs_1.ChangeMessageVisibilityBatchCommand(params), this.sqsSendOptions);
}
catch (err) {
this.emit("error", (0, errors_js_1.toSQSError)(err, `Error changing visibility timeout: ${err.message}`, this.extendedAWSErrors, this.queueUrl, messages), messages);
}
}
/**
* Trigger the applications handleMessage function
* @param message The message that was received from SQS
*/
async executeHandler(message) {
let handleMessageTimeoutId = undefined;
try {
let result;
if (this.handleMessageTimeout) {
const pending = new Promise((_, reject) => {
handleMessageTimeoutId = setTimeout(() => {
reject(new errors_js_1.TimeoutError());
}, this.handleMessageTimeout);
});
result = await Promise.race([this.handleMessage(message), pending]);
}
else {
result = await this.handleMessage(message);
}
if (this.alwaysAcknowledge) {
return message;
}
if (result instanceof Object) {
return result;
}
if (result === undefined) {
return null;
}
if (result === null) {
if (this.strictReturn) {
throw new Error("strictReturn is enabled: handleMessage must return a Message object or an object with the same MessageId. Returning null is not allowed.");
}
console.warn("[DEPRECATION] Future versions will throw on void/null returns. Enable `strictReturn` now to prepare.");
return null;
}
return null;
}
catch (err) {
if (err instanceof errors_js_1.TimeoutError) {
throw (0, errors_js_1.toTimeoutError)(err, `Message handler timed out after ${this.handleMessageTimeout}ms: Operation timed out.`, message);
}
if (err instanceof Error) {
throw (0, errors_js_1.toStandardError)(err, `Unexpected message handler failure: ${err.message}`, message);
}
throw err;
}
finally {
if (handleMessageTimeoutId) {
clearTimeout(handleMessageTimeoutId);
}
}
}
/**
* Execute the application's message batch handler
* @param messages The messages that should be forwarded from the SQS queue
*/
async executeBatchHandler(messages) {
try {
const result = await this.handleMessageBatch(messages);
if (this.alwaysAcknowledge) {
return messages;
}
if (Array.isArray(result)) {
return result;
}
if (result === undefined) {
return [];
}
if (result === null) {
if (this.strictReturn) {
throw new Error("strictReturn is enabled: handleMessageBatch must return an array of Message objects. Returning null is not allowed.");
}
console.warn("[DEPRECATION] Future versions will throw on void/null returns. Enable `strictReturn` now to prepare.");
return [];
}
return [];
}
catch (err) {
if (err instanceof Error) {
throw (0, errors_js_1.toStandardError)(err, `Unexpected message batch handler failure: ${err.message}`, messages);
}
throw err;
}
}
/**
* Delete a single message from SQS
* @param message The message to delete from the SQS queue
*/
async deleteMessage(message) {
if (!this.shouldDeleteMessages) {
logger_js_1.logger.debug("skipping_delete", {
detail: "Skipping message delete since shouldDeleteMessages is set to false",
});
return;
}
logger_js_1.logger.debug("deleting_message", { messageId: message.MessageId });
const deleteParams = {
QueueUrl: this.queueUrl,
ReceiptHandle: message.ReceiptHandle,
};
try {
await this.sqs.send(new client_sqs_1.DeleteMessageCommand(deleteParams), this.sqsSendOptions);
}
catch (err) {
throw (0, errors_js_1.toSQSError)(err, `SQS delete message failed: ${err.message}`, this.extendedAWSErrors, this.queueUrl, message);
}
}
/**
* Delete a batch of messages from the SQS queue.
* @param messages The messages that should be deleted from SQS
*/
async deleteMessageBatch(messages) {
if (!this.shouldDeleteMessages) {
logger_js_1.logger.debug("skipping_delete", {
detail: "Skipping message delete since shouldDeleteMessages is set to false",
});
return;
}
logger_js_1.logger.debug("deleting_messages", {
messageIds: messages.map((msg) => msg.MessageId),
});
const deleteParams = {
QueueUrl: this.queueUrl,
Entries: messages.map((message) => ({
Id: message.MessageId,
ReceiptHandle: message.ReceiptHandle,
})),
};
try {
await this.sqs.send(new client_sqs_1.DeleteMessageBatchCommand(deleteParams), this.sqsSendOptions);
}
catch (err) {
throw (0, errors_js_1.toSQSError)(err, `SQS delete message failed: ${err.message}`, this.extendedAWSErrors, this.queueUrl, messages);
}
}
}
exports.Consumer = Consumer;