@iamdeniz/aws-sqs-consumer
Version:
Advanced AWS SQS message consumer with retry, DLQ, batch processing, metrics, and middleware support
839 lines • 33.5 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const client_sqs_1 = require("@aws-sdk/client-sqs");
const credential_providers_1 = require("@aws-sdk/credential-providers");
const node_http_handler_1 = require("@aws-sdk/node-http-handler");
const events_1 = require("events");
const https_proxy_agent_1 = require("https-proxy-agent");
const events_2 = require("../enums/events");
/**
* QueueConsumer class for processing AWS SQS queue messages
* Provides both individual message processing and batch processing capabilities
* Supports middleware, retry logic, dead letter queues, and performance metrics
*/
class QueueConsumer extends events_1.EventEmitter {
_client;
_url;
_handler;
_batchHandler;
_isShuttingDown = false;
_isRunning = false;
_stoppedFunction = null;
_retryOptions;
_dlqOptions;
_metricsOptions;
_batchOptions;
_middlewareOptions;
_awsConfig;
// Middleware stacks
_messageMiddleware = [];
_batchMiddleware = [];
// Metrics tracking
_startTime = new Date();
_messagesProcessed = 0;
_messagesFailed = 0;
_messagesSentToDlq = 0;
_totalRetries = 0;
_totalProcessingTime = 0;
_messagesFiltered = 0;
static DEFAULT_RETRY_OPTIONS = {
maxRetries: 3,
initialDelayMs: 500,
backoffMultiplier: 2,
maxDelayMs: 30000,
extendVisibilityTimeout: true,
};
static DEFAULT_DLQ_OPTIONS = {
queueUrl: '',
enabled: false,
includeFailureMetadata: true,
};
static DEFAULT_METRICS_OPTIONS = {
enabled: true,
includeMessageBody: false,
emitPerformanceMetrics: true,
};
static DEFAULT_BATCH_OPTIONS = {
enabled: false,
maxBatchSize: 10,
batchDeletes: true,
waitTimeSeconds: 5,
visibilityTimeout: 115,
atomicBatches: false,
};
static DEFAULT_MIDDLEWARE_OPTIONS = {
enabled: false,
respectFiltering: true,
applyToBatches: true,
};
static DEFAULT_AWS_CONFIG = {
useDefaultCredentialProviderChain: true,
};
/**
* Creates a new QueueConsumer instance
* @param options Configuration options for the consumer
*/
constructor(options) {
super();
this._url = options.url;
this._handler = options.handler;
this._batchHandler = options.batchHandler;
this._retryOptions = {
...QueueConsumer.DEFAULT_RETRY_OPTIONS,
...options.retryOptions || {},
};
this._dlqOptions = {
...QueueConsumer.DEFAULT_DLQ_OPTIONS,
...options.deadLetterQueueOptions || {},
};
this._metricsOptions = {
...QueueConsumer.DEFAULT_METRICS_OPTIONS,
...options.metricsOptions || {},
};
this._batchOptions = {
...QueueConsumer.DEFAULT_BATCH_OPTIONS,
...options.batchOptions || {},
};
this._middlewareOptions = {
...QueueConsumer.DEFAULT_MIDDLEWARE_OPTIONS,
...options.middlewareOptions || {},
};
this._awsConfig = {
...QueueConsumer.DEFAULT_AWS_CONFIG,
...options.awsConfig || {},
};
// Initialize SQS client with credentials and configuration
if (options.sqs) {
this._client = options.sqs;
}
else {
const region = this.getRegionFromQueueUrl(this._url);
const clientConfig = {
region: this._awsConfig.region || region,
};
// Apply AWS configuration if provided
if (this._awsConfig.accessKeyId && this._awsConfig.secretAccessKey) {
// Use explicit credentials
clientConfig.credentials = {
accessKeyId: this._awsConfig.accessKeyId,
secretAccessKey: this._awsConfig.secretAccessKey,
sessionToken: this._awsConfig.sessionToken
};
}
else if (this._awsConfig.profile) {
// Use profile credentials
clientConfig.credentials = (0, credential_providers_1.fromIni)({
profile: this._awsConfig.profile,
});
}
// Apply custom endpoint if provided (for testing or LocalStack)
if (this._awsConfig.endpoint) {
clientConfig.endpoint = this._awsConfig.endpoint;
}
// Apply HTTP options if provided
if (this._awsConfig.httpOptions) {
if (this._awsConfig.httpOptions.proxy) {
// Create a proxy agent if a proxy URL is provided
const proxyAgent = new https_proxy_agent_1.HttpsProxyAgent(this._awsConfig.httpOptions.proxy);
clientConfig.requestHandler = new node_http_handler_1.NodeHttpHandler({
httpAgent: proxyAgent,
httpsAgent: proxyAgent,
});
}
// Configure timeouts
if (this._awsConfig.httpOptions.timeout || this._awsConfig.httpOptions.connectTimeout) {
clientConfig.requestHandler = new node_http_handler_1.NodeHttpHandler({
connectionTimeout: this._awsConfig.httpOptions.connectTimeout,
requestTimeout: this._awsConfig.httpOptions.timeout,
});
}
}
this._client = new client_sqs_1.SQSClient(clientConfig);
}
}
/**
* Starts the queue consumer
* @returns This instance for chaining
*/
async run() {
if (this._isRunning) {
return;
}
this._isRunning = true;
this._startTime = new Date();
// Reset metrics on start
this._messagesProcessed = 0;
this._messagesFailed = 0;
this._messagesSentToDlq = 0;
this._totalRetries = 0;
this._totalProcessingTime = 0;
this._messagesFiltered = 0;
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.STARTED, this.getMetrics());
}
// Start polling for messages
await this.pollMessages();
}
/**
* Stops the queue consumer gracefully
* @returns Promise that resolves when consumer has fully stopped
*/
stop() {
if (!this._isRunning) {
return;
}
this._isShuttingDown = true;
}
/**
* Returns the number of available messages in the queue
* @returns Promise resolving to the number of available messages
*/
async getAvailableQueueNumber() {
const data = await this._client.send(new client_sqs_1.GetQueueAttributesCommand({
QueueUrl: this._url,
AttributeNames: ['ApproximateNumberOfMessages'],
}));
const availableMessages = data?.Attributes?.ApproximateNumberOfMessages;
if (availableMessages == null)
return 0;
return parseInt(availableMessages);
}
/**
* Sets a function to be called when the consumer stops
* @param stopFunction Function to execute when consumer stops
*/
setStoppedFunction(stopFunction) {
this._stoppedFunction = stopFunction;
}
/**
* Registers a message middleware function
* @param middleware The middleware function to register
* @returns This instance for chaining
*/
use(middleware) {
this._messageMiddleware.push(middleware);
return this;
}
/**
* Registers a batch middleware function
* @param middleware The middleware function to register
* @returns This instance for chaining
*/
useBatch(middleware) {
this._batchMiddleware.push(middleware);
return this;
}
get isRunning() {
return this._isRunning;
}
/**
* Extract region from queue URL
* @param queueUrl The SQS queue URL
* @returns The AWS region extracted from the URL
*/
getRegionFromQueueUrl(queueUrl) {
try {
// Parse queue URL to extract region
// Format: https://sqs.{region}.amazonaws.com/{account}/{queue}
const matches = queueUrl.match(/sqs\.([^.]+)\.amazonaws\.com/);
return matches && matches[1] ? matches[1] : 'us-east-1'; // Default to us-east-1
}
catch (err) {
console.warn('Could not extract region from queue URL, using default');
return 'us-east-1';
}
}
/**
* Process received messages
* @param messages Array of SQS messages to process
*/
async processMessages(messages) {
try {
// For standard queues, process all messages with the same group ID
const groupId = 'standard-queue-group';
if (this._batchOptions.enabled && this._batchHandler) {
// Process the entire batch with the batch handler
await this.processBatchWithBatchHandler(groupId, messages);
}
else {
// Process each message individually
await this.processMessageGroup(groupId, messages);
}
}
catch (err) {
console.error('Error processing messages', err);
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.ERROR, {
error: err,
context: 'message_processing',
});
}
}
}
/**
* Get consumer metrics
* @returns Object containing current consumer metrics
*/
getMetrics() {
const now = new Date();
const activeTimeMs = now.getTime() - this._startTime.getTime();
return {
queueUrl: this._url,
activeTimeMs,
messagesProcessed: this._messagesProcessed,
messagesFailed: this._messagesFailed,
messagesSentToDlq: this._messagesSentToDlq,
retryCount: this._totalRetries,
startTime: this._startTime.toISOString(),
averageProcessingTimeMs: this._messagesProcessed > 0
? this._totalProcessingTime / this._messagesProcessed
: 0,
};
}
/**
* Process a group of messages
* @param groupId Group ID for the messages
* @param messages Array of SQS messages to process
*/
async processMessageGroup(groupId, messages) {
for (const message of messages) {
try {
await this.processIndividualMessage(message, groupId);
}
catch (err) {
console.error('Error processing individual message', {
messageId: message.MessageId,
error: err
});
}
}
}
/**
* Process an individual message
* @param message SQS message to process
* @param groupId Group ID for the message
*/
async processIndividualMessage(message, groupId) {
// Implementation for processing individual messages
// This would include middleware, retry logic, etc.
// For now, just a placeholder
if (!this._handler)
return;
const parsed = this.parseMessage(message, groupId);
if (!parsed)
return;
try {
// Apply middleware if enabled
if (this._middlewareOptions.enabled) {
const middlewareContext = {
message,
body: parsed.body,
messageId: parsed.id,
groupId,
metadata: {},
shouldProcess: true
};
await this.executeMessageMiddlewarePipeline(middlewareContext);
// Skip processing if middleware indicated to filter this message
if (this._middlewareOptions.respectFiltering && !middlewareContext.shouldProcess) {
this._messagesFiltered++;
return;
}
// Use potentially modified body from middleware
parsed.body = middlewareContext.body;
}
await this._handler(parsed.body);
// Handle success, delete message, etc.
}
catch (err) {
// Handle error, retry logic, etc.
await this.sendToDeadLetterQueue(message, err, 0);
}
}
/**
* Execute batch middleware pipeline
* @param context Batch middleware context
*/
async executeBatchMiddlewarePipeline(context) {
if (!this._middlewareOptions.enabled || this._batchMiddleware.length === 0) {
return;
}
let index = 0;
const next = async () => {
if (index < this._batchMiddleware.length) {
const middleware = this._batchMiddleware[index++];
await middleware(context, next);
}
};
await next();
}
/**
* Execute message middleware pipeline
* @param context Message middleware context
*/
async executeMessageMiddlewarePipeline(context) {
if (!this._middlewareOptions.enabled || this._messageMiddleware.length === 0) {
return;
}
let index = 0;
const next = async () => {
if (index < this._messageMiddleware.length) {
const middleware = this._messageMiddleware[index++];
await middleware(context, next);
}
};
await next();
}
/**
* Send a failed message to the Dead Letter Queue
* @param message Failed SQS message
* @param error Error that caused the failure
* @param retryCount Number of retries attempted
*/
async sendToDeadLetterQueue(message, error, retryCount) {
// Implementation for sending to DLQ
// This is a placeholder
if (!this._dlqOptions.enabled || !this._dlqOptions.queueUrl) {
console.warn('DLQ not configured, failed message will be lost', {
messageId: message.MessageId,
error
});
return;
}
try {
// Send to DLQ implementation would go here
this._messagesSentToDlq++;
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.MESSAGE_SENT_TO_DLQ, {
messageId: message.MessageId,
error,
retryCount
});
}
}
catch (err) {
console.error('Failed to send message to DLQ', {
messageId: message.MessageId,
error: err
});
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.MESSAGE_DLQ_FAILED, {
messageId: message.MessageId,
error: err
});
}
}
}
/**
* Calculate retry backoff delay using exponential backoff
* @param retryCount Current retry attempt number
* @returns Delay in milliseconds before next retry
*/
calculateBackoffDelay(retryCount) {
const { initialDelayMs, backoffMultiplier, maxDelayMs } = this._retryOptions;
const delay = initialDelayMs * Math.pow(backoffMultiplier, retryCount);
return Math.min(delay, maxDelayMs);
}
/**
* Extend visibility timeout for a message
* @param message SQS message
* @param visibilityTimeout New visibility timeout in seconds
*/
async extendMessageVisibility(message, visibilityTimeout) {
if (!message.ReceiptHandle)
return;
try {
await this._client.send(new client_sqs_1.ChangeMessageVisibilityCommand({
QueueUrl: this._url,
VisibilityTimeout: visibilityTimeout,
ReceiptHandle: message.ReceiptHandle
}));
}
catch (err) {
console.error('Failed to extend message visibility timeout', {
messageId: message.MessageId,
error: err
});
}
}
/**
* Sleep for a specified duration
* @param ms Time to sleep in milliseconds
*/
async sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Main polling loop for retrieving messages from the queue
*/
async pollMessages() {
this._isShuttingDown = false;
while (!this._isShuttingDown) {
try {
const command = new client_sqs_1.ReceiveMessageCommand({
QueueUrl: this._url,
MaxNumberOfMessages: this._batchOptions.enabled ? this._batchOptions.maxBatchSize : 4,
WaitTimeSeconds: this._batchOptions.waitTimeSeconds,
VisibilityTimeout: this._batchOptions.visibilityTimeout,
MessageSystemAttributeNames: ['MessageGroupId'],
});
const response = await this._client.send(command);
if (response.Messages && response.Messages.length > 0) {
await this.processMessages(response.Messages);
}
}
catch (err) {
console.error('Error when get new queue', err);
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.ERROR, {
error: err,
context: 'polling',
});
}
}
// Emit performance metrics periodically
if (this._metricsOptions.enabled && this._metricsOptions.emitPerformanceMetrics) {
this.emit(events_2.QueueConsumerEvents.PERFORMANCE_METRICS, this.getMetrics());
}
}
if (this._stoppedFunction !== null) {
await this._stoppedFunction();
}
// Final metrics emission
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.STOPPED, this.getMetrics());
}
this._isRunning = false;
}
/**
* Parses an SQS message into a more usable format
* @param message Raw SQS message
* @param groupId Group ID for the message
* @returns Parsed message or null if parsing failed
*/
parseMessage(message, groupId) {
try {
const body = JSON.parse(message.Body || 'null');
if (body == null) {
return null; // Invalid message
}
return {
original: message,
body,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
id: message.MessageId,
groupId,
};
}
catch (err) {
console.error('Failed to parse message body', {
messageId: message.MessageId,
error: err,
});
return null;
}
}
/**
* Deletes a batch of messages in a single SQS request
* @param messages Array of SQS messages to delete
* @returns Object containing arrays of successful and failed message IDs
*/
async deleteMessageBatch(messages) {
if (!messages.length) {
return { successful: [], failed: [] };
}
try {
const entries = messages.map(message => ({
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
Id: message.MessageId,
ReceiptHandle: message.ReceiptHandle,
}));
const response = await this._client.send(new client_sqs_1.DeleteMessageBatchCommand({
QueueUrl: this._url,
Entries: entries,
}));
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const successful = (response.Successful || []).map(s => s.Id);
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const failed = (response.Failed || []).map(f => f.Id);
if (failed.length && this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.ERROR, {
error: new Error(`Failed to delete ${failed.length} messages`),
context: 'batch_delete',
messageIds: failed,
});
}
return { successful, failed };
}
catch (err) {
console.error('Failed to delete message batch', { error: err });
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.ERROR, {
error: err,
context: 'batch_delete',
});
}
return {
successful: [],
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
failed: messages.map(m => m.MessageId),
};
}
}
/**
* Process a batch of messages using the batch handler
* @param groupId Group ID for the batch
* @param messages Array of SQS messages to process as a batch
*/
async processBatchWithBatchHandler(groupId, messages) {
// Safeguard: fall back to individual processing if no batch handler
if (!this._batchHandler) {
await this.processMessageGroup(groupId, messages);
return;
}
const batchStartTime = new Date();
let retryCount = 0;
let succeeded = false;
// If batch middleware is enabled, apply it to the entire group first
if (this._middlewareOptions.enabled && this._middlewareOptions.applyToBatches) {
// Parse messages first for middleware
const messageContexts = messages.map(message => {
try {
const body = JSON.parse(message.Body || 'null');
return {
message,
body,
messageId: message.MessageId || 'unknown',
groupId,
metadata: {},
shouldProcess: body != null
};
}
catch (err) {
return {
message,
body: null,
messageId: message.MessageId || 'unknown',
groupId,
metadata: {},
shouldProcess: false
};
}
});
const batchContext = {
messages: messageContexts,
groupId,
metadata: {}
};
// Apply batch middleware
await this.executeBatchMiddlewarePipeline(batchContext);
// Filter out messages that should be skipped based on middleware
if (this._middlewareOptions.respectFiltering) {
const filteredMessages = messages.filter((_msg, index) => {
const shouldProcess = messageContexts[index].shouldProcess;
if (!shouldProcess) {
this._messagesFiltered++;
}
return shouldProcess;
});
// If all messages were filtered out, we're done
if (filteredMessages.length === 0) {
return;
}
// Process only the filtered messages
messages = filteredMessages;
}
}
// Parse messages first
const parsedMessages = [];
for (const message of messages) {
const parsed = this.parseMessage(message, groupId);
if (parsed) {
parsedMessages.push(parsed);
}
else {
this._messagesFailed++;
// Send invalid messages to DLQ
await this.sendToDeadLetterQueue(message, new Error('Invalid message body'), 0);
}
}
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.BATCH_PROCESSING_STARTED, {
groupId,
messageCount: parsedMessages.length,
startTime: batchStartTime.toISOString(),
});
// Emit received events for each message
for (const parsed of parsedMessages) {
this.emit(events_2.QueueConsumerEvents.MESSAGE_RECEIVED, {
messageId: parsed.id,
groupId,
receivedAt: new Date().toISOString(),
});
}
}
if (parsedMessages.length === 0) {
// All messages were invalid, nothing to process
const batchEndTime = new Date();
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.BATCH_PROCESSING_COMPLETED, {
groupId,
messageCount: 0,
processingTimeMs: batchEndTime.getTime() - batchStartTime.getTime(),
startTime: batchStartTime.toISOString(),
endTime: batchEndTime.toISOString(),
});
}
return;
}
// Process the batch with retry logic
while (retryCount <= this._retryOptions.maxRetries && !succeeded) {
const processingStartTime = new Date();
if (this._metricsOptions.enabled) {
this.emit(events_2.QueueConsumerEvents.MESSAGE_PROCESSING_STARTED, {
batchSize: parsedMessages.length,
groupId,
retryAttempt: retryCount,
startTime: processingStartTime.toISOString(),
});
}
try {
// Process the entire batch with the non-null batchHandler (already checked above)
const batchHandler = this._batchHandler;
const result = await batchHandler(parsedMessages);
const processingEndTime = new Date();
const processingTimeMs = processingEndTime.getTime() - processingStartTime.getTime();
// If atomic batches are required and any message failed, throw to retry the whole batch
if (this._batchOptions.atomicBatches && result.failed.length > 0) {
const failedIds = result.failed.map((f) => f.id).join(', ');
// noinspection ExceptionCaughtLocallyJS
throw new Error(`Atomic batch processing failed for messages: ${failedIds}`);
}
// Count successful messages
this._messagesProcessed += result.successful.length;
this._totalProcessingTime += processingTimeMs;
if (result.failed.length > 0) {
this._messagesFailed += result.failed.length;
}
// Delete successful messages
const successfulMessages = result.successful
.map((id) => parsedMessages.find(m => m.id === id)?.original)
.filter((m) => m);
// Delete successful messages (either in batch or individually)
if (successfulMessages.length > 0) {
if (this._batchOptions.batchDeletes) {
await this.deleteMessageBatch(successfulMessages);
}
else {
for (const message of successfulMessages) {
await this._client.send(new client_sqs_1.DeleteMessageCommand({
QueueUrl: this._url,
ReceiptHandle: message.ReceiptHandle,
}));
}
}
}
// Send failed messages to DLQ
for (const item of result.failed) {
const message = parsedMessages.find(m => m.id === item.id)?.original;
if (message) {
await this.sendToDeadLetterQueue(message, item.error, retryCount);
}
}
// Emit metrics
if (this._metricsOptions.enabled) {
// Emit success events for successful messages
for (const id of result.successful) {
const parsed = parsedMessages.find((m) => m.id === id);
if (parsed) {
const processingMetrics = {
messageId: parsed.id,
groupId,
processingTimeMs,
retryCount,
startTime: processingStartTime.toISOString(),
endTime: processingEndTime.toISOString(),
body: this._metricsOptions.includeMessageBody ? parsed.body : undefined,
};
this.emit(events_2.QueueConsumerEvents.MESSAGE_PROCESSED, processingMetrics);
}
}
// Emit failure events for failed messages
for (const failed of result.failed) {
const parsed = parsedMessages.find((m) => m.id === failed.id);
if (parsed) {
const errorMetrics = {
messageId: parsed.id,
error: failed.error,
retryAttempt: retryCount,
willRetry: false,
body: this._metricsOptions.includeMessageBody ? parsed.body : undefined,
};
this.emit(events_2.QueueConsumerEvents.MESSAGE_PROCESSING_FAILED, errorMetrics);
}
}
// Emit overall batch metrics
this.emit(events_2.QueueConsumerEvents.BATCH_PROCESSING_COMPLETED, {
groupId,
messageCount: parsedMessages.length,
processingTimeMs,
startTime: processingStartTime.toISOString(),
endTime: processingEndTime.toISOString(),
successful: result.successful.length,
failed: result.failed.length,
});
}
// If we had partial success (some messages failed but not all), we're done
succeeded = true;
}
catch (err) {
const processingEndTime = new Date();
const processingTimeMs = processingEndTime.getTime() - processingStartTime.getTime();
const isLastRetry = retryCount >= this._retryOptions.maxRetries;
console.error('Batch processing error', {
error: err,
groupId,
retryAttempt: retryCount,
willRetry: !isLastRetry,
duration: processingTimeMs
});
if (this._metricsOptions.enabled) {
const errorMetrics = {
groupId,
error: err,
retryAttempt: retryCount,
willRetry: !isLastRetry,
batchSize: parsedMessages.length,
};
this.emit(isLastRetry
? events_2.QueueConsumerEvents.BATCH_PROCESSING_FAILED
: events_2.QueueConsumerEvents.BATCH_PROCESSING_RETRY, errorMetrics);
}
if (isLastRetry) {
// After all retries are exhausted, send each message to DLQ
this._messagesFailed += parsedMessages.length;
for (const parsed of parsedMessages) {
await this.sendToDeadLetterQueue(parsed.original, err, retryCount);
}
break; // No more retries
}
else {
this._totalRetries++;
}
// Calculate and wait for backoff delay
const delayMs = this.calculateBackoffDelay(retryCount);
// If configured, extend visibility timeout during retry delay for all messages
if (this._retryOptions.extendVisibilityTimeout) {
const visibilityExtensionSeconds = Math.ceil(delayMs / 1000) + 5;
for (const parsed of parsedMessages) {
await this.extendMessageVisibility(parsed.original, visibilityExtensionSeconds);
}
}
await this.sleep(delayMs);
retryCount++;
}
}
}
}
exports.default = QueueConsumer;
//# sourceMappingURL=QueueConsumer.js.map