@message-queue-toolkit/sqs
Version:
SQS adapter for message-queue-toolkit
359 lines • 17.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.AbstractSqsConsumer = void 0;
const client_sqs_1 = require("@aws-sdk/client-sqs");
const core_1 = require("@message-queue-toolkit/core");
const sqs_consumer_1 = require("sqs-consumer");
const messageUtils_1 = require("../utils/messageUtils");
const sqsInitter_1 = require("../utils/sqsInitter");
const sqsMessageReader_1 = require("../utils/sqsMessageReader");
const sqsUtils_1 = require("../utils/sqsUtils");
const AbstractSqsPublisher_1 = require("./AbstractSqsPublisher");
const AbstractSqsService_1 = require("./AbstractSqsService");
const ABORT_EARLY_EITHER = {
error: 'abort',
};
const DEFAULT_MAX_RETRY_DURATION = 4 * 24 * 60 * 60; // 4 days in seconds
class AbstractSqsConsumer extends AbstractSqsService_1.AbstractSqsService {
consumers;
concurrentConsumersAmount;
transactionObservabilityManager;
consumerOptionsOverride;
handlerContainer;
deadLetterQueueOptions;
isDeduplicationEnabled;
maxRetryDuration;
deadLetterQueueUrl;
errorResolver;
executionContext;
_messageSchemaContainer;
constructor(dependencies, options, executionContext) {
super(dependencies, options);
this.transactionObservabilityManager = dependencies.transactionObservabilityManager;
this.errorResolver = dependencies.consumerErrorResolver;
this.consumerOptionsOverride = options.consumerOverrides ?? {};
this.deadLetterQueueOptions = options.deadLetterQueue;
this.maxRetryDuration = options.maxRetryDuration ?? DEFAULT_MAX_RETRY_DURATION;
this.executionContext = executionContext;
this.consumers = [];
this.concurrentConsumersAmount = options.concurrentConsumersAmount ?? 1;
this._messageSchemaContainer = this.resolveConsumerMessageSchemaContainer(options);
this.handlerContainer = new core_1.HandlerContainer({
messageTypeField: this.messageTypeField,
messageHandlers: options.handlers,
});
this.isDeduplicationEnabled = !!options.enableConsumerDeduplication;
}
async init() {
await super.init();
await this.initDeadLetterQueue();
}
async initDeadLetterQueue() {
if (!this.deadLetterQueueOptions)
return;
const { deletionConfig, locatorConfig, creationConfig, redrivePolicy } = this.deadLetterQueueOptions;
if (deletionConfig && creationConfig) {
await (0, sqsInitter_1.deleteSqs)(this.sqsClient, deletionConfig, creationConfig);
}
const result = await (0, sqsInitter_1.initSqs)(this.sqsClient, locatorConfig, creationConfig);
await this.sqsClient.send(new client_sqs_1.SetQueueAttributesCommand({
QueueUrl: this.queueUrl,
Attributes: {
RedrivePolicy: JSON.stringify({
deadLetterTargetArn: result.queueArn,
maxReceiveCount: redrivePolicy.maxReceiveCount,
}),
},
}));
this.deadLetterQueueUrl = result.queueUrl;
}
async start() {
await this.init();
this.stopExistingConsumers();
const visibilityTimeout = await this.getQueueVisibilityTimeout();
this.consumers = Array.from({ length: this.concurrentConsumersAmount }, () => this.createConsumer({ visibilityTimeout }));
for (const consumer of this.consumers) {
consumer.on('error', (err) => {
this.handleError(err, { queueName: this.queueName });
});
consumer.start();
}
}
async close(abort) {
await super.close();
this.stopExistingConsumers(abort ?? false);
}
createConsumer(options) {
return sqs_consumer_1.Consumer.create({
sqs: this.sqsClient,
queueUrl: this.queueUrl,
visibilityTimeout: options.visibilityTimeout,
messageAttributeNames: [`${AbstractSqsPublisher_1.PAYLOAD_OFFLOADING_ATTRIBUTE_PREFIX}*`],
...this.consumerOptionsOverride,
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: <explanation>
handleMessage: async (message) => {
if (message === null)
return;
const messageProcessingStartTimestamp = Date.now();
const deserializedMessage = await this.deserializeMessage(message);
if (deserializedMessage.error === 'abort') {
await this.failProcessing(message);
const messageId = this.tryToExtractId(message);
this.handleMessageProcessed({
message: null,
processingResult: { status: 'error', errorReason: 'invalidMessage' },
messageProcessingStartTimestamp,
queueName: this.queueName,
messageId: messageId.result,
});
return;
}
const { parsedMessage, originalMessage } = deserializedMessage.result;
if (this.isDeduplicationEnabledForMessage(parsedMessage) &&
(await this.isMessageDuplicated(parsedMessage, core_1.DeduplicationRequester.Consumer))) {
this.handleMessageProcessed({
message: parsedMessage,
processingResult: { status: 'consumed', skippedAsDuplicate: true },
messageProcessingStartTimestamp,
queueName: this.queueName,
messageId: this.tryToExtractId(message).result,
});
return;
}
const acquireLockResult = this.isDeduplicationEnabledForMessage(parsedMessage)
? await this.acquireLockForMessage(parsedMessage)
: { result: core_1.noopReleasableLock };
// Lock cannot be acquired as it is already being processed by another consumer.
// We don't want to discard message yet as we don't know if the other consumer will be able to process it successfully.
// We're re-queueing the message, so it can be processed later.
if (acquireLockResult.error) {
await this.handleRetryLater(message, originalMessage, parsedMessage, messageProcessingStartTimestamp);
return message;
}
// While the consumer was waiting for a lock to be acquired, the message might have been processed
// by another consumer already, hence we need to check again if the message is not marked as duplicated.
if (this.isDeduplicationEnabledForMessage(parsedMessage) &&
(await this.isMessageDuplicated(parsedMessage, core_1.DeduplicationRequester.Consumer))) {
await acquireLockResult.result?.release();
this.handleMessageProcessed({
message: parsedMessage,
processingResult: { status: 'consumed', skippedAsDuplicate: true },
messageProcessingStartTimestamp,
queueName: this.queueName,
messageId: this.tryToExtractId(message).result,
});
return;
}
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
const messageType = parsedMessage[this.messageTypeField];
const transactionSpanId = `queue_${this.queueName}:${messageType}`;
// @ts-ignore
const uniqueTransactionKey = parsedMessage[this.messageIdField];
this.transactionObservabilityManager?.start(transactionSpanId, uniqueTransactionKey);
if (this.logMessages) {
const resolvedLogMessage = this.resolveMessageLog(parsedMessage, messageType);
this.logMessage(resolvedLogMessage);
}
const result = await this.internalProcessMessage(parsedMessage, messageType)
.catch((err) => {
this.handleError(err);
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
return { error: err };
})
.finally(() => {
this.transactionObservabilityManager?.stop(uniqueTransactionKey);
});
// success
if (result.result) {
await this.deduplicateMessage(parsedMessage, core_1.DeduplicationRequester.Consumer);
await acquireLockResult.result?.release();
this.handleMessageProcessed({
message: originalMessage,
processingResult: { status: 'consumed' },
messageProcessingStartTimestamp,
queueName: this.queueName,
});
return message;
}
if (result.error === 'retryLater') {
await acquireLockResult.result?.release();
await this.handleRetryLater(message, originalMessage, parsedMessage, messageProcessingStartTimestamp);
return message;
}
await acquireLockResult.result?.release();
this.handleMessageProcessed({
message: parsedMessage,
processingResult: { status: 'error', errorReason: 'handlerError' },
messageProcessingStartTimestamp,
queueName: this.queueName,
});
return Promise.reject(result.error);
},
});
}
async handleRetryLater(message, originalMessage, parsedMessage, messageProcessingStartTimestamp) {
if (this.shouldBeRetried(originalMessage, this.maxRetryDuration)) {
await this.sqsClient.send(new client_sqs_1.SendMessageCommand({
QueueUrl: this.queueUrl,
DelaySeconds: this.getMessageRetryDelayInSeconds(originalMessage),
MessageBody: JSON.stringify(this.updateInternalProperties(originalMessage)),
}));
this.handleMessageProcessed({
message: parsedMessage,
processingResult: { status: 'retryLater' },
messageProcessingStartTimestamp,
queueName: this.queueName,
});
}
else {
await this.failProcessing(message);
this.handleMessageProcessed({
message: parsedMessage,
processingResult: { status: 'error', errorReason: 'retryLaterExceeded' },
messageProcessingStartTimestamp,
queueName: this.queueName,
});
}
}
stopExistingConsumers(abort) {
for (const consumer of this.consumers) {
consumer.stop({
abort,
});
}
}
async internalProcessMessage(message, messageType) {
const preHandlerOutput = await this.processPrehandlers(message, messageType);
const barrierResult = await this.preHandlerBarrier(message, messageType, preHandlerOutput);
if (barrierResult.isPassing) {
return this.processMessage(message, messageType, {
preHandlerOutput,
barrierOutput: barrierResult.output,
});
}
return { error: 'retryLater' };
}
processMessage(message, messageType,
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
preHandlingOutputs) {
const handler = this.handlerContainer.resolveHandler(messageType);
return handler.handler(message, this.executionContext, preHandlingOutputs);
}
processPrehandlers(message, messageType) {
const handlerConfig = this.handlerContainer.resolveHandler(messageType);
return this.processPrehandlersInternal(handlerConfig.preHandlers, message);
}
preHandlerBarrier(message, messageType, preHandlerOutput) {
const handler = this.handlerContainer.resolveHandler(messageType);
return this.preHandlerBarrierInternal(handler.preHandlerBarrier, message, this.executionContext, preHandlerOutput);
}
resolveSchema(message) {
return this._messageSchemaContainer.resolveSchema(message);
}
// eslint-disable-next-line max-params
resolveNextFunction(preHandlers, message, index, preHandlerOutput, resolve, reject) {
return this.resolveNextPreHandlerFunctionInternal(preHandlers, this.executionContext, message, index, preHandlerOutput, resolve, reject);
}
resolveMessageLog(message, messageType) {
const handler = this.handlerContainer.resolveHandler(messageType);
return handler.messageLogFormatter(message);
}
resolveMessage(message) {
return (0, sqsMessageReader_1.readSqsMessage)(message, this.errorResolver);
}
isDeduplicationEnabledForMessage(message) {
return this.isDeduplicationEnabled && super.isDeduplicationEnabledForMessage(message);
}
async resolveMaybeOffloadedPayloadMessage(message) {
const resolveMessageResult = this.resolveMessage(message);
if ((0, core_1.isMessageError)(resolveMessageResult.error)) {
this.handleError(resolveMessageResult.error);
return ABORT_EARLY_EITHER;
}
// Empty content for whatever reason
if (!resolveMessageResult.result || !resolveMessageResult.result.body) {
return ABORT_EARLY_EITHER;
}
if ((0, messageUtils_1.hasOffloadedPayload)(resolveMessageResult.result)) {
const retrieveOffloadedMessagePayloadResult = await this.retrieveOffloadedMessagePayload(resolveMessageResult.result.body);
if (retrieveOffloadedMessagePayloadResult.error) {
this.handleError(retrieveOffloadedMessagePayloadResult.error);
return ABORT_EARLY_EITHER;
}
resolveMessageResult.result.body = retrieveOffloadedMessagePayloadResult.result;
}
return resolveMessageResult;
}
tryToExtractId(message) {
const resolveMessageResult = this.resolveMessage(message);
if ((0, core_1.isMessageError)(resolveMessageResult.error)) {
this.handleError(resolveMessageResult.error);
return ABORT_EARLY_EITHER;
}
const resolvedMessage = resolveMessageResult.result;
// Empty content for whatever reason
if (!resolvedMessage || !resolvedMessage.body)
return ABORT_EARLY_EITHER;
// @ts-ignore
if (this.messageIdField in resolvedMessage.body) {
return {
// @ts-ignore
result: resolvedMessage.body[this.messageIdField],
};
}
return ABORT_EARLY_EITHER;
}
async deserializeMessage(message) {
if (message === null) {
return ABORT_EARLY_EITHER;
}
const resolveMessageResult = await this.resolveMaybeOffloadedPayloadMessage(message);
if (resolveMessageResult.error) {
return ABORT_EARLY_EITHER;
}
const resolveSchemaResult = this.resolveSchema(resolveMessageResult.result.body);
if (resolveSchemaResult.error) {
this.handleError(resolveSchemaResult.error);
return ABORT_EARLY_EITHER;
}
const deserializationResult = (0, core_1.parseMessage)(resolveMessageResult.result.body, resolveSchemaResult.result, this.errorResolver);
if ((0, core_1.isMessageError)(deserializationResult.error)) {
this.handleError(deserializationResult.error);
return ABORT_EARLY_EITHER;
}
// Empty content for whatever reason
if (!deserializationResult.result) {
return ABORT_EARLY_EITHER;
}
return {
result: deserializationResult.result,
};
}
async failProcessing(message) {
if (!this.deadLetterQueueUrl)
return;
const command = new client_sqs_1.SendMessageCommand({
QueueUrl: this.deadLetterQueueUrl,
MessageBody: message.Body,
});
await this.sqsClient.send(command);
}
async getQueueVisibilityTimeout() {
let visibilityTimeoutString;
if (this.creationConfig) {
visibilityTimeoutString = this.creationConfig.queue.Attributes?.VisibilityTimeout;
}
else {
// if user is using locatorConfig, we should look into queue config
const queueAttributes = await (0, sqsUtils_1.getQueueAttributes)(this.sqsClient, this.queueUrl, [
'VisibilityTimeout',
]);
visibilityTimeoutString = queueAttributes.result?.attributes?.VisibilityTimeout;
}
// parseInt is safe because if the value is not a number process should have failed on init
return visibilityTimeoutString ? Number.parseInt(visibilityTimeoutString) : undefined;
}
}
exports.AbstractSqsConsumer = AbstractSqsConsumer;
//# sourceMappingURL=AbstractSqsConsumer.js.map