@rewaa/event-broker
Version:
A broker for all the events that Rewaa will ever produce or consume
968 lines • 47.2 kB
JavaScript
"use strict";
var __rest = (this && this.__rest) || function (s, e) {
var t = {};
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
t[p] = s[p];
if (s != null && typeof Object.getOwnPropertySymbols === "function")
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
t[p[i]] = s[p[i]];
}
return t;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SqnsEmitter = void 0;
const sqs_consumer_1 = require("sqs-consumer");
const constants_1 = require("../constants");
const uuid_1 = require("uuid");
const types_1 = require("../types");
const producer_sns_1 = require("../producers/producer.sns");
const producer_sqs_1 = require("../producers/producer.sqs");
const lambda_client_1 = require("../utils/lambda.client");
const outbox_sqns_1 = require("../outbox/outbox.sqns");
const utils_1 = require("../utils/utils");
const dynamo_client_1 = require("../utils/dynamo.client");
const types_2 = require("../utils/types");
const constants_2 = require("../utils/constants");
const crypto_1 = require("crypto");
class SqnsEmitter {
constructor(logger, options) {
this.logger = logger;
this.topicListeners = new Map();
this.topics = new Map();
this.queues = new Map();
this.consumersStarted = false;
this.getQueueName = (topic, isDLQ = false) => {
var _a, _b;
let qName = "";
const queuePrefix = isDLQ ? constants_1.DLQ_PREFIX : constants_1.SOURCE_QUEUE_PREFIX;
const { exchangeType } = topic;
const separateConsumerGroup = this.getSeparateConsumer(topic);
if (separateConsumerGroup) {
qName = separateConsumerGroup;
}
else {
if (topic.isFifo) {
qName = ((_a = this.options.defaultQueueOptions) === null || _a === void 0 ? void 0 : _a.fifo.name) || "";
}
else {
qName = ((_b = this.options.defaultQueueOptions) === null || _b === void 0 ? void 0 : _b.standard.name) || "";
}
}
if (exchangeType === types_1.ExchangeType.Queue && !separateConsumerGroup) {
qName = topic.name;
}
qName = qName.replace(".fifo", "");
return `${this.options.environment}_${queuePrefix}_${qName}${this.isConsumerFifo(topic) ? ".fifo" : ""}`;
};
this.getTopicName = (topic) => {
const topicName = topic.name.replace(".fifo", "");
return `${this.options.environment}_${topicName}${topic.isFifo ? ".fifo" : ""}`;
};
this.logFailedEvent = (data) => {
if (!this.options.eventOnFailure) {
return;
}
this.localEmitter.emit(this.options.eventOnFailure, data);
};
this.getBatchMessagesForQueue = (topicName, messages) => messages.map((message) => {
return {
/**
* @todo Un-array this when switching to payload version 2
*/
data: [message.data],
deduplicationId: message.deduplicationId,
eventName: topicName,
messageAttributes: message.MessageAttributes,
messageGroupId: message.partitionKey || topicName,
id: message.id,
delay: message.delay || constants_1.DEFAULT_MESSAGE_DELAY,
};
});
this.getBatchMessagesForTopic = (topicName, messages) => messages.map((message) => {
return {
/**
* @todo Un-array this when switching to payload version 2
*/
data: [message.data],
deduplicationId: message.deduplicationId,
eventName: topicName,
messageAttributes: message.MessageAttributes,
messageGroupId: message.partitionKey || topicName,
id: message.id,
};
});
this.handleMessageReceipt = async (message, queueUrl, deleteOptions) => {
var _a;
const executionContext = {
executionTraceId: (0, uuid_1.v4)(),
messageId: message.MessageId,
receiptHandler: message.ReceiptHandle,
};
this.logger.info(`Message started ${queueUrl}_${executionContext.executionTraceId}_${new Date()}_${(_a = message === null || message === void 0 ? void 0 : message.Body) === null || _a === void 0 ? void 0 : _a.toString()}`);
await this.onMessageReceived(message, queueUrl, executionContext);
if (deleteOptions) {
await this.sqsProducer.deleteMessage(deleteOptions.queueUrl, deleteOptions.receiptHandle);
}
this.logger.info(`Message ended ${queueUrl}_${executionContext.executionTraceId}_${new Date()}`);
};
this.options = options;
this.logger = logger;
if (!this.options.awsConfig) {
throw new Error(`awsConfig is required in options when using external broker.`);
}
this.localEmitter = options.localEmitter;
this.snsProducer = new producer_sns_1.SNSProducer(this.logger, this.options.snsConfig || Object.assign({}, this.options.awsConfig));
this.sqsProducer = new producer_sqs_1.SQSProducer(this.logger, this.options.sqsConfig || Object.assign({}, this.options.awsConfig));
this.lambdaClient = new lambda_client_1.LambdaClient(this.logger, this.options.lambdaConfig || Object.assign({}, this.options.awsConfig));
this.dynamoClient = new dynamo_client_1.DynamoClient(this.logger, this.options.dynamoConfig || Object.assign({}, this.options.awsConfig));
this.addDefaultTopics();
if (this.options.outboxConfig) {
this.configureOutbox(this.options.outboxConfig);
}
}
getUniqueKeyForTopicListener(eventName, queueName) {
return `${eventName}-${queueName}`;
}
getTopicListeners(eventName, queueName) {
return this.topicListeners.get(this.getUniqueKeyForTopicListener(eventName, queueName));
}
addTopicListener(eventName, queueName, listener) {
var _a;
const listeners = (_a = this.getTopicListeners(eventName, queueName)) !== null && _a !== void 0 ? _a : [];
listeners.push(listener);
this.topicListeners.set(this.getUniqueKeyForTopicListener(eventName, queueName), listeners);
}
async bootstrap(topics) {
if (topics === null || topics === void 0 ? void 0 : topics.length) {
topics.forEach((topic) => {
this.on(topic.name, async () => { }, topic);
});
}
await this.createTopics();
await this.createQueues();
await this.tagQueues();
await this.subscribeToTopics();
await this.createEventSourceMappings();
await this.dynamoClient.createTable(constants_2.DynamoTablesStructure[types_2.DynamoTable.Idempotency]);
}
async tagQueues() {
const queues = Array.from(this.queues, ([_, value]) => {
return value;
}).filter((queue) => queue.url && queue.tags);
for (let i = 0; i < queues.length; i += constants_1.TAG_QUEUE_CHUNK_SIZE) {
const queueChunk = queues.slice(i, i + constants_1.TAG_QUEUE_CHUNK_SIZE);
const promises = [];
for (const queue of queueChunk) {
promises.push(this.sqsProducer.tagQueue(queue.url, queue.tags));
}
await Promise.all(promises);
await (0, utils_1.delay)(1000);
}
this.logger.info(`Queues tagged`);
}
async createEventSourceMappings() {
const promises = [];
const uniqueQueueMap = new Map();
this.topics.forEach((topic) => {
const queueName = this.getQueueName(topic);
if (topic.lambdaHandler && !uniqueQueueMap.has(queueName)) {
uniqueQueueMap.set(queueName, true);
const lambdaName = topic.lambdaHandler.functionName;
promises.push(this.lambdaClient.createQueueMappingForLambda({
functionName: lambdaName,
queueARN: this.getQueueArn(queueName),
batchSize: topic.batchSize || constants_1.DEFAULT_BATCH_SIZE,
maximumConcurrency: topic.lambdaHandler.maximumConcurrency,
}));
}
});
await Promise.all(promises);
if (promises.length) {
this.logger.info(`Event source mappings created`);
}
else {
this.logger.info(`No event source mappings created`);
}
}
addDefaultTopics() {
if (!this.options.defaultQueueOptions) {
this.logger.info(`No default queues specified.`);
return;
}
this.topics.set(this.options.defaultQueueOptions.fifo.name, Object.assign(Object.assign({}, this.options.defaultQueueOptions.fifo), { isDefaultQueue: true, exchangeType: types_1.ExchangeType.Queue }));
this.topics.set(this.options.defaultQueueOptions.standard.name, Object.assign(Object.assign({}, this.options.defaultQueueOptions.standard), { isDefaultQueue: true, exchangeType: types_1.ExchangeType.Queue }));
}
configureOutbox(outboxConfig) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
this.outbox = new outbox_sqns_1.Outbox(outboxConfig);
const name = (_a = outboxConfig.consumerName) !== null && _a !== void 0 ? _a : constants_1.DEFAULT_OUTBOX_TOPIC_NAME;
const fifoOptions = Object.assign(Object.assign({}, (_b = outboxConfig.consumeOptions) === null || _b === void 0 ? void 0 : _b.fifo), { name, isFifo: true, exchangeType: types_1.ExchangeType.Queue, delay: (_e = (_d = (_c = outboxConfig.consumeOptions) === null || _c === void 0 ? void 0 : _c.fifo) === null || _d === void 0 ? void 0 : _d.delay) !== null && _e !== void 0 ? _e : constants_1.DEFAULT_OUTBOX_TOPIC_DELAY });
this.topics.set(name, fifoOptions);
const nonFifoOptions = Object.assign(Object.assign({}, (_f = outboxConfig.consumeOptions) === null || _f === void 0 ? void 0 : _f.nonFifo), { name, isFifo: false, delay: (_j = (_h = (_g = outboxConfig.consumeOptions) === null || _g === void 0 ? void 0 : _g.nonFifo) === null || _h === void 0 ? void 0 : _h.delay) !== null && _j !== void 0 ? _j : constants_1.DEFAULT_OUTBOX_TOPIC_DELAY, exchangeType: types_1.ExchangeType.Queue });
this.topics.set(name, nonFifoOptions);
this.on(name, async () => { }, fifoOptions);
this.on(name, async () => { }, nonFifoOptions);
}
async createTopic(topic) {
let topicAttributes = {};
await this.snsProducer.createTopic(this.getTopicName(topic), topicAttributes);
}
async createTopics() {
const topicCreationPromises = [];
this.topics.forEach((topic) => {
if (topic.exchangeType === types_1.ExchangeType.Queue) {
return;
}
topicCreationPromises.push(this.createTopic(topic));
});
await Promise.all(topicCreationPromises);
this.logger.info(`Topics created`);
}
async createQueue(topic) {
if (topic.deadLetterQueueEnabled) {
const queueName = this.getQueueName(topic, true);
await this.sqsProducer.createQueueFromTopic({
queueName,
topic,
isDLQ: true,
queueArn: this.getQueueArn(this.getQueueName(topic)),
dlqArn: this.getQueueArn(this.getQueueName(topic, true)),
});
}
const queueName = this.getQueueName(topic);
await this.sqsProducer.createQueueFromTopic({
queueName,
topic,
isDLQ: false,
queueArn: this.getQueueArn(this.getQueueName(topic)),
dlqArn: this.getQueueArn(this.getQueueName(topic, true)),
});
}
async createQueues() {
const queueCreationPromises = [];
const queues = Array.from(this.queues, ([_, value]) => {
return value;
});
queues.forEach((queue) => {
const topic = queue.topic;
queueCreationPromises.push(this.createQueue(Object.assign(Object.assign({}, topic), { visibilityTimeout: queue.visibilityTimeout, batchSize: queue.batchSize, delay: queue.delay, retentionPeriod: queue.retentionPeriod, maxRetryCount: queue.maxRetryCount, tags: queue.tags })));
});
const responses = await Promise.allSettled(queueCreationPromises);
responses.forEach((response, index) => {
if (response.status === "rejected") {
// Checking this for localstack since it throws when queue already exists
if (response.reason.code !== "QueueAlreadyExists") {
throw new Error(`Queue creation failed: ${queues[index].name} - ${response.reason}`);
}
}
});
this.logger.info(`Queues created`);
}
getTopicArn(topicName) {
var _a, _b;
return `arn:aws:sns:${(_a = this.options.awsConfig) === null || _a === void 0 ? void 0 : _a.region}:${(_b = this.options.awsConfig) === null || _b === void 0 ? void 0 : _b.accountId}:${topicName}`;
}
getQueueArn(queueName) {
var _a, _b;
return `arn:aws:sqs:${(_a = this.options.awsConfig) === null || _a === void 0 ? void 0 : _a.region}:${(_b = this.options.awsConfig) === null || _b === void 0 ? void 0 : _b.accountId}:${queueName}`;
}
getQueueUrl(queueName) {
var _a, _b, _c, _d;
if (this.options.isLocal) {
return `${(_a = this.options.sqsConfig) === null || _a === void 0 ? void 0 : _a.endpoint}${(_b = this.options.awsConfig) === null || _b === void 0 ? void 0 : _b.accountId}/${queueName}`;
}
return `https://sqs.${(_c = this.options.awsConfig) === null || _c === void 0 ? void 0 : _c.region}.amazonaws.com/${(_d = this.options.awsConfig) === null || _d === void 0 ? void 0 : _d.accountId}/${queueName}`;
}
async subscribeToTopics() {
let subscriptionPromises = [];
const queues = Array.from(this.queues, ([_, value]) => {
return value;
});
for (const queue of queues) {
const queueTopics = queue.allTopics;
for (let i = 0; i < queueTopics.length; i += constants_1.TOPIC_SUBSCRIBE_CHUNK_SIZE) {
const chunk = queueTopics.slice(i, i + constants_1.TOPIC_SUBSCRIBE_CHUNK_SIZE);
for (const topic of chunk) {
if (topic.exchangeType === types_1.ExchangeType.Queue) {
continue;
}
const queueArn = this.getQueueArn(this.getQueueName(topic));
const topicArn = this.getTopicArn(this.getTopicName(topic));
subscriptionPromises.push(this.snsProducer.subscribeToTopic(topicArn, queueArn, topic.filterPolicy, true));
}
await Promise.all(subscriptionPromises);
await (0, utils_1.delay)(1000);
subscriptionPromises = [];
}
}
this.logger.info(`Topic subscription complete`);
}
async emitToTopic(topic, options, payload) {
const topicArn = this.getTopicArn(this.getTopicName(topic));
return await this.snsProducer.send(topicArn, {
messageGroupId: (options === null || options === void 0 ? void 0 : options.partitionKey) || topic.name,
eventName: topic.name,
messageAttributes: options === null || options === void 0 ? void 0 : options.MessageAttributes,
deduplicationId: options === null || options === void 0 ? void 0 : options.deduplicationId,
/**
* @todo Un-array this when switching to payload version 2
*/
data: [payload],
});
}
async emitToQueue(topic, options, payload) {
const queueUrl = this.getQueueUrl(this.getQueueName(topic));
return await this.sqsProducer.send(queueUrl, {
messageGroupId: (options === null || options === void 0 ? void 0 : options.partitionKey) || topic.name,
eventName: topic.name,
/**
* @todo Un-array this when switching to payload version 2
*/
data: [payload],
messageAttributes: options === null || options === void 0 ? void 0 : options.MessageAttributes,
deduplicationId: options === null || options === void 0 ? void 0 : options.deduplicationId,
}, {
delay: (options === null || options === void 0 ? void 0 : options.delay) || constants_1.DEFAULT_MESSAGE_DELAY,
});
}
getEmitPayload(eventName, options, payload) {
const topic = {
name: eventName,
isFifo: !!(options === null || options === void 0 ? void 0 : options.isFifo),
exchangeType: (options === null || options === void 0 ? void 0 : options.exchangeType) || types_1.ExchangeType.Fanout,
separateConsumerGroup: options === null || options === void 0 ? void 0 : options.consumerGroup,
};
const message = {
messageGroupId: (options === null || options === void 0 ? void 0 : options.partitionKey) || topic.name,
eventName: topic.name,
/**
* @todo Un-array this when switching to payload version 2
*/
data: [payload],
messageAttributes: options === null || options === void 0 ? void 0 : options.MessageAttributes,
deduplicationId: options === null || options === void 0 ? void 0 : options.deduplicationId,
};
if (topic.exchangeType === types_1.ExchangeType.Queue) {
const queueUrl = this.getQueueUrl(this.getQueueName(topic));
return this.sqsProducer.getSendMessageRequestInput(queueUrl, message, {
delay: (options === null || options === void 0 ? void 0 : options.delay) || constants_1.DEFAULT_MESSAGE_DELAY,
});
}
else {
const topicArn = this.getTopicArn(this.getTopicName(topic));
return this.snsProducer.getPublishInput(topicArn, message);
}
}
async emit(eventName, options, payload) {
await this.internalEmit(eventName, options, payload);
}
async internalEmit(eventName, options, payload) {
var _a, _b, _c, _d;
let modifiedArgs;
try {
if (options === null || options === void 0 ? void 0 : options.outboxData) {
return await this.saveEventToOutbox(eventName, options, payload);
}
const topic = {
name: eventName,
isFifo: !!(options === null || options === void 0 ? void 0 : options.isFifo),
exchangeType: (options === null || options === void 0 ? void 0 : options.exchangeType) || types_1.ExchangeType.Fanout,
separateConsumerGroup: options === null || options === void 0 ? void 0 : options.consumerGroup,
};
modifiedArgs =
(await ((_b = (_a = this.options.hooks) === null || _a === void 0 ? void 0 : _a.beforeEmit) === null || _b === void 0 ? void 0 : _b.call(_a, eventName, payload))) || payload;
let response;
if (topic.exchangeType === types_1.ExchangeType.Queue) {
response = await this.emitToQueue(topic, options, modifiedArgs);
}
else {
response = await this.emitToTopic(topic, options, modifiedArgs);
}
await ((_d = (_c = this.options.hooks) === null || _c === void 0 ? void 0 : _c.afterEmit) === null || _d === void 0 ? void 0 : _d.call(_c, eventName, modifiedArgs));
return response;
}
catch (error) {
this.logger.error(`Message producing failed:
Event Name: ${eventName}
Payload: ${payload ? JSON.stringify(payload) : undefined}
Error ${JSON.stringify(error)}`);
this.logFailedEvent({
failureType: types_1.FailedEventCategory.MessageProducingFailed,
topic: eventName,
event: modifiedArgs !== null && modifiedArgs !== void 0 ? modifiedArgs : payload,
error: error,
});
throw error;
}
}
async emitBatchToTopic(topic, messages) {
var _a;
const topicArn = this.getTopicArn(this.getTopicName(topic));
const result = await this.snsProducer.sendBatch(topicArn, this.getBatchMessagesForTopic(topic.name, messages));
return (((_a = result.Failed) === null || _a === void 0 ? void 0 : _a.map((failed) => ({
id: failed.Id,
code: failed.Code,
message: failed.Message,
wasSenderFault: failed.SenderFault,
}))) || []);
}
async emitBatchToQueue(topic, messages) {
var _a;
const queueUrl = this.getQueueUrl(this.getQueueName(topic));
const result = await this.sqsProducer.sendBatch(queueUrl, this.getBatchMessagesForQueue(topic.name, messages));
return (((_a = result.Failed) === null || _a === void 0 ? void 0 : _a.map((failed) => ({
id: failed.Id,
code: failed.Code,
message: failed.Message,
wasSenderFault: failed.SenderFault,
}))) || []);
}
getBatchEmitPayload(eventName, messages, options) {
const topic = {
name: eventName,
isFifo: !!(options === null || options === void 0 ? void 0 : options.isFifo),
exchangeType: (options === null || options === void 0 ? void 0 : options.exchangeType) || types_1.ExchangeType.Fanout,
separateConsumerGroup: options === null || options === void 0 ? void 0 : options.consumerGroup,
};
if (topic.exchangeType === types_1.ExchangeType.Queue) {
const queueUrl = this.getQueueUrl(this.getQueueName(topic));
return this.sqsProducer.getBatchMessageRequest(queueUrl, this.getBatchMessagesForQueue(topic.name, messages));
}
else {
const topicArn = this.getTopicArn(this.getTopicName(topic));
return this.snsProducer.getBatchPublishInput(topicArn, this.getBatchMessagesForTopic(topic.name, messages));
}
}
async emitBatch(eventName, messages, options) {
try {
if (options === null || options === void 0 ? void 0 : options.outboxData) {
await this.saveEventToOutbox(eventName, options, messages, true);
return [];
}
const topic = {
name: eventName,
isFifo: !!(options === null || options === void 0 ? void 0 : options.isFifo),
exchangeType: (options === null || options === void 0 ? void 0 : options.exchangeType) || types_1.ExchangeType.Fanout,
separateConsumerGroup: options === null || options === void 0 ? void 0 : options.consumerGroup,
};
if (topic.exchangeType === types_1.ExchangeType.Queue) {
return await this.emitBatchToQueue(topic, messages);
}
return await this.emitBatchToTopic(topic, messages);
}
catch (error) {
this.logger.error(`Batch Message producing failed: ${eventName} ${JSON.stringify(error)}`);
throw error;
}
}
async startConsumers() {
if (this.consumersStarted) {
return;
}
this.queues.forEach((queue) => {
if (!queue || queue.isDLQ || queue.listenerIsLambda) {
return;
}
this.startConsumer(queue);
});
this.consumersStarted = true;
this.logger.info(`Consumers started`);
}
addConsumerWorkerToQueue(queue) {
var _a, _b;
if (!queue.url) {
return;
}
const consumer = sqs_consumer_1.Consumer.create({
/**
* Handling delete message explcitly because sqs-consumer
* does not delete the successful ones if one of the message
* in the batch throws
*/
shouldDeleteMessages: false,
sqs: this.sqsProducer.client,
region: (_a = this.options.awsConfig) === null || _a === void 0 ? void 0 : _a.region,
queueUrl: queue.url,
messageAttributeNames: ["All"],
attributeNames: ["ApproximateReceiveCount"],
handleMessageBatch: async (messages) => {
await this.processMessages(messages, {
shouldDeleteMessage: true,
queueReference: queue.url,
});
},
batchSize: queue.batchSize || constants_1.DEFAULT_BATCH_SIZE,
visibilityTimeout: queue.visibilityTimeout || constants_1.DEFAULT_VISIBILITY_TIMEOUT,
});
consumer.on("error", (error, message) => {
this.logger.error(`Queue error ${queue.topic.name} ${queue.url} ${JSON.stringify(error)}`);
this.logFailedEvent({
failureType: types_1.FailedEventCategory.QueueError,
topic: queue.topic.name,
topicReference: queue.url,
event: message,
error,
});
});
consumer.on("processing_error", (error, message) => {
this.logger.error(`Queue Processing error ${JSON.stringify(error)}`);
this.logFailedEvent({
failureType: types_1.FailedEventCategory.QueueProcessingError,
topic: "",
event: message,
error,
});
});
consumer.on("stopped", () => {
this.logger.error("Queue stopped");
this.logFailedEvent({
failureType: types_1.FailedEventCategory.QueueStopped,
topic: "",
event: "Queue stopped",
});
});
consumer.on("timeout_error", () => {
this.logger.error("Queue timed out");
this.logFailedEvent({
failureType: types_1.FailedEventCategory.QueueTimedOut,
topic: "",
event: "Queue timed out",
});
});
consumer.on("empty", () => {
if (!(consumer === null || consumer === void 0 ? void 0 : consumer.isRunning)) {
this.logger.info(`Queue not running`);
}
});
consumer.start();
if (!((_b = queue.consumers) === null || _b === void 0 ? void 0 : _b.length)) {
queue.consumers = [];
}
queue.consumers.push(consumer);
}
startConsumer(queue) {
var _a;
for (let i = 0; i < ((_a = queue.workers) !== null && _a !== void 0 ? _a : 1); i++) {
this.addConsumerWorkerToQueue(queue);
}
}
removeListener(eventName, listener, consumeOptions) {
const topic = this.getTopicFromEventNameAndConsumeOptions(eventName, consumeOptions);
const queueName = this.getQueueName(topic);
this.topicListeners.delete(this.getUniqueKeyForTopicListener(eventName, queueName));
}
removeAllListener() {
this.topicListeners.clear();
}
getTopicFromEventNameAndConsumeOptions(eventName, options) {
return Object.assign(Object.assign({}, options), { name: eventName, exchangeType: (options === null || options === void 0 ? void 0 : options.exchangeType) || types_1.ExchangeType.Fanout });
}
on(eventName, listener, options) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l;
const topic = this.getTopicFromEventNameAndConsumeOptions(eventName, options);
const queueName = this.getQueueName(topic);
this.addTopicListener(eventName, queueName, listener);
this.topics.set(eventName, topic);
if (!this.queues.has(queueName)) {
if (!this.getSeparateConsumer(topic) &&
topic.exchangeType === types_1.ExchangeType.Fanout &&
!this.options.defaultQueueOptions) {
this.logger.warn(`No consumer specified for fanout topic ${topic.name}`);
return;
}
const queue = {
name: topic.exchangeType === types_1.ExchangeType.Queue
? topic.name
: this.getSeparateConsumer(topic) ||
(topic.isFifo
? (_a = this.options.defaultQueueOptions) === null || _a === void 0 ? void 0 : _a.fifo.name
: (_b = this.options.defaultQueueOptions) === null || _b === void 0 ? void 0 : _b.standard.name) ||
"",
isFifo: this.isConsumerFifo(topic),
batchSize: ((_c = topic.consumerGroup) === null || _c === void 0 ? void 0 : _c.batchSize) ||
topic.batchSize ||
constants_1.DEFAULT_BATCH_SIZE,
visibilityTimeout: ((_d = topic.consumerGroup) === null || _d === void 0 ? void 0 : _d.visibilityTimeout) ||
topic.visibilityTimeout ||
constants_1.DEFAULT_VISIBILITY_TIMEOUT,
delay: ((_e = topic.consumerGroup) === null || _e === void 0 ? void 0 : _e.delay) || topic.delay,
maxRetryCount: ((_f = topic.consumerGroup) === null || _f === void 0 ? void 0 : _f.maxRetryCount) || topic.maxRetryCount,
retentionPeriod: ((_g = topic.consumerGroup) === null || _g === void 0 ? void 0 : _g.retentionPeriod) || topic.retentionPeriod,
tags: ((_h = topic.consumerGroup) === null || _h === void 0 ? void 0 : _h.tags) || topic.tags,
url: this.getQueueUrl(queueName),
arn: this.getQueueArn(this.getQueueName(topic)),
isDLQ: false,
listenerIsLambda: !!topic.lambdaHandler,
topic,
allTopics: [topic],
workers: ((_j = topic.consumerGroup) === null || _j === void 0 ? void 0 : _j.workers) || topic.workers,
consumerIdempotencyOptions: ((_k = topic.consumerGroup) === null || _k === void 0 ? void 0 : _k.consumerIdempotencyOptions) ||
topic.consumerIdempotencyOptions ||
this.options.consumerIdempotencyOptions,
};
this.queues.set(queueName, queue);
}
else {
(_l = this.queues.get(queueName)) === null || _l === void 0 ? void 0 : _l.allTopics.push(topic);
}
}
getSeparateConsumer(topic) {
const { separateConsumerGroup, consumerGroup } = topic;
if (separateConsumerGroup && consumerGroup)
throw new Error(`separateConsumerGroup and consumerGroup cannot be used together`);
return separateConsumerGroup || (consumerGroup === null || consumerGroup === void 0 ? void 0 : consumerGroup.name);
}
isConsumerFifo(topic) {
if (topic.consumerGroup !== undefined)
return !!topic.consumerGroup.isFifo;
return !!topic.isFifo;
}
async onMessageReceived(receivedMessage, queueUrl, executionContext) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
let message;
try {
message = this.parseDataFromMessage(receivedMessage);
}
catch (error) {
this.logger.error(`Failed to parse message. Trace Id: ${executionContext.executionTraceId}`);
this.logFailedEvent({
failureType: types_1.FailedEventCategory.IncomingMessageFailedToParse,
topicReference: queueUrl,
event: receivedMessage.Body,
error: `Failed to parse message`,
executionContext,
});
throw new Error(`Failed to parse message`);
}
const payloadStructureVersion = ((_b = (_a = message.messageAttributes) === null || _a === void 0 ? void 0 : _a.PayloadVersion) === null || _b === void 0 ? void 0 : _b.StringValue) ||
((_d = (_c = message.messageAttributes) === null || _c === void 0 ? void 0 : _c.PayloadVersion) === null || _d === void 0 ? void 0 : _d.stringValue);
if (payloadStructureVersion !== constants_1.PAYLOAD_STRUCTURE_VERSION_V2) {
message.data = message.data[0];
}
if (this.options.outboxConfig &&
message.eventName === this.getOutboxTopicName()) {
await this.handleOutboxEvent(message.data);
return;
}
const listeners = this.getTopicListeners(message.eventName, this.getQueueNameFromUrl(queueUrl));
if (!listeners) {
this.logger.error(`No listener found. Trace Id: ${executionContext.executionTraceId}. Message: ${JSON.stringify(message)}`);
this.logFailedEvent({
failureType: types_1.FailedEventCategory.NoListenerFound,
topic: message.eventName,
event: message,
error: `No listener found`,
executionContext,
});
throw new Error(`No listener found for event: ${message.eventName}`);
}
const metadata = {
executionContext,
messageId: message.id,
messageAttributes: message.messageAttributes,
approximateReceiveCount: this.getApproximateReceiveCount(receivedMessage),
};
const queue = this.queues.get(this.getQueueNameFromUrl(queueUrl));
let consumerDeduplicationKey;
let isAlreadyProcessed = false;
if (queue) {
consumerDeduplicationKey = this.getDeduplicationKey(queue, message, metadata);
}
if (consumerDeduplicationKey) {
isAlreadyProcessed = await this.isMessageAlreadyProcessed(consumerDeduplicationKey, queueUrl, executionContext.executionTraceId);
}
if (isAlreadyProcessed) {
return;
}
try {
const data = await ((_f = (_e = this.options.hooks) === null || _e === void 0 ? void 0 : _e.beforeConsume) === null || _f === void 0 ? void 0 : _f.call(_e, message.eventName, message.data));
for (const listener of listeners) {
await listener(data || message.data, metadata);
}
await ((_h = (_g = this.options.hooks) === null || _g === void 0 ? void 0 : _g.afterConsume) === null || _h === void 0 ? void 0 : _h.call(_g, message.eventName, message.data));
if (consumerDeduplicationKey) {
await this.dynamoClient.putItem({
TableName: types_2.DynamoTable.Idempotency,
Item: {
partitionKey: { S: consumerDeduplicationKey },
},
}, (_j = queue === null || queue === void 0 ? void 0 : queue.consumerIdempotencyOptions) === null || _j === void 0 ? void 0 : _j.expiry);
}
}
catch (error) {
this.logFailedEvent({
failureType: types_1.FailedEventCategory.MessageProcessingFailed,
topic: message.eventName,
event: message,
error: error,
executionContext,
});
// Doing this because i don't want to mess with stack trace of rethrowing error
error["executionTraceId"] = executionContext.executionTraceId;
throw error;
}
}
getDeduplicationKey(queue, message, metadata) {
var _a;
let consumerDeduplicationKey;
if (!queue || !queue.consumerIdempotencyOptions) {
return;
}
const idempotency = queue.consumerIdempotencyOptions;
if (idempotency.strategy === types_1.ConsumerIdempotencyStrategy.DeduplicationId &&
message.deduplicationId) {
consumerDeduplicationKey = `${message.eventName}-${queue.url}-${message.deduplicationId}`;
}
else if (idempotency.strategy === types_1.ConsumerIdempotencyStrategy.PayloadHash) {
consumerDeduplicationKey = `${message.eventName}-${queue.url}-${(0, crypto_1.createHash)("sha256")
.update(JSON.stringify(message.data))
.digest("hex")}`;
}
else if (idempotency.strategy === types_1.ConsumerIdempotencyStrategy.Custom) {
consumerDeduplicationKey = `${message.eventName}-${queue.url}-${(_a = idempotency.key) === null || _a === void 0 ? void 0 : _a.call(idempotency, message.data, metadata)}`;
}
return consumerDeduplicationKey;
}
async isMessageAlreadyProcessed(deduplicationKey, queueUrl, executionTraceId) {
var _a;
const item = await this.dynamoClient.getItem({
TableName: types_2.DynamoTable.Idempotency,
Key: {
partitionKey: { S: deduplicationKey },
},
});
if (item && (!((_a = item.expiresAt) === null || _a === void 0 ? void 0 : _a.N) || +item.expiresAt.N * 1000 > Date.now())) {
this.logger.info(`Message already processed. ${queueUrl}_${executionTraceId}_Consumer Deduplication ID: ${deduplicationKey}}`);
return true;
}
return false;
}
async handleOutboxEvent(data) {
if (!this.outbox) {
throw new Error("Outbox config is not configured");
}
const outboxEvents = (await this.outbox.getOutboxEvents(data.ids)) || [];
if (data.isBatch) {
const outboxBatchPromises = outboxEvents.map((event) => this.emitBatch(event.topicName, event.payload, event.options));
const batchMessages = await Promise.allSettled(outboxBatchPromises);
for (let i = 0; i < batchMessages.length; i++) {
const message = batchMessages[i];
const value = message
.value || [];
const reason = message.reason;
outboxEvents[i] = this.outbox.handleBatchEvent(outboxEvents[i], value, reason);
}
}
else {
const outboxPromises = outboxEvents.map((event) => this.internalEmit(event.topicName, event.options, event.payload));
const messages = await Promise.allSettled(outboxPromises);
for (let i = 0; i < messages.length; i++) {
const message = messages[i];
const reason = message.reason;
outboxEvents[i] = this.outbox.handleEvent(outboxEvents[i], reason);
}
}
await this.outbox.updateEvents(outboxEvents);
}
parseDataFromMessage(receivedMessage) {
let snsMessage;
let message;
const body = receivedMessage.Body || receivedMessage.body;
snsMessage = JSON.parse(body.toString());
message = snsMessage;
message.messageAttributes = receivedMessage.MessageAttributes;
if (snsMessage.TopicArn) {
message = JSON.parse(snsMessage.Message);
message.messageAttributes = this.mapMessageAttributesFromSNS(snsMessage.MessageAttributes);
}
return message;
}
mapMessageAttributesFromSNS(attributes) {
const messageAttributes = {};
Object.keys(attributes).forEach((key) => {
const { Type, Value } = attributes[key] || {};
const valueKey = Type === "Binary"
? "BinaryValue"
: Type === "String"
? "StringValue"
: "Value";
const typeKey = valueKey === "Value" ? "Type" : "DataType";
messageAttributes[key] = {
[typeKey]: Type,
[valueKey]: Value,
};
});
return messageAttributes;
}
async deleteMessages(queueUrl, messages, results) {
const receiptsToDelete = [];
results.forEach((result, index) => {
if (result.status === "fulfilled") {
receiptsToDelete.push(messages[index].ReceiptHandle);
}
});
if (receiptsToDelete.length) {
await this.sqsProducer.deleteMessages(queueUrl, receiptsToDelete);
}
}
async processFifoQueueMessages(queueUrl, messages, options) {
var _a;
let i = 0;
try {
for (i = 0; i < messages.length; i++) {
await this.processMessage(messages[i], options);
}
return {
batchItemFailures: [],
};
}
catch (error) {
this.logger.error(`Fifo queue message failed :: ${queueUrl} Execution Trace ID ${(_a = error["executionTraceId"]) !== null && _a !== void 0 ? _a : ""} :: ${JSON.stringify(messages[i])}`);
return {
batchItemFailures: messages.slice(i, undefined).map((message) => {
return {
itemIdentifier: this.getMessageIdFromMessage(message),
};
}),
};
}
}
async processStandardQueueMessages(queueUrl, messages, options) {
const results = await Promise.allSettled(messages.map((message) => this.processMessage(message, {
queueReference: options === null || options === void 0 ? void 0 : options.queueReference,
})));
if (options === null || options === void 0 ? void 0 : options.shouldDeleteMessage) {
await this.deleteMessages(queueUrl, messages, results);
}
const failedMessages = [];
results.forEach((result, index) => {
if (result.status === "rejected") {
failedMessages.push(messages[index]);
}
});
return {
batchItemFailures: failedMessages.map((message) => {
return {
itemIdentifier: this.getMessageIdFromMessage(message),
};
}),
};
}
async processMessages(messages, options) {
const queueUrl = (options === null || options === void 0 ? void 0 : options.queueReference) || this.getQueueUrlFromMessage(messages[0]);
const isFifoQueue = this.sqsProducer.isFifoQueue(queueUrl);
if (isFifoQueue) {
return await this.processFifoQueueMessages(queueUrl, messages, options);
}
return await this.processStandardQueueMessages(queueUrl, messages, options);
}
async processMessage(message, options) {
/**
* The lambda interface provides keys with camel case
* but the SQS.Message type has Pascal case
*/
if (!message.Body) {
message.Body = message.body;
}
if (!message.ReceiptHandle) {
message.ReceiptHandle = message.receiptHandle;
}
if (!message.MessageAttributes) {
message.MessageAttributes = message.messageAttributes;
}
const queueUrl = (options === null || options === void 0 ? void 0 : options.queueReference) || this.getQueueUrlFromMessage(message);
let deleteOptions;
if (options === null || options === void 0 ? void 0 : options.shouldDeleteMessage) {
deleteOptions = {
queueUrl,
receiptHandle: message.ReceiptHandle,
};
}
return await this.handleMessageReceipt(message, queueUrl, deleteOptions);
}
getTopicReference(topic) {
return this.getTopicArn(this.getTopicName(topic)) || "";
}
getInternalTopicName(topic) {
return this.getTopicName(topic) || "";
}
getQueues() {
const queues = [];
this.queues.forEach((queue) => {
queues.push(queue);
});
return queues;
}
getQueueReference(topic) {
return this.getQueueArn(this.getQueueName(topic));
}
getInternalQueueName(topic) {
return this.getQueueName(topic);
}
getQueueUrlFromMessage(message) {
const receivedMessage = message;
const queueUrl = this.getQueueUrlFromARN(receivedMessage.eventSourceARN);
if (!queueUrl) {
throw new Error(`QueueUrl or eventSourceARN not found in the message`);
}
return queueUrl;
}
getQueueUrlFromARN(arn) {
var _a, _b;
if (!arn)
return;
const parts = arn.split(":");
const service = parts[2];
const region = parts[3];
const accountId = parts[4];
const queueName = parts[5];
if (this.options.isLocal) {
return `${(_a = this.options.sqsConfig) === null || _a === void 0 ? void 0 : _a.endpoint}${(_b = this.options.awsConfig) === null || _b === void 0 ? void 0 : _b.accountId}/${queueName}`;
}
return `https://${service}.${region}.amazonaws.com/${accountId}/${queueName}`;
}
getMessageIdFromMessage(message) {
const messageId = message.MessageId || message.messageId;
return messageId;
}
getQueueNameFromUrl(queueUrl) {
const urlParts = queueUrl.split("/");
return urlParts[urlParts.length - 1];
}
async saveEventToOutbox(eventName, options, payload, isBatch = false) {
if (!this.outbox) {
throw new Error("Outbox is not configured");
}
const outboxEvent = await this.outbox.createEvent(eventName, options, payload, isBatch);
const outboxTopicName = this.getOutboxTopicName();
const emitOptionsForOutbox = this.getOutboxEmitOptions(options);
const outboxEventPayload = {
ids: [outboxEvent.id],
isFifo: options.isFifo,
isBatch,
};
return await this.internalEmit(outboxTopicName, emitOptionsForOutbox, outboxEventPayload);
}
getOutboxTopicName() {
if (this.options.outboxConfig.consumerName) {
return `${this.options.outboxConfig.consumerName}`;
}
return `${this.options.environment}_${this.options.serviceName}_${constants_1.DEFAULT_OUTBOX_TOPIC_NAME}`;
}
getOutboxEmitOptions(options) {
var _a, _b, _c, _d;
const { outboxData } = options, emitOptions = __rest(options, ["outboxData"]);
const delay = options.isFifo
? undefined
: (_a = options.delay) !== null && _a !== void 0 ? _a : (_d = (_c = (_b = this.options.outboxConfig) === null || _b === void 0 ? void 0 : _b.consumeOptions) === null || _c === void 0 ? void 0 : _c.nonFifo) === null || _d === void 0 ? void 0 : _d.delay;
return Object.assign(Object.assign({}, emitOptions), { exchangeType: types_1.ExchangeType.Queue, delay });
}
getApproximateReceiveCount(receivedMessage) {
var _a;
return (((_a = receivedMessage.Attributes) === null || _a === void 0 ? void 0 : _a.ApproximateReceiveCount) ||
receivedMessage.attributes.ApproximateReceiveCount);
}
}
exports.SqnsEmitter = SqnsEmitter;
//# sourceMappingURL=emitter.sqns.js.map