redis-smq
Version:
A simple high-performance Redis message queue for Node.js.
318 lines • 16 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Producer = void 0;
const redis_smq_common_1 = require("redis-smq-common");
const redis_client_js_1 = require("../../common/redis-client/redis-client.js");
const scripts_js_1 = require("../../common/redis-client/scripts/scripts.js");
const redis_keys_js_1 = require("../../common/redis-keys/redis-keys.js");
const index_js_1 = require("../../config/index.js");
const index_js_2 = require("../../common/index.js");
const _get_exchange_queues_js_1 = require("../exchange/_/_get-exchange-queues.js");
const index_js_3 = require("../exchange/index.js");
const index_js_4 = require("../message/index.js");
const message_envelope_js_1 = require("../message/message-envelope.js");
const index_js_5 = require("../queue/index.js");
const _schedule_message_js_1 = require("./_/_schedule-message.js");
const index_js_6 = require("./errors/index.js");
const event_bus_publisher_js_1 = require("./event-bus-publisher.js");
const queue_consumer_groups_cache_js_1 = require("./queue-consumer-groups-cache.js");
class Producer extends redis_smq_common_1.Runnable {
constructor() {
super();
this.queueConsumerGroupsHandler = null;
this.initQueueConsumerGroupsHandler = (cb) => {
this.logger.debug('Initializing queue consumer groups handler');
this.queueConsumerGroupsHandler = new queue_consumer_groups_cache_js_1.QueueConsumerGroupsCache(this, this.redisClient, this.eventBus);
this.queueConsumerGroupsHandler.run((err) => {
if (err) {
this.logger.error('Failed to initialize queue consumer groups handler', err);
}
else {
this.logger.debug('Queue consumer groups handler initialized successfully');
}
cb(err);
});
};
this.shutDownQueueConsumerGroupsHandler = (cb) => {
if (this.queueConsumerGroupsHandler) {
this.logger.debug('Shutting down queue consumer groups handler');
this.queueConsumerGroupsHandler.shutdown(() => {
this.logger.debug('Queue consumer groups handler shut down successfully');
this.queueConsumerGroupsHandler = null;
cb();
});
}
else {
this.logger.debug('No queue consumer groups handler to shut down');
cb();
}
};
this.redisClient = new redis_client_js_1.RedisClient();
this.redisClient.on('error', (err) => this.handleError(err));
this.eventBus = new index_js_2.EventBus();
this.eventBus.on('error', (err) => this.handleError(err));
this.logger = redis_smq_common_1.logger.getLogger(index_js_1.Configuration.getSetConfig().logger, this.constructor.name.toLowerCase());
this.logger.info(`Producer instance created with ID: ${this.getId()}`);
if (index_js_1.Configuration.getSetConfig().eventBus.enabled) {
this.logger.debug('Event bus is enabled, initializing event bus publisher');
(0, event_bus_publisher_js_1.eventBusPublisher)(this, this.eventBus, this.logger);
}
else {
this.logger.debug('Event bus is disabled, skipping event bus publisher initialization');
}
}
getLogger() {
return this.logger;
}
goingUp() {
this.logger.info(`Producer ${this.getId()} is starting up`);
return super.goingUp().concat([
this.redisClient.init,
this.eventBus.init,
(cb) => {
this.logger.debug(`Emitting producer.goingUp event for producer ${this.id}`);
this.emit('producer.goingUp', this.id);
cb();
},
this.initQueueConsumerGroupsHandler,
]);
}
up(cb) {
super.up(() => {
this.logger.info(`Producer ${this.getId()} is now up and running`);
this.emit('producer.up', this.id);
cb(null, true);
});
}
goingDown() {
this.logger.info(`Producer ${this.getId()} is shutting down`);
this.emit('producer.goingDown', this.id);
return [
this.shutDownQueueConsumerGroupsHandler,
this.redisClient.shutdown,
].concat(super.goingDown());
}
down(cb) {
super.down(() => {
this.logger.info(`Producer ${this.getId()} is now down`);
this.emit('producer.down', this.id);
this.logger.debug('Shutting down event bus with 1 second delay');
setTimeout(() => {
this.eventBus.shutdown(() => {
this.logger.debug('Event bus shut down successfully');
cb(null, true);
});
}, 1000);
});
}
getQueueConsumerGroupsHandler() {
if (!this.queueConsumerGroupsHandler) {
const error = new redis_smq_common_1.PanicError(`Expected an instance of QueueConsumerGroupsHandler`);
this.logger.error('Queue consumer groups handler not initialized', error);
throw error;
}
return this.queueConsumerGroupsHandler;
}
enqueue(redisClient, message, cb) {
const messageState = message.getMessageState();
messageState.setPublishedAt(Date.now());
const messageId = message.getId();
const destinationQueue = message.getDestinationQueue();
const queueName = `${destinationQueue.name}@${destinationQueue.ns}`;
const consumerGroupId = message.getConsumerGroupId();
this.logger.debug(`Enqueuing message ${messageId} to queue ${queueName}${consumerGroupId ? ` for consumer group ${consumerGroupId}` : ''}`);
const keys = redis_keys_js_1.redisKeys.getQueueKeys(destinationQueue, consumerGroupId);
const { keyMessage } = redis_keys_js_1.redisKeys.getMessageKeys(messageId);
const priority = message.producibleMessage.getPriority();
this.logger.debug(`Message ${messageId} details: priority=${priority !== null && priority !== void 0 ? priority : 'none'}, queue=${queueName}`);
const scriptArgs = [
index_js_5.EQueueProperty.QUEUE_TYPE,
index_js_5.EQueueProperty.MESSAGES_COUNT,
index_js_5.EQueueType.PRIORITY_QUEUE,
index_js_5.EQueueType.LIFO_QUEUE,
index_js_5.EQueueType.FIFO_QUEUE,
priority !== null && priority !== void 0 ? priority : '',
messageId,
index_js_4.EMessageProperty.STATUS,
index_js_4.EMessagePropertyStatus.PENDING,
index_js_4.EMessageProperty.STATE,
JSON.stringify(messageState),
index_js_4.EMessageProperty.MESSAGE,
JSON.stringify(message.toJSON()),
];
redisClient.runScript(scripts_js_1.ELuaScriptName.PUBLISH_MESSAGE, [
keys.keyQueueProperties,
keys.keyQueuePriorityPending,
keys.keyQueuePending,
keys.keyQueueMessages,
keyMessage,
], scriptArgs, (err, reply) => {
if (err) {
this.logger.error(`Failed to enqueue message ${messageId}`, err);
return cb(err);
}
switch (reply) {
case 'OK':
this.logger.debug(`Successfully enqueued message ${messageId} to queue ${queueName}`);
return cb();
case 'QUEUE_NOT_FOUND':
this.logger.error(`Queue ${queueName} not found for message ${messageId}`);
return cb(new index_js_6.ProducerQueueNotFoundError());
case 'MESSAGE_PRIORITY_REQUIRED':
this.logger.error(`Priority required for message ${messageId} but not provided`);
return cb(new index_js_6.ProducerMessagePriorityRequiredError());
case 'PRIORITY_QUEUING_NOT_ENABLED':
this.logger.error(`Priority queuing not enabled for queue ${queueName}`);
return cb(new index_js_6.ProducerPriorityQueuingNotEnabledError());
case 'UNKNOWN_QUEUE_TYPE':
this.logger.error(`Unknown queue type for queue ${queueName}`);
return cb(new index_js_6.ProducerUnknownQueueTypeError());
default:
this.logger.error(`Unknown error while enqueuing message ${messageId}: ${reply}`);
return cb(new index_js_6.ProducerError());
}
});
}
produceMessageItem(redisClient, message, queue, cb) {
const messageId = message
.setDestinationQueue(queue)
.getMessageState()
.getId();
const queueName = `${queue.name}@${queue.ns}`;
this.logger.debug(`Producing message item ${messageId} for queue ${queueName}${message.isSchedulable() ? ' (scheduled)' : ''}`);
const handleResult = (err) => {
if (err) {
this.logger.error(`Failed to produce message ${messageId} for queue ${queueName}`, err);
cb(err);
}
else {
const action = message.isSchedulable() ? 'scheduled' : 'published';
this.logger.info(`Message (ID ${messageId}) has been ${action} to queue ${queueName}`);
if (!message.isSchedulable()) {
this.logger.debug(`Emitting messagePublished event for message ${messageId}`);
this.emit('producer.messagePublished', messageId, { queueParams: queue, groupId: message.getConsumerGroupId() }, this.id);
}
cb(null, messageId);
}
};
if (message.isSchedulable()) {
this.logger.debug(`Scheduling message ${messageId} for future delivery`);
(0, _schedule_message_js_1._scheduleMessage)(redisClient, message, handleResult);
}
else {
this.logger.debug(`Enqueueing message ${messageId} for immediate delivery`);
this.enqueue(redisClient, message, handleResult);
}
}
produceMessage(redisClient, message, queue, cb) {
const queueName = `${queue.name}@${queue.ns}`;
this.logger.debug(`Producing message for queue ${queueName}`);
const { exists, consumerGroups } = this.getQueueConsumerGroupsHandler().getConsumerGroups(queue);
if (exists) {
this.logger.debug(`Queue ${queueName} exists with ${consumerGroups.length} consumer groups`);
if (!consumerGroups.length) {
this.logger.error(`Queue ${queueName} has no consumer groups`);
cb(new index_js_6.ProducerQueueMissingConsumerGroupsError());
return;
}
const ids = [];
this.logger.debug(`Producing message for ${consumerGroups.length} consumer groups in queue ${queueName}`);
redis_smq_common_1.async.eachOf(consumerGroups, (group, _, done) => {
this.logger.debug(`Producing message for consumer group ${group} in queue ${queueName}`);
const msg = new message_envelope_js_1.MessageEnvelope(message).setConsumerGroupId(group);
this.produceMessageItem(redisClient, msg, queue, (err, reply) => {
if (err) {
this.logger.error(`Failed to produce message for consumer group ${group}`, err);
done(err);
}
else {
this.logger.debug(`Successfully produced message ${reply} for consumer group ${group}`);
ids.push(String(reply));
done();
}
});
}, (err) => {
if (err) {
this.logger.error(`Failed to produce messages for some consumer groups in queue ${queueName}`, err);
cb(err);
}
else {
this.logger.info(`Successfully produced ${ids.length} messages for queue ${queueName}`);
cb(null, ids);
}
});
}
else {
this.logger.debug(`Queue ${queueName} has no consumer groups, producing message directly`);
const msg = new message_envelope_js_1.MessageEnvelope(message);
this.produceMessageItem(redisClient, msg, queue, (err, reply) => {
if (err) {
this.logger.error(`Failed to produce message for queue ${queueName}`, err);
cb(err);
}
else {
this.logger.info(`Successfully produced message ${reply} for queue ${queueName}`);
cb(null, [String(reply)]);
}
});
}
}
produce(msg, cb) {
if (!this.isUp()) {
this.logger.error('Cannot produce message: Producer instance is not running');
return cb(new index_js_6.ProducerInstanceNotRunningError());
}
const exchangeParams = msg.getExchange();
if (!exchangeParams) {
this.logger.error('Cannot produce message: No exchange parameters provided');
return cb(new index_js_6.ProducerMessageExchangeRequiredError());
}
this.logger.debug(`Producing message with exchange type ${index_js_3.EExchangeType[exchangeParams.type]}`);
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error) {
this.logger.error('Cannot produce message: Redis client error', redisClient);
return cb(redisClient);
}
if (exchangeParams.type === index_js_3.EExchangeType.DIRECT) {
const queue = exchangeParams.params;
this.logger.debug(`Direct exchange: producing message for queue ${queue.name}@${queue.ns}`);
return this.produceMessage(redisClient, msg, queue, cb);
}
this.logger.debug(`Fanout exchange: getting queues for exchange ${exchangeParams.type}`);
(0, _get_exchange_queues_js_1._getExchangeQueues)(redisClient, exchangeParams, (err, queues) => {
if (err) {
this.logger.error('Failed to get exchange queues', err);
return cb(err);
}
if (!(queues === null || queues === void 0 ? void 0 : queues.length)) {
this.logger.error('No matching queues found for exchange');
return cb(new index_js_6.ProducerExchangeNoMatchedQueueError());
}
this.logger.info(`Found ${queues.length} matching queues for exchange`);
const messages = [];
redis_smq_common_1.async.eachOf(queues, (queue, index, done) => {
this.logger.debug(`Producing message for queue ${queue.name}@${queue.ns} (${index + 1}/${queues.length})`);
this.produceMessage(redisClient, msg, queue, (err, reply) => {
if (err) {
this.logger.error(`Failed to produce message for queue ${queue.name}@${queue.ns}`, err);
return done(err);
}
if (reply) {
this.logger.debug(`Successfully produced ${reply.length} messages for queue ${queue.name}@${queue.ns}`);
messages.push(...reply);
}
done();
});
}, (err) => {
if (err) {
this.logger.error('Failed to produce messages for some queues', err);
return cb(err);
}
this.logger.info(`Successfully produced ${messages.length} messages across ${queues.length} queues`);
cb(null, messages);
});
});
}
}
exports.Producer = Producer;
//# sourceMappingURL=producer.js.map