redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
292 lines • 14.1 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MessageHandler = void 0;
const path_1 = __importDefault(require("path"));
const redis_smq_common_1 = require("redis-smq-common");
const scripts_js_1 = require("../../common/redis/scripts.js");
const redis_keys_js_1 = require("../../common/redis/redis-keys/redis-keys.js");
const _parse_message_js_1 = require("../../message-manager/_/_parse-message.js");
const index_js_1 = require("../../message/index.js");
const index_js_2 = require("../../queue-manager/index.js");
const consume_message_js_1 = require("./consume-message/consume-message.js");
const dequeue_message_js_1 = require("./dequeue-message/dequeue-message.js");
const event_publisher_js_1 = require("./event-publisher.js");
const connection_pool_js_1 = require("../../common/redis/redis-connection-pool/types/connection-pool.js");
const redis_connection_pool_js_1 = require("../../common/redis/redis-connection-pool/redis-connection-pool.js");
const _delete_ephemeral_consumer_group_js_1 = require("./_/_delete-ephemeral-consumer-group.js");
const _prepare_consumer_group_js_1 = require("./_/_prepare-consumer-group.js");
const _subscribe_consumer_js_1 = require("./_/_subscribe-consumer.js");
const _unsubscribe_consumer_js_1 = require("./_/_unsubscribe-consumer.js");
const redis_config_js_1 = require("../../common/redis/redis-config.js");
const WORKERS_DIR = path_1.default.resolve(redis_smq_common_1.env.getCurrentDir(), './queue-workers/workers');
class MessageHandler extends redis_smq_common_1.Runnable {
constructor(consumerContext, handlerParams, autoDequeue = true) {
super();
this.dequeueMessage = null;
this.consumeMessage = null;
this.queueWorkerCluster = null;
this.redisClient = null;
this.ephemeralConsumerGroupId = null;
this.onMessageReceived = (messageId) => {
this.processMessage(messageId);
};
this.onMessageNext = () => {
this.timer.schedule(() => this.next(), 1000);
};
this.runWorkerCluster = (cb) => {
const redisClient = this.getRedisClient();
if (redisClient instanceof Error) {
return cb(redisClient);
}
const { keyQueueWorkersLock } = redis_keys_js_1.redisKeys.getQueueKeys(this.queue.queueParams.ns, this.queue.queueParams.name, this.queue.groupId);
this.queueWorkerCluster = new redis_smq_common_1.WorkerCluster(redisClient, this.logger, keyQueueWorkersLock);
this.queueWorkerCluster.on('workerCluster.error', (err) => this.handleError(err));
this.queueWorkerCluster.loadFromDir(WORKERS_DIR, {
config: this.config,
redisConfig: redis_config_js_1.RedisConfig.getConfig(),
queueParsedParams: this.queue,
loggerContext: {
namespaces: this.logger.getNamespaces(),
},
consumerId: this.consumerContext.consumerId,
}, (err) => {
var _a;
if (err)
return cb(err);
(_a = this.queueWorkerCluster) === null || _a === void 0 ? void 0 : _a.run(() => void 0);
cb();
});
};
this.shutdownWorkerCluster = (cb) => {
if (this.queueWorkerCluster) {
this.queueWorkerCluster.shutdown(() => {
this.queueWorkerCluster = null;
cb();
});
}
else
cb();
};
this.consumerContext = consumerContext;
this.logger = consumerContext.logger.createLogger(this.constructor.name);
this.config = consumerContext.config;
const { queue, messageHandler } = handlerParams;
this.queue = queue;
this.messageHandler = messageHandler;
this.autoDequeue = autoDequeue;
this.timer = new redis_smq_common_1.Timer(this.logger);
(0, event_publisher_js_1.eventPublisher)(this);
}
getRedisClient() {
if (!this.redisClient)
return new redis_smq_common_1.PanicError({ message: 'A RedisClient instance is required.' });
return this.redisClient;
}
handleError(err) {
if (!this.isOperational())
return;
this.logger.error(`MessageHandler error: ${err.message}`, err);
this.emit('consumer.messageHandler.error', err, this.consumerContext.consumerId, this.queue);
super.handleError(err);
}
createDequeueMessageInstance() {
return new dequeue_message_js_1.DequeueMessage(this.consumerContext, this.queue);
}
goingUp() {
return super.goingUp().concat([
(cb) => this.timer.run(cb),
(cb) => {
redis_connection_pool_js_1.RedisConnectionPool.getInstance().acquire(connection_pool_js_1.ERedisConnectionAcquisitionMode.SHARED, (err, redisClient) => {
if (err)
return cb(err);
if (!redisClient)
return cb(new redis_smq_common_1.CallbackEmptyReplyError());
this.redisClient = redisClient;
cb();
});
},
(cb) => {
(0, _prepare_consumer_group_js_1._prepareConsumerGroup)(this.queue, this.consumerContext.consumerId, (err, effectiveGroupId) => {
if (err)
return cb(err);
if (effectiveGroupId && this.queue.groupId !== effectiveGroupId) {
this.queue = Object.assign(Object.assign({}, this.queue), { groupId: effectiveGroupId });
this.ephemeralConsumerGroupId = effectiveGroupId;
}
cb();
}, this.logger);
},
(cb) => {
(0, _subscribe_consumer_js_1._subscribeConsumer)(this.consumerContext.consumerId, this.queue, cb);
},
(cb) => {
this.consumeMessage = new consume_message_js_1.ConsumeMessage(this.consumerContext, this.queue, this.getId(), this.messageHandler);
this.consumeMessage.on('consumer.consumeMessage.error', (err) => this.handleError(err));
this.consumeMessage.on('consumer.consumeMessage.next', () => {
this.next();
});
this.consumeMessage.run(cb);
},
(cb) => {
this.dequeueMessage = this.createDequeueMessageInstance();
this.dequeueMessage.on('consumer.dequeueMessage.error', (err) => this.handleError(err));
this.dequeueMessage.on('consumer.dequeueMessage.messageReceived', this.onMessageReceived);
this.dequeueMessage.on('consumer.dequeueMessage.nextMessage', this.onMessageNext);
this.dequeueMessage.run(cb);
},
this.runWorkerCluster,
(cb) => {
if (this.autoDequeue) {
this.dequeue();
}
cb();
},
]);
}
goingDown() {
return [
(cb) => this.timer.shutdown(cb),
(cb) => {
const ephemeral = this.ephemeralConsumerGroupId;
if (!ephemeral)
return cb();
(0, _delete_ephemeral_consumer_group_js_1._deleteEphemeralConsumerGroup)(this.queue.queueParams, this.consumerContext.consumerId, ephemeral, (err) => {
if (err) {
this.logger.warn(`Failed to delete ephemeral consumer group '${ephemeral}': ${err.message}`);
}
this.ephemeralConsumerGroupId = null;
cb();
});
},
this.shutdownWorkerCluster,
(cb) => {
if (this.dequeueMessage) {
this.dequeueMessage.shutdown(() => {
var _a, _b, _c;
(_a = this.dequeueMessage) === null || _a === void 0 ? void 0 : _a.removeListener('consumer.dequeueMessage.error', (err) => this.handleError(err));
(_b = this.dequeueMessage) === null || _b === void 0 ? void 0 : _b.removeListener('consumer.dequeueMessage.messageReceived', this.onMessageReceived);
(_c = this.dequeueMessage) === null || _c === void 0 ? void 0 : _c.removeListener('consumer.dequeueMessage.nextMessage', this.onMessageNext);
this.dequeueMessage = null;
cb();
});
}
else
cb();
},
(cb) => {
if (this.consumeMessage) {
this.consumeMessage.shutdown(() => {
var _a;
(_a = this.consumeMessage) === null || _a === void 0 ? void 0 : _a.removeListener('consumer.consumeMessage.error', (err) => this.handleError(err));
this.consumeMessage = null;
cb();
});
}
else
cb();
},
(cb) => {
(0, _unsubscribe_consumer_js_1._unsubscribeConsumer)(this.consumerContext.consumerId, this.queue, cb);
},
(cb) => {
if (this.redisClient) {
redis_connection_pool_js_1.RedisConnectionPool.getInstance().release(this.redisClient);
this.redisClient = null;
}
cb();
},
].concat(super.goingDown());
}
processMessage(messageId) {
if (!this.isOperational() || !this.consumeMessage) {
return;
}
const consumeMessage = this.consumeMessage;
const redisClient = this.getRedisClient();
if (redisClient instanceof Error) {
return this.handleError(redisClient);
}
const { keyMessage } = redis_keys_js_1.redisKeys.getMessageKeys(messageId);
const { keyQueueProperties } = redis_keys_js_1.redisKeys.getQueueKeys(this.queue.queueParams.ns, this.queue.queueParams.name, this.queue.groupId);
const keys = [keyMessage, keyQueueProperties];
const argv = [
index_js_1.EMessageProperty.PROCESSING_STARTED_AT,
index_js_1.EMessageProperty.LAST_PROCESSED_AT,
Date.now(),
index_js_1.EMessageProperty.STATUS,
index_js_1.EMessagePropertyStatus.PROCESSING,
index_js_1.EMessagePropertyStatus.PENDING,
index_js_1.EMessageProperty.ATTEMPTS,
index_js_2.EQueueProperty.PROCESSING_MESSAGES_COUNT,
index_js_2.EQueueProperty.PENDING_MESSAGES_COUNT,
index_js_2.EQueueProperty.OPERATIONAL_STATE,
index_js_2.EQueueOperationalState.ACTIVE,
index_js_2.EQueueOperationalState.PAUSED,
index_js_2.EQueueOperationalState.STOPPED,
index_js_2.EQueueOperationalState.LOCKED,
];
redisClient.runScript(scripts_js_1.ERedisScriptName.CHECKOUT_MESSAGE, keys, argv, (err, reply) => {
if (err)
return this.handleError(err);
if (reply === 'QUEUE_STOPPED') {
this.logger.warn(`Cannot checkout message ${messageId}: Queue is in STOPPED state. Consumer will stop processing from this queue.`);
this.shutdown(() => {
this.logger.debug(`MessageHandler for queue ${this.queue.queueParams.name} shut down due to STOPPED state`);
});
return;
}
if (reply === 'QUEUE_LOCKED') {
this.logger.warn(`Cannot checkout message ${messageId}: Queue is in LOCKED state. Consumer will stop processing from this queue.`);
this.shutdown(() => {
this.logger.debug(`MessageHandler for queue ${this.queue.queueParams.name} shut down due to LOCKED state`);
});
return;
}
if (reply === 'QUEUE_INVALID_STATE') {
this.logger.warn(`Cannot checkout message ${messageId}: Queue is in invalid state. Consumer will stop processing from this queue.`);
this.shutdown(() => {
this.logger.debug(`MessageHandler for queue ${this.queue.queueParams.name} shut down due to invalid state`);
});
return;
}
if (reply === 'MESSAGE_NOT_FOUND') {
this.logger.warn(`Message [${messageId}] not found. It may have been deleted or expired.`);
this.next();
return;
}
if (reply === 'MESSAGE_NOT_PENDING') {
this.logger.warn(`Message [${messageId}] could not be fetched. It may have been processed by another consumer.`);
this.next();
return;
}
if (!reply) {
this.logger.warn(`Message [${messageId}] could not be fetched. It may have been processed by another consumer.`);
this.next();
return;
}
if (!Array.isArray(reply)) {
return this.handleError(new redis_smq_common_1.CallbackInvalidReplyError());
}
const message = (0, _parse_message_js_1._parseMessage)(reply);
consumeMessage.handleReceivedMessage(message);
});
}
next() {
if (this.isOperational()) {
this.dequeue();
}
}
dequeue() {
if (this.isOperational() && this.dequeueMessage) {
this.dequeueMessage.dequeue();
}
}
getQueue() {
return this.queue;
}
}
exports.MessageHandler = MessageHandler;
//# sourceMappingURL=message-handler.js.map