redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
305 lines • 13.1 kB
JavaScript
import { async, Runnable, Timer } from 'redis-smq-common';
import { MessageHandlerAlreadyExistsError } from '../../errors/index.js';
import { MessageHandler } from '../message-handler/message-handler.js';
import { eventPublisher } from './event-publisher.js';
import { QueueStateChangeHandler } from './queue-state-change-handler.js';
import { _validateOperation } from '../../queue-operation-validator/_/_validate-operation.js';
import { withSharedPoolConnection } from '../../common/redis/redis-connection-pool/with-shared-pool-connection.js';
import { EQueueOperation } from '../../queue-operation-validator/index.js';
export class MessageHandlerRunner extends Runnable {
handlerReconciliationInterval = 5000;
consumerContext;
supervisorTimer;
queueStateChangeHandler;
logger;
messageHandlerInstances = [];
messageHandlers = [];
constructor(consumerContext) {
super();
this.consumerContext = consumerContext;
this.logger = this.consumerContext.logger.createLogger(this.constructor.name);
this.queueStateChangeHandler = new QueueStateChangeHandler(this, this.logger);
eventPublisher(this);
this.supervisorTimer = new Timer(this.logger);
this.logger.debug(`MessageHandlerRunner with ID: ${this.id} initialized.`);
}
getQueueIdentifier(queue) {
return `${queue.queueParams.ns}:${queue.queueParams.name}:${queue.groupId ?? ''}`;
}
isSameQueue(q1, q2) {
return this.getQueueIdentifier(q1) === this.getQueueIdentifier(q2);
}
isQueueActive(queue) {
return this.queueStateChangeHandler.isQueueActive(queue.queueParams);
}
isQueueStopped(queue) {
return this.queueStateChangeHandler.isQueueStopped(queue.queueParams);
}
isQueuePaused(queue) {
return this.queueStateChangeHandler.isQueuePaused(queue.queueParams);
}
isQueueLocked(queue) {
return this.queueStateChangeHandler.isQueueLocked(queue.queueParams);
}
scheduleReconciliation = () => {
if (this.isOperational()) {
this.supervisorTimer.schedule(this.reconcileHandlers, this.handlerReconciliationInterval);
}
};
reconcileHandlers = () => {
if (!this.isOperational())
return;
const runningQueues = new Set(this.messageHandlerInstances.map((i) => this.getQueueIdentifier(i.getQueue())));
const zombieHandlers = this.messageHandlers.filter((i) => !runningQueues.has(this.getQueueIdentifier(i.queue)) &&
this.isQueueActive(i.queue));
if (zombieHandlers.length > 0) {
this.logger.warn(`Found ${zombieHandlers.length} zombie handler(s). Attempting to restart sequentially...`);
const tasks = zombieHandlers.map((handlerParams) => {
return (done) => {
if (!this.getMessageHandler(handlerParams.queue)) {
this.logger.warn(`Handler for queue ${handlerParams.queue.queueParams.name} was removed during reconciliation. Skipping restart.`);
return done();
}
if (!this.isQueueActive(handlerParams.queue)) {
this.logger.debug(`Queue ${handlerParams.queue.queueParams.name} is not active, skipping restart`);
return done();
}
this.logger.debug(`Reconciling handler for queue: ${handlerParams.queue.queueParams.name}`);
this.runMessageHandler(handlerParams, (err) => {
if (err) {
this.logger.error(`Failed to restart zombie handler for queue ${handlerParams.queue.queueParams.name}.`);
}
done();
});
};
});
async.series(tasks, () => {
this.logger.debug('Finished reconciliation series for this tick.');
this.scheduleReconciliation();
});
}
else {
this.scheduleReconciliation();
}
};
getMessageHandlerInstance(queue) {
return this.messageHandlerInstances.find((i) => this.isSameQueue(i.getQueue(), queue));
}
getMessageHandler(queue) {
return this.messageHandlers.find((i) => this.isSameQueue(i.queue, queue));
}
createMessageHandlerInstance(handlerParams) {
const instance = new MessageHandler(this.consumerContext, handlerParams, true);
instance.on('consumer.messageHandler.error', (err) => {
this.logger.error(`MessageHandler [${instance.getId()}] has experienced a runtime error: ${err.message}. Shutting down instance. The supervisor will attempt to restart it.`);
this.shutdownMessageHandler(instance, (err) => {
if (err) {
this.logger.error(`Failed to shutdown handler ${instance.getId()}: ${err.message}`);
}
});
});
this.messageHandlerInstances.push(instance);
return instance;
}
runMessageHandler(handlerParams, cb) {
if (!this.isQueueActive(handlerParams.queue)) {
this.logger.debug(`Queue ${handlerParams.queue.queueParams.name} is not active, skipping start`);
return cb();
}
if (this.getMessageHandlerInstance(handlerParams.queue)) {
this.logger.warn(`A message handler instance for queue ${handlerParams.queue.queueParams.name} is already running.`);
return cb();
}
const handler = this.createMessageHandlerInstance(handlerParams);
handler.run((err) => {
if (err) {
this.logger.error(`Failed to run message handler for queue ${handlerParams.queue.queueParams.name}. Removing configuration.`, err);
this.removeMessageHandler(handlerParams.queue, () => cb(err));
}
else {
cb();
}
});
}
shutdownMessageHandler(messageHandler, cb) {
messageHandler.shutdown(() => {
this.messageHandlerInstances = this.messageHandlerInstances.filter((handler) => handler.getId() !== messageHandler.getId());
cb();
});
}
runMessageHandlers = (cb) => {
const handlersToStart = this.messageHandlers.filter((handler) => this.isQueueActive(handler.queue));
async.each(handlersToStart, (handlerParams, _, done) => {
this.runMessageHandler(handlerParams, done);
}, cb);
};
shutDownMessageHandlers = (cb) => {
async.each(this.messageHandlerInstances, (handler, _, done) => {
this.shutdownMessageHandler(handler, done);
}, () => {
this.messageHandlerInstances = [];
cb();
});
};
goingUp() {
return super.goingUp().concat([
(cb) => this.supervisorTimer.run(cb),
this.runMessageHandlers,
(cb) => {
this.reconcileHandlers();
cb();
},
]);
}
goingDown() {
return [
(cb) => this.supervisorTimer.shutdown(cb),
this.queueStateChangeHandler.shutdown,
this.shutDownMessageHandlers,
].concat(super.goingDown());
}
handleError(err) {
if (!this.isOperational())
return;
this.logger.error(`MessageHandlerRunner error: ${err.message}`, err);
this.emit('consumer.messageHandlerRunner.error', err, this.consumerContext.consumerId);
super.handleError(err);
}
stopMessageHandler(queue, cb) {
const handlerInstance = this.getMessageHandlerInstance(queue);
if (!handlerInstance) {
const hasConfig = !!this.getMessageHandler(queue);
this.logger.debug(`Stop requested for queue: ${queue.queueParams.name} (no instance)`);
return cb(null, hasConfig);
}
this.shutdownMessageHandler(handlerInstance, (err) => {
if (err) {
this.logger.error(`Failed to stop handler for queue ${queue.queueParams.name}:`, err);
cb(err);
}
else {
this.logger.debug(`Stopped message handler for queue: ${queue.queueParams.name} (configuration kept)`);
cb(null, true);
}
});
}
startMessageHandler(queue, cb) {
const handlerConfig = this.getMessageHandler(queue);
if (!handlerConfig) {
this.logger.warn(`No handler configuration found for queue: ${queue.queueParams.name}`);
return cb(null, false);
}
if (this.getMessageHandlerInstance(queue)) {
this.logger.debug(`Handler already running for queue: ${queue.queueParams.name}`);
return cb(null, false);
}
if (!this.isQueueActive(queue)) {
this.logger.debug(`Queue ${queue.queueParams.name} is not active, cannot start handler`);
return cb(null, false);
}
this.runMessageHandler(handlerConfig, (err) => {
if (err) {
this.logger.error(`Failed to start handler for queue ${queue.queueParams.name}:`, err);
cb(err);
}
else {
this.logger.debug(`Started message handler for queue: ${queue.queueParams.name}`);
cb(null, true);
}
});
}
removeMessageHandler(queue, cb) {
const hadConfig = this.getMessageHandler(queue);
this.messageHandlers = this.messageHandlers.filter((h) => !this.isSameQueue(h.queue, queue));
const handlerInstance = this.getMessageHandlerInstance(queue);
if (handlerInstance) {
this.shutdownMessageHandler(handlerInstance, cb);
}
else {
if (hadConfig) {
this.logger.debug(`Removed handler configuration for queue: ${queue.queueParams.name}`);
}
cb();
}
}
addMessageHandler(queue, messageHandler, cb) {
if (this.getMessageHandler(queue)) {
this.logger.warn(`Message handler for queue ${queue.queueParams.name} already exists`);
return cb(new MessageHandlerAlreadyExistsError());
}
async.series([
(cb) => withSharedPoolConnection((client, cb) => {
_validateOperation(client, queue.queueParams, EQueueOperation.CONSUME, cb);
}, cb),
(cb) => {
const handlerParams = {
queue,
messageHandler,
};
this.messageHandlers.push(handlerParams);
this.logger.debug(`Message handler registered for queue: ${queue.queueParams.name}. Total handlers: ${this.messageHandlers.length}`);
if (this.isOperational() && this.isQueueActive(queue)) {
this.runMessageHandler(handlerParams, cb);
}
else {
cb();
}
},
], (err) => cb(err));
}
getQueueWithStatus() {
const queues = this.getQueues();
return queues.map((queue) => {
const status = this.getMessageHandlerInstance(queue)?.isRunning()
? 'active'
: 'stopped';
return {
queue,
status,
};
});
}
getQueues() {
return this.messageHandlers.map((i) => i.queue);
}
getActiveQueues() {
return this.messageHandlers
.filter((handler) => this.isQueueActive(handler.queue))
.map((i) => i.queue);
}
getStoppedQueues() {
return this.messageHandlers
.filter((handler) => this.isQueueStopped(handler.queue))
.map((i) => i.queue);
}
getPausedQueues() {
return this.messageHandlers
.filter((handler) => this.isQueuePaused(handler.queue))
.map((i) => i.queue);
}
getLockedQueues() {
return this.messageHandlers
.filter((handler) => this.isQueueLocked(handler.queue))
.map((i) => i.queue);
}
isMessageHandlerStopped(queue) {
return this.isQueueStopped(queue);
}
isMessageHandlerRunning(queue) {
const instance = this.getMessageHandlerInstance(queue);
return !!instance && instance.isOperational();
}
getHandlerCount() {
const total = this.messageHandlers.length;
const active = this.getActiveQueues().length;
const stopped = this.getStoppedQueues().length;
const paused = this.getPausedQueues().length;
const locked = this.getLockedQueues().length;
const accountedFor = active + stopped + paused + locked;
if (accountedFor !== total) {
this.logger.warn(`Queue state accounting mismatch: total=${total}, accounted=${accountedFor}`);
}
return { total, active, stopped, paused, locked };
}
}
//# sourceMappingURL=message-handler-runner.js.map