redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
289 lines • 12.7 kB
JavaScript
import path from 'path';
import { CallbackEmptyReplyError, CallbackInvalidReplyError, env, PanicError, Runnable, Timer, WorkerCluster, } from 'redis-smq-common';
import { ERedisScriptName } from '../../common/redis/scripts.js';
import { redisKeys } from '../../common/redis/redis-keys/redis-keys.js';
import { _parseMessage } from '../../message-manager/_/_parse-message.js';
import { EMessageProperty, EMessagePropertyStatus, } from '../../message/index.js';
import { EQueueOperationalState, EQueueProperty, } from '../../queue-manager/index.js';
import { ConsumeMessage } from './consume-message/consume-message.js';
import { DequeueMessage } from './dequeue-message/dequeue-message.js';
import { eventPublisher } from './event-publisher.js';
import { ERedisConnectionAcquisitionMode } from '../../common/redis/redis-connection-pool/types/connection-pool.js';
import { RedisConnectionPool } from '../../common/redis/redis-connection-pool/redis-connection-pool.js';
import { _deleteEphemeralConsumerGroup } from './_/_delete-ephemeral-consumer-group.js';
import { _prepareConsumerGroup } from './_/_prepare-consumer-group.js';
import { _subscribeConsumer } from './_/_subscribe-consumer.js';
import { _unsubscribeConsumer } from './_/_unsubscribe-consumer.js';
import { RedisConfig } from '../../common/redis/redis-config.js';
const WORKERS_DIR = path.resolve(env.getCurrentDir(), './queue-workers/workers');
export class MessageHandler extends Runnable {
consumerContext;
config;
logger;
queue;
dequeueMessage = null;
consumeMessage = null;
messageHandler;
autoDequeue;
queueWorkerCluster = null;
redisClient = null;
timer;
ephemeralConsumerGroupId = null;
constructor(consumerContext, handlerParams, autoDequeue = true) {
super();
this.consumerContext = consumerContext;
this.logger = consumerContext.logger.createLogger(this.constructor.name);
this.config = consumerContext.config;
const { queue, messageHandler } = handlerParams;
this.queue = queue;
this.messageHandler = messageHandler;
this.autoDequeue = autoDequeue;
this.timer = new Timer(this.logger);
eventPublisher(this);
}
getRedisClient() {
if (!this.redisClient)
return new PanicError({ message: 'A RedisClient instance is required.' });
return this.redisClient;
}
onMessageReceived = (messageId) => {
this.processMessage(messageId);
};
onMessageNext = () => {
this.timer.schedule(() => this.next(), 1000);
};
handleError(err) {
if (!this.isOperational())
return;
this.logger.error(`MessageHandler error: ${err.message}`, err);
this.emit('consumer.messageHandler.error', err, this.consumerContext.consumerId, this.queue);
super.handleError(err);
}
runWorkerCluster = (cb) => {
const redisClient = this.getRedisClient();
if (redisClient instanceof Error) {
return cb(redisClient);
}
const { keyQueueWorkersLock } = redisKeys.getQueueKeys(this.queue.queueParams.ns, this.queue.queueParams.name, this.queue.groupId);
this.queueWorkerCluster = new WorkerCluster(redisClient, this.logger, keyQueueWorkersLock);
this.queueWorkerCluster.on('workerCluster.error', (err) => this.handleError(err));
this.queueWorkerCluster.loadFromDir(WORKERS_DIR, {
config: this.config,
redisConfig: RedisConfig.getConfig(),
queueParsedParams: this.queue,
loggerContext: {
namespaces: this.logger.getNamespaces(),
},
consumerId: this.consumerContext.consumerId,
}, (err) => {
if (err)
return cb(err);
this.queueWorkerCluster?.run(() => void 0);
cb();
});
};
shutdownWorkerCluster = (cb) => {
if (this.queueWorkerCluster) {
this.queueWorkerCluster.shutdown(() => {
this.queueWorkerCluster = null;
cb();
});
}
else
cb();
};
createDequeueMessageInstance() {
return new DequeueMessage(this.consumerContext, this.queue);
}
goingUp() {
return super.goingUp().concat([
(cb) => this.timer.run(cb),
(cb) => {
RedisConnectionPool.getInstance().acquire(ERedisConnectionAcquisitionMode.SHARED, (err, redisClient) => {
if (err)
return cb(err);
if (!redisClient)
return cb(new CallbackEmptyReplyError());
this.redisClient = redisClient;
cb();
});
},
(cb) => {
_prepareConsumerGroup(this.queue, this.consumerContext.consumerId, (err, effectiveGroupId) => {
if (err)
return cb(err);
if (effectiveGroupId && this.queue.groupId !== effectiveGroupId) {
this.queue = { ...this.queue, groupId: effectiveGroupId };
this.ephemeralConsumerGroupId = effectiveGroupId;
}
cb();
}, this.logger);
},
(cb) => {
_subscribeConsumer(this.consumerContext.consumerId, this.queue, cb);
},
(cb) => {
this.consumeMessage = new ConsumeMessage(this.consumerContext, this.queue, this.getId(), this.messageHandler);
this.consumeMessage.on('consumer.consumeMessage.error', (err) => this.handleError(err));
this.consumeMessage.on('consumer.consumeMessage.next', () => {
this.next();
});
this.consumeMessage.run(cb);
},
(cb) => {
this.dequeueMessage = this.createDequeueMessageInstance();
this.dequeueMessage.on('consumer.dequeueMessage.error', (err) => this.handleError(err));
this.dequeueMessage.on('consumer.dequeueMessage.messageReceived', this.onMessageReceived);
this.dequeueMessage.on('consumer.dequeueMessage.nextMessage', this.onMessageNext);
this.dequeueMessage.run(cb);
},
this.runWorkerCluster,
(cb) => {
if (this.autoDequeue) {
this.dequeue();
}
cb();
},
]);
}
goingDown() {
return [
(cb) => this.timer.shutdown(cb),
(cb) => {
const ephemeral = this.ephemeralConsumerGroupId;
if (!ephemeral)
return cb();
_deleteEphemeralConsumerGroup(this.queue.queueParams, this.consumerContext.consumerId, ephemeral, (err) => {
if (err) {
this.logger.warn(`Failed to delete ephemeral consumer group '${ephemeral}': ${err.message}`);
}
this.ephemeralConsumerGroupId = null;
cb();
});
},
this.shutdownWorkerCluster,
(cb) => {
if (this.dequeueMessage) {
this.dequeueMessage.shutdown(() => {
this.dequeueMessage?.removeListener('consumer.dequeueMessage.error', (err) => this.handleError(err));
this.dequeueMessage?.removeListener('consumer.dequeueMessage.messageReceived', this.onMessageReceived);
this.dequeueMessage?.removeListener('consumer.dequeueMessage.nextMessage', this.onMessageNext);
this.dequeueMessage = null;
cb();
});
}
else
cb();
},
(cb) => {
if (this.consumeMessage) {
this.consumeMessage.shutdown(() => {
this.consumeMessage?.removeListener('consumer.consumeMessage.error', (err) => this.handleError(err));
this.consumeMessage = null;
cb();
});
}
else
cb();
},
(cb) => {
_unsubscribeConsumer(this.consumerContext.consumerId, this.queue, cb);
},
(cb) => {
if (this.redisClient) {
RedisConnectionPool.getInstance().release(this.redisClient);
this.redisClient = null;
}
cb();
},
].concat(super.goingDown());
}
processMessage(messageId) {
if (!this.isOperational() || !this.consumeMessage) {
return;
}
const consumeMessage = this.consumeMessage;
const redisClient = this.getRedisClient();
if (redisClient instanceof Error) {
return this.handleError(redisClient);
}
const { keyMessage } = redisKeys.getMessageKeys(messageId);
const { keyQueueProperties } = redisKeys.getQueueKeys(this.queue.queueParams.ns, this.queue.queueParams.name, this.queue.groupId);
const keys = [keyMessage, keyQueueProperties];
const argv = [
EMessageProperty.PROCESSING_STARTED_AT,
EMessageProperty.LAST_PROCESSED_AT,
Date.now(),
EMessageProperty.STATUS,
EMessagePropertyStatus.PROCESSING,
EMessagePropertyStatus.PENDING,
EMessageProperty.ATTEMPTS,
EQueueProperty.PROCESSING_MESSAGES_COUNT,
EQueueProperty.PENDING_MESSAGES_COUNT,
EQueueProperty.OPERATIONAL_STATE,
EQueueOperationalState.ACTIVE,
EQueueOperationalState.PAUSED,
EQueueOperationalState.STOPPED,
EQueueOperationalState.LOCKED,
];
redisClient.runScript(ERedisScriptName.CHECKOUT_MESSAGE, keys, argv, (err, reply) => {
if (err)
return this.handleError(err);
if (reply === 'QUEUE_STOPPED') {
this.logger.warn(`Cannot checkout message ${messageId}: Queue is in STOPPED state. Consumer will stop processing from this queue.`);
this.shutdown(() => {
this.logger.debug(`MessageHandler for queue ${this.queue.queueParams.name} shut down due to STOPPED state`);
});
return;
}
if (reply === 'QUEUE_LOCKED') {
this.logger.warn(`Cannot checkout message ${messageId}: Queue is in LOCKED state. Consumer will stop processing from this queue.`);
this.shutdown(() => {
this.logger.debug(`MessageHandler for queue ${this.queue.queueParams.name} shut down due to LOCKED state`);
});
return;
}
if (reply === 'QUEUE_INVALID_STATE') {
this.logger.warn(`Cannot checkout message ${messageId}: Queue is in invalid state. Consumer will stop processing from this queue.`);
this.shutdown(() => {
this.logger.debug(`MessageHandler for queue ${this.queue.queueParams.name} shut down due to invalid state`);
});
return;
}
if (reply === 'MESSAGE_NOT_FOUND') {
this.logger.warn(`Message [${messageId}] not found. It may have been deleted or expired.`);
this.next();
return;
}
if (reply === 'MESSAGE_NOT_PENDING') {
this.logger.warn(`Message [${messageId}] could not be fetched. It may have been processed by another consumer.`);
this.next();
return;
}
if (!reply) {
this.logger.warn(`Message [${messageId}] could not be fetched. It may have been processed by another consumer.`);
this.next();
return;
}
if (!Array.isArray(reply)) {
return this.handleError(new CallbackInvalidReplyError());
}
const message = _parseMessage(reply);
consumeMessage.handleReceivedMessage(message);
});
}
next() {
if (this.isOperational()) {
this.dequeue();
}
}
dequeue() {
if (this.isOperational() && this.dequeueMessage) {
this.dequeueMessage.dequeue();
}
}
getQueue() {
return this.queue;
}
}
//# sourceMappingURL=message-handler.js.map