redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
364 lines • 15.8 kB
JavaScript
import { stat } from 'fs';
import path from 'path';
import { async, AsyncCallbackTimeoutError, CallableWorker, CallbackEmptyReplyError, Runnable, } from 'redis-smq-common';
import { redisKeys } from '../../../common/redis/redis-keys/redis-keys.js';
import { EMessageUnacknowledgementAction, EMessageUnacknowledgementCause, } from './types/index.js';
import { InvalidMessageHandlerSignatureError, InvalidMessageHandlerTypeError, MessageHandlerFileError, MessageHandlerFilenameExtensionError, } from '../../../errors/index.js';
import { MessageUnacknowledger } from './message-unacknowledger.js';
import { eventPublisher } from './event-publisher.js';
import { ERedisConnectionAcquisitionMode } from '../../../common/redis/redis-connection-pool/types/connection-pool.js';
import { RedisConnectionPool } from '../../../common/redis/redis-connection-pool/redis-connection-pool.js';
import { MessageAcknowledger } from './message-acknowledger.js';
const isCallbackHandler = (handler) => {
return handler.length === 2;
};
const isPromiseHandler = (handler) => {
return handler.length === 1;
};
export class ConsumeMessage extends Runnable {
consumerId;
config;
logger;
keyQueueProperties;
keyQueueProcessing;
keyQueueAcknowledged;
queue;
messageHandler;
messageHandlerId;
consumerOptions;
redisClient = null;
consumeMessageWorker = null;
messageUnacknowledger;
messageAcknowledger;
constructor(consumerContext, queue, messageHandlerId, messageHandler) {
super();
this.consumerId = consumerContext.consumerId;
this.logger = consumerContext.logger.createLogger(this.constructor.name);
this.config = consumerContext.config;
this.consumerOptions = consumerContext.consumerOptions;
this.queue = queue;
this.messageHandler = messageHandler;
this.messageHandlerId = messageHandlerId;
this.messageAcknowledger = this.initializeMessageAcknowledger();
this.messageUnacknowledger = this.initializeMessageUnacknowledger();
const { keyQueueProcessing } = redisKeys.getQueueConsumerKeys(this.queue.queueParams, this.consumerId);
const { keyQueueProperties, keyQueueAcknowledged } = redisKeys.getQueueKeys(this.queue.queueParams.ns, this.queue.queueParams.name, this.queue.groupId);
this.keyQueueProperties = keyQueueProperties;
this.keyQueueAcknowledged = keyQueueAcknowledged;
this.keyQueueProcessing = keyQueueProcessing;
eventPublisher(this);
this.logger.debug(`${this.constructor.name} initialized for queue ${queue.queueParams.name}@${queue.queueParams.ns}`);
}
initializeMessageAcknowledger() {
const messageAcknowledger = new MessageAcknowledger(this.queue, this.consumerId, this.logger, this.consumerOptions);
messageAcknowledger.on('messageAcknowledger.error', (err) => {
this.handleError(err);
});
messageAcknowledger.on('messageAcknowledger.messageAcknowledged', (message) => {
this.onMessageAcknowledged(message);
});
return messageAcknowledger;
}
initializeMessageUnacknowledger() {
const messageUnacknowledger = new MessageUnacknowledger(this.consumerId, this.queue, this.logger, this.consumerOptions);
messageUnacknowledger.on('messageUnacknowledger.error', (err) => {
this.handleError(err);
});
messageUnacknowledger.on('messageUnacknowledger.messagesUnacknowledged', (status) => {
this.handleUnacknowledgementResult(status);
});
return messageUnacknowledger;
}
handleReceivedMessage(message) {
const messageId = message.getId();
this.logger.debug(`Received message ${messageId}`);
if (!this.isOperational()) {
this.logger.warn(`Ignoring message ${messageId} - consumer not running`);
return;
}
if (message.getSetExpired()) {
this.logger.info(`Message ${messageId} expired, unacknowledging`);
this.unacknowledgeMessage(message, EMessageUnacknowledgementCause.TTL_EXPIRED);
}
else {
this.logger.debug(`Message ${messageId} valid, proceeding with consumption`);
this.consumeMessage(message);
}
}
acknowledgeMessage(message) {
if (this.messageAcknowledger) {
this.messageAcknowledger.add(message);
}
else {
this.logger.error('MessageAcknowledger not initialized');
}
this.emit('consumer.consumeMessage.next');
}
unacknowledgeMessage(message, cause) {
this.messageUnacknowledger.add(message, cause);
this.emit('consumer.consumeMessage.next');
}
onMessageAcknowledged(message) {
const messageId = message.getId();
this.logger.info(`Message ${messageId} acknowledged successfully`);
this.emit('consumer.consumeMessage.messageAcknowledged', messageId, this.queue, this.messageHandlerId, this.consumerId);
}
handleUnacknowledgementResult(result) {
const messageCount = Object.keys(result).length;
if (messageCount === 0)
return;
this.logger.debug(`Handling unacknowledgement result for ${messageCount} messages`);
for (const [messageId, details] of Object.entries(result)) {
this.logger.info(`Message ${messageId} unacknowledged: ${EMessageUnacknowledgementCause[details.cause]}`);
this.emit('consumer.consumeMessage.messageUnacknowledged', messageId, this.queue, this.messageHandlerId, this.consumerId, details.cause);
if (details.action === EMessageUnacknowledgementAction.DEAD_LETTER) {
this.logger.info(`Message ${messageId} moved to dead letter queue`);
this.emit('consumer.consumeMessage.messageDeadLettered', messageId, this.queue, this.messageHandlerId, this.consumerId, details.deadLetterCause);
}
else if (details.action === EMessageUnacknowledgementAction.DELAY) {
this.logger.info(`Message ${messageId} delayed for retry`);
this.emit('consumer.consumeMessage.messageDelayed', messageId, this.queue, this.messageHandlerId, this.consumerId);
}
else {
this.logger.info(`Message ${messageId} re-queued for retry`);
this.emit('consumer.consumeMessage.messageRequeued', messageId, this.queue, this.messageHandlerId, this.consumerId);
}
}
}
consumeMessage(message) {
const messageId = message.getId();
this.logger.debug(`Consuming message ${messageId}`);
let isCallbackHandled = false;
try {
let onConsumed = (err) => {
if (isCallbackHandled) {
this.logger.debug(`Callback for ${messageId} already handled`);
return;
}
isCallbackHandled = true;
if (!this.isOperational()) {
this.logger.debug(`Consumer not running, ignoring callback for ${messageId}`);
return;
}
if (err) {
this.logger.error(`Error consuming ${messageId}: ${err.message}`);
let reason = EMessageUnacknowledgementCause.UNACKNOWLEDGED;
if (err instanceof AsyncCallbackTimeoutError) {
reason = EMessageUnacknowledgementCause.TIMEOUT;
}
if (err instanceof InvalidMessageHandlerSignatureError) {
reason = EMessageUnacknowledgementCause.INVALID_HANDLER_SIGNATURE;
}
this.unacknowledgeMessage(message, reason);
return;
}
this.logger.debug(`Message ${messageId} consumed successfully`);
this.acknowledgeMessage(message);
};
const consumeTimeout = message.producibleMessage.getConsumeTimeout();
if (consumeTimeout) {
this.logger.debug(`Setting timeout ${consumeTimeout}ms for ${messageId}`);
onConsumed = async.withTimeout(onConsumed, consumeTimeout);
}
this.invokeMessageHandler(this.messageHandler, message.transfer(), onConsumed);
}
catch (error) {
this.logger.error(`Exception consuming ${messageId}: ${error}`);
isCallbackHandled = true;
this.unacknowledgeMessage(message, EMessageUnacknowledgementCause.CONSUME_ERROR);
this.emit('consumer.consumeMessage.next');
}
}
invokeMessageHandler(handler, msg, cb) {
const messageId = msg.id;
if (typeof handler === 'string') {
this.getWorker(handler).call(msg, (err) => {
if (err) {
this.logger.error(`Worker handler failed for ${messageId}: ${err.message}`);
}
cb(err);
});
return;
}
let isCompleted = false;
const complete = (err) => {
if (isCompleted)
return;
isCompleted = true;
if (err) {
this.logger.error(`Handler failed for ${messageId}: ${err.message}`);
cb(err);
}
else {
this.logger.debug(`Handler succeeded for ${messageId}`);
cb(null);
}
};
try {
if (isPromiseHandler(handler)) {
const result = handler(msg);
if (result && typeof result.then === 'function') {
result
.then(() => complete())
.catch((err) => {
const error = err instanceof Error
? err
: new Error(String(err ?? 'Promise rejected'));
complete(error);
});
}
else {
this.logger.warn(`Handler for ${messageId} didn't return a Promise`);
complete();
}
}
else if (isCallbackHandler(handler)) {
handler(msg, (err) => {
complete(err ?? undefined);
});
}
else {
this.logger.error(`Handler for ${messageId} has invalid signature`);
complete(new InvalidMessageHandlerSignatureError({
metadata: {
queue: this.queue,
},
}));
}
}
catch (err) {
const error = err instanceof Error
? err
: new Error(String(err || 'Handler threw error'));
complete(error);
}
}
getWorker(filename) {
if (!this.consumeMessageWorker) {
this.logger.debug(`Creating worker for ${filename}`);
this.consumeMessageWorker = new CallableWorker(filename, this.logger);
}
return this.consumeMessageWorker;
}
validateHandler = (cb) => {
const messageHandler = this.messageHandler;
if (typeof messageHandler === 'function') {
if (![1, 2].includes(this.messageHandler.length)) {
this.logger.debug('Function-based handler has invalid signature');
return cb(new InvalidMessageHandlerSignatureError({
metadata: {
queue: this.queue,
},
}));
}
return cb();
}
if (typeof messageHandler === 'string') {
const ext = path.extname(messageHandler);
if (!['.js', '.cjs'].includes(ext)) {
this.logger.error(`Invalid extension: ${ext}`);
return cb(new MessageHandlerFilenameExtensionError());
}
return stat(messageHandler, (err) => {
if (err) {
this.logger.error(`Handler file not found: ${messageHandler}`);
return cb(new MessageHandlerFileError());
}
this.logger.debug(`Handler validated: ${messageHandler}`);
cb();
});
}
this.logger.debug('Invalid message handler type');
return cb(new InvalidMessageHandlerTypeError({
metadata: {
queue: this.queue,
},
}));
};
goingUp() {
return super.goingUp().concat([
(cb) => {
this.logger.debug('Initializing Redis client');
RedisConnectionPool.getInstance().acquire(ERedisConnectionAcquisitionMode.SHARED, (err, client) => {
if (err) {
this.logger.error(`Redis client failed: ${err.message}`);
return cb(err);
}
if (!client)
return cb(new CallbackEmptyReplyError());
this.redisClient = client;
cb();
});
},
(cb) => {
this.logger.debug('Starting MessageUnacknowledger');
this.messageUnacknowledger.run((err) => {
if (err) {
this.logger.error(`Failed to start MessageUnacknowledger: ${err.message}`);
return cb(err);
}
this.logger.debug('MessageUnacknowledger started');
cb();
});
},
(cb) => {
this.messageAcknowledger.run((err) => {
if (err) {
this.logger.error(`Failed to start MessageAcknowledger: ${err.message}`);
return cb(err);
}
this.logger.debug('MessageAcknowledger started');
cb();
});
},
(cb) => {
this.logger.debug('Validating message handler');
this.validateHandler(cb);
},
]);
}
goingDown() {
return [
(cb) => {
if (this.messageAcknowledger) {
this.logger.info('Shutting down MessageAcknowledger');
this.messageAcknowledger.shutdown(cb);
}
else {
cb();
}
},
(cb) => {
this.logger.debug('Shutting down MessageUnacknowledger');
this.messageUnacknowledger.shutdown(cb);
},
(cb) => {
if (this.consumeMessageWorker) {
this.consumeMessageWorker.shutdown((err) => {
if (err)
this.logger.warn(`Worker shutdown error: ${err.message}`);
cb();
});
}
else {
cb();
}
},
(cb) => {
if (this.redisClient) {
RedisConnectionPool.getInstance().release(this.redisClient);
this.redisClient = null;
}
cb();
},
].concat(super.goingDown());
}
handleError(err) {
if (!this.isOperational())
return;
const error = err instanceof Error ? err : new Error(String(err));
this.emit('consumer.consumeMessage.error', error, this.consumerId, this.queue);
super.handleError(err);
}
}
//# sourceMappingURL=consume-message.js.map