redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
233 lines • 9.6 kB
JavaScript
import { async, PanicError, Runnable, } from 'redis-smq-common';
import { redisKeys } from '../../../common/redis/redis-keys/redis-keys.js';
import { _getMessages } from '../../../message-manager/_/_get-message.js';
import { EMessageDeadLetterCause, EMessageUnacknowledgementAction, EMessageUnacknowledgementCause, } from './types/index.js';
import { withSharedPoolConnection } from '../../../common/redis/redis-connection-pool/with-shared-pool-connection.js';
import { _executeUnacknowledgementScript } from './_/_execute-unacknowledgement-script.js';
export class MessageUnacknowledger extends Runnable {
consumerId;
queue;
queueRef;
useBatchUnacks;
batchSize;
batchTimeoutMs;
batch;
logger;
constructor(consumerId, queue, logger, consumerOptions) {
super();
this.consumerId = consumerId;
this.queue = queue;
this.queueRef = `${queue.queueParams.name}@${queue.queueParams.ns}/${queue.groupId}`;
this.logger = logger.createLogger(`${this.constructor.name.toLowerCase()}`);
const { enabled, batchTimeoutMs, batchSize } = consumerOptions.batchUnacks;
this.useBatchUnacks = enabled;
this.batchSize = batchSize;
this.batchTimeoutMs = batchTimeoutMs;
this.batch = {
messages: [],
callbacks: [],
timer: null,
};
this.logger.debug(`MessageUnacknowledger initialized for queue ${this.queueRef}`);
}
getResolution(message, cause) {
if (cause === EMessageUnacknowledgementCause.TTL_EXPIRED) {
return {
cause,
action: EMessageUnacknowledgementAction.DEAD_LETTER,
deadLetterCause: EMessageDeadLetterCause.TTL_EXPIRED,
};
}
if (message.isPeriodic()) {
return {
cause,
action: EMessageUnacknowledgementAction.DEAD_LETTER,
deadLetterCause: EMessageDeadLetterCause.PERIODIC_MESSAGE,
};
}
if (message.hasRetryThresholdExceeded()) {
return {
cause,
action: EMessageUnacknowledgementAction.DEAD_LETTER,
deadLetterCause: EMessageDeadLetterCause.RETRY_THRESHOLD_EXCEEDED,
};
}
const delay = message.producibleMessage.getRetryDelay();
return delay
? { cause, action: EMessageUnacknowledgementAction.DELAY }
: { cause, action: EMessageUnacknowledgementAction.REQUEUE };
}
buildPendingMessages(messages, cause) {
return messages.map((m) => {
const resolution = this.getResolution(m, cause);
const { keyMessage } = redisKeys.getMessageKeys(m.getId());
return {
messageId: m.getId(),
keyMessage,
message: m,
resolution,
};
});
}
processBatch() {
if (this.batch.timer) {
clearTimeout(this.batch.timer);
this.batch.timer = null;
}
if (this.batch.messages.length === 0) {
this.batch.callbacks.forEach((cb) => cb(null, {}));
this.batch.callbacks = [];
return;
}
const messages = [...this.batch.messages];
const callbacks = [...this.batch.callbacks];
this.batch.messages = [];
this.batch.callbacks = [];
this.logger.debug(`Processing batch of ${messages.length} messages for queue ${this.queueRef}`);
_executeUnacknowledgementScript(this.queue, messages, this.consumerId, this.logger, (err, result) => {
if (err) {
callbacks.forEach((cb) => cb(err));
return this.handleError(err);
}
if (result) {
this.emit('messageUnacknowledger.messagesUnacknowledged', result);
callbacks.forEach((cb) => cb(null, result));
}
});
}
getMessagesFromQueue(client, cb) {
const { keyQueueProcessing } = redisKeys.getQueueConsumerKeys(this.queue.queueParams, this.consumerId);
async.waterfall([
(next) => {
client.lrange(keyQueueProcessing, 0, -1, (err, reply) => next(err, reply ?? []));
},
(ids, next) => {
if (!ids.length)
return next(null, []);
_getMessages(client, ids, next);
},
], cb);
}
goingDown() {
return [
(next) => {
if (this.batch.messages.length === 0) {
this.logger.debug('No pending batch to flush during shutdown');
return next();
}
this.logger.info(`Flushing pending batch of ${this.batch.messages.length} messages on shutdown`);
this.flush((err) => {
if (err) {
this.logger.error(`Error flushing batch during shutdown: ${err.message}`);
}
next();
});
},
(next) => {
this.logger.info('Unacknowledging messages in processing queue');
this.unacknowledgeProcessingQueue(EMessageUnacknowledgementCause.SHUTTING_DOWN, () => next());
},
];
}
handleError(err) {
if (!this.isOperational())
return;
const error = err instanceof Error ? err : new Error(String(err));
this.logger.error(`MessageUnacknowledger error: ${error.message}`);
this.emit('messageUnacknowledger.error', error);
super.handleError(err);
}
add(message, cause) {
if (!this.isOperational()) {
this.logger.warn(`Cannot add message ${message.getId()} - not operational`);
return;
}
if (!this.useBatchUnacks) {
this.unacknowledgeMessage(message, cause, (err, result) => {
if (err)
this.handleError(err);
if (result) {
this.emit('messageUnacknowledger.messagesUnacknowledged', result);
}
});
return;
}
const resolution = this.getResolution(message, cause);
this.logger.debug(`Adding message ${message.getId()} to unacknowledger, action: ${resolution.action}`);
this.batch.messages.push({
message,
resolution,
});
if (this.batch.messages.length >= this.batchSize) {
this.logger.debug(`Batch full, flushing immediately`);
this.processBatch();
}
else if (!this.batch.timer) {
this.logger.debug(`Setting timer: ${this.batchTimeoutMs}ms`);
this.batch.timer = setTimeout(() => {
if (this.batch.messages.length > 0) {
this.logger.debug(`Timer expired, flushing`);
this.processBatch();
}
}, this.batchTimeoutMs);
this.batch.timer.unref();
}
}
unacknowledgeMessage(message, cause, cb) {
if (!this.isOperational()) {
return cb(new PanicError({ message: 'MessageUnacknowledger not operational' }));
}
const resolution = this.getResolution(message, cause);
this.logger.debug(`Immediately unacknowledging message ${message.getId()}`);
_executeUnacknowledgementScript(this.queue, [
{
message,
resolution,
},
], this.consumerId, this.logger, cb);
}
unacknowledgeProcessingQueue(cause, cb) {
if (this.isDown() && !this.isGoingUp()) {
return cb(new PanicError({ message: 'MessageUnacknowledger is fully down' }));
}
this.logger.info(`Unacknowledging all messages in processing queue ${this.queueRef}`);
async.waterfall([
(next) => {
withSharedPoolConnection((client, done) => {
this.getMessagesFromQueue(client, done);
}, next);
},
(messages, next) => {
const messageCount = messages?.length || 0;
this.logger.debug(`Found ${messageCount} messages in processing queue ${this.queueRef}`);
const pending = this.buildPendingMessages(messages || [], cause);
_executeUnacknowledgementScript(this.queue, pending, this.consumerId, this.logger, next);
},
], (err, result) => {
if (err) {
this.logger.error(`Failed to unacknowledge queue ${this.queueRef}: ${err.message}`);
}
if (result) {
const unackedCount = Object.keys(result).length;
this.logger.info(`Unacknowledged ${unackedCount} messages from processing queue ${this.queueRef}`);
this.emit('messageUnacknowledger.messagesUnacknowledged', result);
}
cb(err, result);
});
}
flush(cb) {
if (this.isDown() && !this.isGoingUp()) {
return cb?.(new PanicError({ message: 'MessageUnacknowledger is fully down' }));
}
if (this.batch.messages.length === 0) {
this.logger.debug('No pending messages to flush');
cb?.(null, {});
return;
}
this.logger.debug(`Flushing ${this.batch.messages.length} pending messages`);
if (cb)
this.batch.callbacks.push(cb);
this.processBatch();
}
}
//# sourceMappingURL=message-unacknowledger.js.map