redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
189 lines • 10.5 kB
JavaScript
import { async, PanicError } from 'redis-smq-common';
import { ERedisScriptName } from '../../../../common/redis/scripts.js';
import { redisKeys } from '../../../../common/redis/redis-keys/redis-keys.js';
import { _fromMessage } from '../../../../message-manager/_/_from-message.js';
import { _getMessages } from '../../../../message-manager/_/_get-message.js';
import { EMessageProperty, EMessagePropertyStatus, } from '../../../../message/index.js';
import { EQueueOperationalState, EQueueProperty, EQueueType, } from '../../../../queue-manager/index.js';
import { withSharedPoolConnection } from '../../../../common/redis/redis-connection-pool/with-shared-pool-connection.js';
import { UnexpectedScriptReplyError } from '../../../../errors/index.js';
import { QueueWorkerAbstract } from '../queue-worker-abstract.js';
export class PublishScheduledWorker extends QueueWorkerAbstract {
fetchMessageIds = (cb) => {
withSharedPoolConnection((redisClient, cb) => {
const { keyQueueScheduled } = redisKeys.getQueueKeys(this.queueParsedParams.queueParams.ns, this.queueParsedParams.queueParams.name, this.queueParsedParams.groupId);
const currentTimestamp = Date.now();
redisClient.zrangebyscore(keyQueueScheduled, 0, currentTimestamp, 0, 99, (err, ids) => {
if (err) {
this.logger.error('Error fetching scheduled message IDs', err);
return cb(err);
}
cb(null, ids || []);
});
}, cb);
};
fetchMessages = (ids, cb) => {
if (!ids.length) {
cb(null, []);
return;
}
this.logger.debug(`Fetching ${ids.length} messages from storage`);
withSharedPoolConnection((redisClient, cb) => {
_getMessages(redisClient, ids, (err, messages) => {
if (err) {
this.logger.error('Error fetching messages', err);
cb(err);
}
else {
const messageCount = messages?.length || 0;
this.logger.debug(`Successfully retrieved ${messageCount} messages`);
cb(null, messages || []);
}
});
}, cb);
};
enqueueMessages = (messages, cb) => {
if (!messages.length)
return cb();
this.logger.debug(`Preparing to enqueue ${messages.length} messages`);
withSharedPoolConnection((redisClient, cb) => {
const { keyQueueProperties, keyQueuePending, keyQueuePriority, keyQueueScheduled, keyQueuePublished, keyQueueDeadLetter, keyQueueConsumerGroups, } = redisKeys.getQueueKeys(this.queueParsedParams.queueParams.ns, this.queueParsedParams.queueParams.name, this.queueParsedParams.groupId);
const keys = [
keyQueueProperties,
keyQueuePending,
keyQueuePublished,
keyQueuePriority,
keyQueueScheduled,
keyQueueDeadLetter,
keyQueueConsumerGroups,
];
const argv = [
EQueueProperty.QUEUE_TYPE,
EQueueProperty.MESSAGES_COUNT,
EQueueProperty.PENDING_MESSAGES_COUNT,
EQueueProperty.SCHEDULED_MESSAGES_COUNT,
EQueueProperty.DEAD_LETTERED_MESSAGES_COUNT,
EQueueType.PRIORITY_QUEUE,
EQueueType.LIFO_QUEUE,
EQueueType.FIFO_QUEUE,
EQueueProperty.OPERATIONAL_STATE,
EQueueProperty.LOCK_ID,
EQueueOperationalState.ACTIVE,
EQueueOperationalState.PAUSED,
EQueueOperationalState.STOPPED,
EQueueOperationalState.LOCKED,
EMessagePropertyStatus.PENDING,
EMessagePropertyStatus.SCHEDULED,
EMessagePropertyStatus.DEAD_LETTERED,
EMessageProperty.ID,
EMessageProperty.STATUS,
EMessageProperty.MESSAGE,
EMessageProperty.SCHEDULED_AT,
EMessageProperty.PUBLISHED_AT,
EMessageProperty.PROCESSING_STARTED_AT,
EMessageProperty.DEAD_LETTERED_AT,
EMessageProperty.ACKNOWLEDGED_AT,
EMessageProperty.UNACKNOWLEDGED_AT,
EMessageProperty.LAST_UNACKNOWLEDGED_AT,
EMessageProperty.LAST_SCHEDULED_AT,
EMessageProperty.REQUEUED_AT,
EMessageProperty.REQUEUE_COUNT,
EMessageProperty.LAST_REQUEUED_AT,
EMessageProperty.LAST_RETRIED_ATTEMPT_AT,
EMessageProperty.SCHEDULED_CRON_FIRED,
EMessageProperty.ATTEMPTS,
EMessageProperty.SCHEDULED_REPEAT_COUNT,
EMessageProperty.EXPIRED,
EMessageProperty.EFFECTIVE_SCHEDULED_DELAY,
EMessageProperty.SCHEDULED_TIMES,
EMessageProperty.SCHEDULED_MESSAGE_PARENT_ID,
EMessageProperty.REQUEUED_MESSAGE_PARENT_ID,
EMessageProperty.LAST_PROCESSED_AT,
];
async.eachOf(messages, (msg, index, done) => {
const messageId = msg.getId();
this.logger.debug(`Processing message ${messageId} (${index + 1}/${messages.length})`);
const ts = Date.now();
const scheduledMessageId = msg.getId();
const consumerGroupId = msg.getConsumerGroupId();
const { keyMessage: keyScheduledMessage } = redisKeys.getMessageKeys(scheduledMessageId);
const nextScheduleTimestamp = msg.getNextScheduledTimestamp();
const scheduledMessageState = msg.getMessageState();
const messagePriority = msg.producibleMessage.getPriority() ?? '';
const scheduledMessageCronFired = Number(scheduledMessageState.isScheduledCronFired());
const scheduledMessageEffectiveScheduledDelay = Number(scheduledMessageState.getEffectiveScheduledDelay());
const scheduledMessageRepeatCount = scheduledMessageState.getScheduledRepeatCount();
let newMessageId = '';
let newMessageJSON = '';
let newMessagePublishedAt = '';
let newKeyMessage = '';
if (nextScheduleTimestamp) {
const newMessage = _fromMessage(msg);
newMessage.producibleMessage.resetScheduledParams();
const newMessageState = newMessage
.getMessageState()
.setPublishedAt(ts)
.setScheduledMessageParentId(scheduledMessageId);
newMessageId = newMessageState.getId();
newKeyMessage = redisKeys.getMessageKeys(newMessageId).keyMessage;
newMessageJSON = JSON.stringify(newMessage.toJSON());
newMessagePublishedAt = ts;
scheduledMessageState.setLastScheduledAt(ts).incrScheduledTimes();
}
else {
scheduledMessageState.setPublishedAt(ts);
}
const scheduledMessageScheduledTimes = scheduledMessageState.getScheduledTimes();
const scheduledMessageLastScheduledAt = scheduledMessageState.getLastScheduledAt() ?? '';
const scheduledMessagePublishedAt = scheduledMessageState.getPublishedAt() ?? '';
keys.push(newKeyMessage, keyScheduledMessage);
argv.push(newMessageId, newMessageJSON, messagePriority, newMessagePublishedAt, scheduledMessageId, nextScheduleTimestamp, scheduledMessageLastScheduledAt, scheduledMessageScheduledTimes, scheduledMessagePublishedAt, scheduledMessageCronFired, scheduledMessageRepeatCount, scheduledMessageEffectiveScheduledDelay, consumerGroupId ?? '', ts);
done();
}, (err) => {
if (err) {
this.logger.error('Error during message processing for enqueue', err);
return cb(err);
}
this.logger.debug(`Executing PUBLISH_SCHEDULED_MESSAGE script for ${messages.length} messages`);
redisClient.runScript(ERedisScriptName.PUBLISH_SCHEDULED, keys, argv, (err, reply) => {
if (err) {
this.logger.error('Error executing publish scheduled message script', err);
return cb(err);
}
if (typeof reply === 'string') {
if (reply.startsWith('PUBLISH_ERROR:')) {
const [, failedMessageId, errorCode] = reply.split(':');
this.logger.error(`Failed to publish repeating message ${failedMessageId}: ${errorCode}`);
return cb(new PanicError({
message: `Failed to publish scheduled message: ${errorCode}`,
}));
}
this.logger.error(`Script execution returned an error: ${reply}`);
return cb(new PanicError({
message: `PUBLISH_SCHEDULED_MESSAGE script failed: ${reply}`,
}));
}
if (typeof reply === 'number') {
if (reply !== messages.length) {
this.logger.warn(`Script reported processing ${reply} messages, but expected ${messages.length}.`);
}
this.logger.info(`Successfully published ${reply} scheduled messages.`);
return cb();
}
this.logger.error(`Script execution returned unexpected response: ${reply}`);
cb(new UnexpectedScriptReplyError({ metadata: { reply } }));
});
});
}, cb);
};
work = (cb) => {
async.waterfall([this.fetchMessageIds, this.fetchMessages, this.enqueueMessages], (err) => {
if (err) {
this.logger.error('Error in publish scheduled messages workflow', err);
}
cb(err);
});
};
}
export default PublishScheduledWorker;
//# sourceMappingURL=publish-scheduled.worker.js.map