redis-smq
Version:
A simple high-performance Redis message queue for Node.js.
258 lines • 10.3 kB
JavaScript
import * as os from 'os';
import { CallbackEmptyReplyError, PanicError, Runnable, Timer, } from 'redis-smq-common';
import { ELuaScriptName } from '../../../../common/redis-client/scripts/scripts.js';
import { redisKeys } from '../../../../common/redis-keys/redis-keys.js';
import { Configuration } from '../../../../config/index.js';
import { _saveConsumerGroup } from '../../../consumer-groups/_/_save-consumer-group.js';
import { EventBus } from '../../../event-bus/index.js';
import { _hasRateLimitExceeded } from '../../../queue-rate-limit/_/_has-rate-limit-exceeded.js';
import { _getQueueProperties } from '../../../queue/_/_get-queue-properties.js';
import { EQueueDeliveryModel, EQueueType, QueueQueueNotFoundError, } from '../../../queue/index.js';
import { ConsumerConsumerGroupIdNotSupportedError, ConsumerConsumerGroupIdRequiredError, } from '../../errors/index.js';
import { eventBusPublisher } from './event-bus-publisher.js';
const IPAddresses = (() => {
const nets = os.networkInterfaces();
const addresses = [];
for (const netInterface in nets) {
const addr = nets[netInterface] ?? [];
for (const netAddr of addr) {
if (netAddr.family === 'IPv4' && !netAddr.internal) {
addresses.push(netAddr.address);
}
}
}
return addresses;
})();
export class DequeueMessage extends Runnable {
redisClient;
queue;
consumerId;
timer;
keyQueues;
keyQueueConsumers;
keyConsumerQueues;
keyQueueProcessingQueues;
keyQueueProcessing;
keyQueuePending;
keyQueuePriorityPending;
logger;
blockUntilMessageReceived;
autoCloseRedisConnection;
eventBus;
queueRateLimit = null;
queueType = null;
idleThreshold = 5;
idleTrigger = 0;
constructor(redisClient, queue, consumer, logger, eventBus, blockUntilMessageReceived = true, autoCloseRedisConnection = true) {
super();
this.queue = queue;
this.consumerId = consumer.getId();
this.logger = logger;
this.blockUntilMessageReceived = blockUntilMessageReceived;
this.autoCloseRedisConnection = autoCloseRedisConnection;
this.redisClient = redisClient;
this.redisClient.on('error', (err) => this.handleError(err));
if (!eventBus) {
this.eventBus = new EventBus();
this.eventBus.on('error', (err) => this.handleError(err));
}
else
this.eventBus = eventBus;
if (Configuration.getSetConfig().eventBus.enabled) {
eventBusPublisher(this, this.eventBus, logger);
}
const { keyConsumerQueues } = redisKeys.getConsumerKeys(this.consumerId);
const { keyQueueProcessing } = redisKeys.getQueueConsumerKeys(this.queue.queueParams, this.consumerId);
const { keyQueues } = redisKeys.getMainKeys();
const { keyQueueProcessingQueues, keyQueuePending, keyQueuePriorityPending, keyQueueConsumers, } = redisKeys.getQueueKeys(this.queue.queueParams, this.queue.groupId);
this.keyQueuePriorityPending = keyQueuePriorityPending;
this.keyQueuePending = keyQueuePending;
this.keyQueueProcessing = keyQueueProcessing;
this.keyQueues = keyQueues;
this.keyQueueConsumers = keyQueueConsumers;
this.keyConsumerQueues = keyConsumerQueues;
this.keyQueueProcessingQueues = keyQueueProcessingQueues;
this.timer = new Timer();
this.timer.on('error', (err) => this.handleError(err));
}
getLogger() {
return this.logger;
}
handleError(err) {
if (this.isRunning()) {
this.emit('consumer.dequeueMessage.error', err, this.consumerId, this.queue);
}
super.handleError(err);
}
goingUp() {
return super.goingUp().concat([
this.redisClient.init,
(cb) => {
const consumerInfo = {
ipAddress: IPAddresses,
hostname: os.hostname(),
pid: process.pid,
createdAt: Date.now(),
};
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error) {
cb(redisClient);
return void 0;
}
redisClient.runScript(ELuaScriptName.INIT_CONSUMER_QUEUE, [
this.keyQueues,
this.keyQueueConsumers,
this.keyConsumerQueues,
this.keyQueueProcessingQueues,
], [
this.consumerId,
JSON.stringify(consumerInfo),
JSON.stringify(this.queue.queueParams),
this.keyQueueProcessing,
], (err, reply) => {
if (err)
cb(err);
else if (!reply)
cb(new QueueQueueNotFoundError());
else
cb();
});
},
(cb) => {
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error) {
cb(redisClient);
return void 0;
}
_getQueueProperties(redisClient, this.queue.queueParams, (err, queueProperties) => {
if (err)
cb(err);
else if (!queueProperties)
cb(new CallbackEmptyReplyError());
else {
this.queueType = queueProperties.queueType;
this.queueRateLimit = queueProperties.rateLimit ?? null;
const { queueParams, groupId } = this.queue;
if (queueProperties.deliveryModel ===
EQueueDeliveryModel.POINT_TO_POINT) {
if (groupId)
cb(new ConsumerConsumerGroupIdNotSupportedError());
else
cb();
}
else if (queueProperties.deliveryModel === EQueueDeliveryModel.PUB_SUB) {
if (!groupId)
cb(new ConsumerConsumerGroupIdRequiredError());
else {
const eventBus = this.eventBus.getInstance();
if (eventBus instanceof Error)
cb(eventBus);
else
_saveConsumerGroup(redisClient, eventBus, queueParams, groupId, (err) => cb(err));
}
}
else
cb(new PanicError('UNKNOWN_DELIVERY_MODEL'));
}
});
},
]);
}
goingDown() {
return [
(cb) => {
this.timer.reset();
if (!this.autoCloseRedisConnection)
return cb();
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error)
return cb();
redisClient.halt(cb);
},
].concat(super.goingDown());
}
updateIdle() {
this.idleTrigger = this.idleTrigger + 1;
}
resetIdle() {
this.idleTrigger = 0;
}
isIdle() {
return this.idleTrigger >= this.idleThreshold;
}
isPriorityQueuingEnabled() {
return this.queueType === EQueueType.PRIORITY_QUEUE;
}
handleMessage = (err, messageId) => {
if (err) {
this.timer.reset();
this.handleError(err);
}
else if (typeof messageId === 'string') {
this.resetIdle();
this.emit('consumer.dequeueMessage.messageReceived', messageId, this.queue, this.consumerId);
}
else {
this.updateIdle();
this.emit('consumer.dequeueMessage.nextMessage');
}
};
dequeueWithRateLimit = (redisClient) => {
if (this.queueRateLimit) {
_hasRateLimitExceeded(redisClient, this.queue.queueParams, this.queueRateLimit, (err, isExceeded) => {
if (err)
this.handleError(err);
else if (isExceeded)
this.timer.setTimeout(() => {
this.dequeue();
}, 1000);
else
this.dequeueWithRateLimitExec(redisClient);
});
return true;
}
return false;
};
dequeueWithRateLimitExec = (redisClient) => {
if (this.isPriorityQueuingEnabled())
this.dequeueWithPriority(redisClient);
else
this.dequeueAndReturn(redisClient);
};
dequeueWithPriority = (redisClient) => {
if (this.isPriorityQueuingEnabled()) {
redisClient.zpoprpush(this.keyQueuePriorityPending, this.keyQueueProcessing, this.handleMessage);
return true;
}
return false;
};
dequeueAndBlock = (redisClient) => {
if (this.blockUntilMessageReceived) {
redisClient.brpoplpush(this.keyQueuePending, this.keyQueueProcessing, 0, this.handleMessage);
return true;
}
return false;
};
dequeueAndReturn = (redisClient) => {
redisClient.rpoplpush(this.keyQueuePending, this.keyQueueProcessing, this.handleMessage);
};
dequeue() {
if (!this.isRunning())
return void 0;
if (this.isIdle()) {
this.resetIdle();
return void this.timer.setTimeout(() => {
this.dequeue();
}, 1000);
}
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error) {
return this.handleError(redisClient);
}
return void (this.dequeueWithRateLimit(redisClient) ||
this.dequeueWithPriority(redisClient) ||
this.dequeueAndBlock(redisClient) ||
this.dequeueAndReturn(redisClient));
}
}
//# sourceMappingURL=dequeue-message.js.map