redis-smq
Version:
A simple high-performance Redis message queue for Node.js.
282 lines • 12.2 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.DequeueMessage = void 0;
const os = __importStar(require("os"));
const redis_smq_common_1 = require("redis-smq-common");
const scripts_js_1 = require("../../../../common/redis-client/scripts/scripts.js");
const redis_keys_js_1 = require("../../../../common/redis-keys/redis-keys.js");
const index_js_1 = require("../../../../config/index.js");
const _save_consumer_group_js_1 = require("../../../consumer-groups/_/_save-consumer-group.js");
const index_js_2 = require("../../../event-bus/index.js");
const _has_rate_limit_exceeded_js_1 = require("../../../queue-rate-limit/_/_has-rate-limit-exceeded.js");
const _get_queue_properties_js_1 = require("../../../queue/_/_get-queue-properties.js");
const index_js_3 = require("../../../queue/index.js");
const index_js_4 = require("../../errors/index.js");
const event_bus_publisher_js_1 = require("./event-bus-publisher.js");
const IPAddresses = (() => {
var _a;
const nets = os.networkInterfaces();
const addresses = [];
for (const netInterface in nets) {
const addr = (_a = nets[netInterface]) !== null && _a !== void 0 ? _a : [];
for (const netAddr of addr) {
if (netAddr.family === 'IPv4' && !netAddr.internal) {
addresses.push(netAddr.address);
}
}
}
return addresses;
})();
class DequeueMessage extends redis_smq_common_1.Runnable {
constructor(redisClient, queue, consumer, logger, eventBus, blockUntilMessageReceived = true, autoCloseRedisConnection = true) {
super();
this.queueRateLimit = null;
this.queueType = null;
this.idleThreshold = 5;
this.idleTrigger = 0;
this.handleMessage = (err, messageId) => {
if (err) {
this.timer.reset();
this.handleError(err);
}
else if (typeof messageId === 'string') {
this.resetIdle();
this.emit('consumer.dequeueMessage.messageReceived', messageId, this.queue, this.consumerId);
}
else {
this.updateIdle();
this.emit('consumer.dequeueMessage.nextMessage');
}
};
this.dequeueWithRateLimit = (redisClient) => {
if (this.queueRateLimit) {
(0, _has_rate_limit_exceeded_js_1._hasRateLimitExceeded)(redisClient, this.queue.queueParams, this.queueRateLimit, (err, isExceeded) => {
if (err)
this.handleError(err);
else if (isExceeded)
this.timer.setTimeout(() => {
this.dequeue();
}, 1000);
else
this.dequeueWithRateLimitExec(redisClient);
});
return true;
}
return false;
};
this.dequeueWithRateLimitExec = (redisClient) => {
if (this.isPriorityQueuingEnabled())
this.dequeueWithPriority(redisClient);
else
this.dequeueAndReturn(redisClient);
};
this.dequeueWithPriority = (redisClient) => {
if (this.isPriorityQueuingEnabled()) {
redisClient.zpoprpush(this.keyQueuePriorityPending, this.keyQueueProcessing, this.handleMessage);
return true;
}
return false;
};
this.dequeueAndBlock = (redisClient) => {
if (this.blockUntilMessageReceived) {
redisClient.brpoplpush(this.keyQueuePending, this.keyQueueProcessing, 0, this.handleMessage);
return true;
}
return false;
};
this.dequeueAndReturn = (redisClient) => {
redisClient.rpoplpush(this.keyQueuePending, this.keyQueueProcessing, this.handleMessage);
};
this.queue = queue;
this.consumerId = consumer.getId();
this.logger = logger;
this.blockUntilMessageReceived = blockUntilMessageReceived;
this.autoCloseRedisConnection = autoCloseRedisConnection;
this.redisClient = redisClient;
this.redisClient.on('error', (err) => this.handleError(err));
if (!eventBus) {
this.eventBus = new index_js_2.EventBus();
this.eventBus.on('error', (err) => this.handleError(err));
}
else
this.eventBus = eventBus;
if (index_js_1.Configuration.getSetConfig().eventBus.enabled) {
(0, event_bus_publisher_js_1.eventBusPublisher)(this, this.eventBus, logger);
}
const { keyConsumerQueues } = redis_keys_js_1.redisKeys.getConsumerKeys(this.consumerId);
const { keyQueueProcessing } = redis_keys_js_1.redisKeys.getQueueConsumerKeys(this.queue.queueParams, this.consumerId);
const { keyQueues } = redis_keys_js_1.redisKeys.getMainKeys();
const { keyQueueProcessingQueues, keyQueuePending, keyQueuePriorityPending, keyQueueConsumers, } = redis_keys_js_1.redisKeys.getQueueKeys(this.queue.queueParams, this.queue.groupId);
this.keyQueuePriorityPending = keyQueuePriorityPending;
this.keyQueuePending = keyQueuePending;
this.keyQueueProcessing = keyQueueProcessing;
this.keyQueues = keyQueues;
this.keyQueueConsumers = keyQueueConsumers;
this.keyConsumerQueues = keyConsumerQueues;
this.keyQueueProcessingQueues = keyQueueProcessingQueues;
this.timer = new redis_smq_common_1.Timer();
this.timer.on('error', (err) => this.handleError(err));
}
getLogger() {
return this.logger;
}
handleError(err) {
if (this.isRunning()) {
this.emit('consumer.dequeueMessage.error', err, this.consumerId, this.queue);
}
super.handleError(err);
}
goingUp() {
return super.goingUp().concat([
this.redisClient.init,
(cb) => {
const consumerInfo = {
ipAddress: IPAddresses,
hostname: os.hostname(),
pid: process.pid,
createdAt: Date.now(),
};
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error) {
cb(redisClient);
return void 0;
}
redisClient.runScript(scripts_js_1.ELuaScriptName.INIT_CONSUMER_QUEUE, [
this.keyQueues,
this.keyQueueConsumers,
this.keyConsumerQueues,
this.keyQueueProcessingQueues,
], [
this.consumerId,
JSON.stringify(consumerInfo),
JSON.stringify(this.queue.queueParams),
this.keyQueueProcessing,
], (err, reply) => {
if (err)
cb(err);
else if (!reply)
cb(new index_js_3.QueueQueueNotFoundError());
else
cb();
});
},
(cb) => {
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error) {
cb(redisClient);
return void 0;
}
(0, _get_queue_properties_js_1._getQueueProperties)(redisClient, this.queue.queueParams, (err, queueProperties) => {
var _a;
if (err)
cb(err);
else if (!queueProperties)
cb(new redis_smq_common_1.CallbackEmptyReplyError());
else {
this.queueType = queueProperties.queueType;
this.queueRateLimit = (_a = queueProperties.rateLimit) !== null && _a !== void 0 ? _a : null;
const { queueParams, groupId } = this.queue;
if (queueProperties.deliveryModel ===
index_js_3.EQueueDeliveryModel.POINT_TO_POINT) {
if (groupId)
cb(new index_js_4.ConsumerConsumerGroupIdNotSupportedError());
else
cb();
}
else if (queueProperties.deliveryModel === index_js_3.EQueueDeliveryModel.PUB_SUB) {
if (!groupId)
cb(new index_js_4.ConsumerConsumerGroupIdRequiredError());
else {
const eventBus = this.eventBus.getInstance();
if (eventBus instanceof Error)
cb(eventBus);
else
(0, _save_consumer_group_js_1._saveConsumerGroup)(redisClient, eventBus, queueParams, groupId, (err) => cb(err));
}
}
else
cb(new redis_smq_common_1.PanicError('UNKNOWN_DELIVERY_MODEL'));
}
});
},
]);
}
goingDown() {
return [
(cb) => {
this.timer.reset();
if (!this.autoCloseRedisConnection)
return cb();
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error)
return cb();
redisClient.halt(cb);
},
].concat(super.goingDown());
}
updateIdle() {
this.idleTrigger = this.idleTrigger + 1;
}
resetIdle() {
this.idleTrigger = 0;
}
isIdle() {
return this.idleTrigger >= this.idleThreshold;
}
isPriorityQueuingEnabled() {
return this.queueType === index_js_3.EQueueType.PRIORITY_QUEUE;
}
dequeue() {
if (!this.isRunning())
return void 0;
if (this.isIdle()) {
this.resetIdle();
return void this.timer.setTimeout(() => {
this.dequeue();
}, 1000);
}
const redisClient = this.redisClient.getInstance();
if (redisClient instanceof Error) {
return this.handleError(redisClient);
}
return void (this.dequeueWithRateLimit(redisClient) ||
this.dequeueWithPriority(redisClient) ||
this.dequeueAndBlock(redisClient) ||
this.dequeueAndReturn(redisClient));
}
}
exports.DequeueMessage = DequeueMessage;
//# sourceMappingURL=dequeue-message.js.map