@imqueue/core
Version:
Simple JSON-based messaging queue for inter service communication
1,278 lines • 42.7 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.RedisQueue = exports.IMQ_SHUTDOWN_TIMEOUT = exports.DEFAULT_IMQ_OPTIONS = void 0;
exports.sha1 = sha1;
exports.intrand = intrand;
exports.pack = pack;
exports.unpack = unpack;
/*!
* Fast messaging queue over Redis
*
* I'm Queue Software Project
* Copyright (C) 2025 imqueue.com <support@imqueue.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* If you want to use this code in a closed source (commercial) project, you can
* purchase a proprietary commercial license. Please contact us at
* <support@imqueue.com> to get commercial licensing options.
*/
const crypto = require("crypto");
const events_1 = require("events");
const os = require("os");
const zlib_1 = require("zlib");
const _1 = require(".");
const redis_1 = require("./redis");
const RX_CLIENT_NAME = /name=(\S+)/g;
const RX_CLIENT_TEST = /:(reader|writer|watcher)/;
const RX_CLIENT_CLEAN = /:(reader|writer|watcher).*$/;
exports.DEFAULT_IMQ_OPTIONS = {
host: 'localhost',
port: 6379,
cleanup: false,
cleanupFilter: '*',
logger: console,
prefix: 'imq',
safeDelivery: false,
safeDeliveryTtl: 5000,
useGzip: false,
watcherCheckDelay: 5000,
};
exports.IMQ_SHUTDOWN_TIMEOUT = +(process.env.IMQ_SHUTDOWN_TIMEOUT || 1000);
/**
* Returns SHA1 hash sum of the given string
*
* @param {string} str
* @returns {string}
*/
function sha1(str) {
const sha = crypto.createHash('sha1');
sha.update(str);
return sha.digest('hex');
}
/**
* Returns random integer between given min and max
*
* @param {number} min
* @param {number} max
* @returns {number}
*/
function intrand(min, max) {
return Math.floor(Math.random() * (max - min + 1)) + min;
}
/**
* Compress given data and returns binary string
*
* @param {any} data
* @returns {string}
*/
// istanbul ignore next
function pack(data) {
return (0, zlib_1.gzipSync)(JSON.stringify(data)).toString('binary');
}
/**
* Decompress binary string and returns plain data
*
* @param {string} data
* @returns {any}
*/
// istanbul ignore next
function unpack(data) {
return JSON.parse((0, zlib_1.gunzipSync)(Buffer.from(data, 'binary')).toString());
}
const IMQ_REDIS_MAX_LISTENERS_LIMIT = +(process.env.IMQ_REDIS_MAX_LISTENERS_LIMIT || 10000);
/**
* Class RedisQueue
* Implements simple messaging queue over redis.
*/
class RedisQueue extends events_1.EventEmitter {
/**
* @constructor
* @param {string} name
* @param {IMQOptions} [options]
* @param {IMQMode} [mode]
*/
constructor(name, options, mode = _1.IMQMode.BOTH) {
super();
this.name = name;
this.mode = mode;
/**
* Init state for this queue instance
*
* @type {boolean}
*/
this.initialized = false;
/**
* Signals if the queue was destroyed
*
* @type {boolean}
*/
this.destroyed = false;
/**
* True if the current instance owns a watcher connection, false otherwise
*
* @type {boolean}
*/
this.watchOwner = false;
/**
* Signals initialization state
*
* @type {boolean}
*/
this.signalsInitialized = false;
/**
* Internal per-channel reconnection state
*/
this.reconnectTimers = {};
this.reconnectAttempts = {};
this.reconnecting = {};
/**
* LUA scripts for redis
*
* @type {{moveDelayed: {code: string}}}
*/
// tslint:disable-next-line:completed-docs
this.scripts = {
moveDelayed: {
code: 'local messages = redis.call(' +
'"zrangebyscore", KEYS[1], "-inf", ARGV[1]) ' +
'local count = table.getn(messages) ' +
'local message ' +
'local i = 1 ' +
'if count > 0 then ' +
'while messages[i] do ' +
'redis.call("lpush", KEYS[2], messages[i]) ' +
'i = i + 1 ' +
'end ' +
'redis.call("zremrangebyscore", KEYS[1], ' +
'"-inf", ARGV[1]) ' +
'end ' +
'return count',
},
};
this.options = (0, _1.buildOptions)(exports.DEFAULT_IMQ_OPTIONS, options);
/* tslint:disable */
this.pack = this.options.useGzip ? pack : JSON.stringify;
this.unpack = this.options.useGzip ? unpack : JSON.parse;
/* tslint:enable */
this.redisKey = `${this.options.host}:${this.options.port}`;
this.verbose(`Initializing queue on ${this.options.host}:${this.options.port} with prefix ${this.options.prefix} and safeDelivery = ${this.options.safeDelivery}, and safeDeliveryTtl = ${this.options.safeDeliveryTtl}, and watcherCheckDelay = ${this.options.watcherCheckDelay}, and useGzip = ${this.options.useGzip}`);
}
verbose(message) {
if (this.options.verbose) {
this.logger.info(`[IMQ-CORE][${this.name}]: ${message}`);
}
}
/**
* Creates a subscription channel over redis and sets up channel
* data read handler
*
* @param {string} channel
* @param {(data: JsonObject) => any} handler
* @return {Promise<void>}
*/
async subscribe(channel, handler) {
// istanbul ignore next
if (!channel) {
throw new TypeError(`${channel}: No subscription channel name provided!`);
}
// istanbul ignore next
if (this.subscriptionName && this.subscriptionName !== channel) {
throw new TypeError(`Invalid channel name provided: expected "${this.subscriptionName}", but "${channel}" given instead!`);
}
else if (!this.subscriptionName) {
this.subscriptionName = channel;
}
const fcn = `${this.options.prefix}:${this.subscriptionName}`;
const chan = await this.connect('subscription', this.options);
await chan.subscribe(fcn);
// istanbul ignore next
chan.on('message', (ch, message) => {
if (ch === fcn && typeof handler === 'function') {
handler(JSON.parse(message));
}
this.verbose(`Received message from ${ch} channel, data: ${JSON.stringify(message)}`);
});
this.verbose(`Subscribed to ${channel} channel`);
}
/**
* Closes subscription channel
*
* @return {Promise<void>}
*/
async unsubscribe() {
if (this.subscription) {
this.verbose('Initialize unsubscribing...');
try {
if (this.subscriptionName) {
await this.subscription.unsubscribe(`${this.options.prefix}:${this.subscriptionName}`);
this.verbose(`Unsubscribed from ${this.subscriptionName} channel`);
}
this.subscription.removeAllListeners();
this.subscription.quit();
this.subscription.disconnect(false);
}
catch (error) {
this.verbose(`Unsubscribe error: ${error}`);
}
}
this.subscriptionName = undefined;
this.subscription = undefined;
}
/**
* Publishes a message to this queue subscription channel for currently
* subscribed clients.
*
* If toName specified will publish to PubSub with a different name. This
* can be used to implement broadcasting some messages to other subscribers
* on other PubSub channels.
*
* @param {string} [toName]
* @param {JsonObject} data
*/
async publish(data, toName) {
if (!this.writer) {
throw new TypeError('Writer is not connected!');
}
const jsonData = JSON.stringify(data);
const name = toName || this.name;
await this.writer.publish(`${this.options.prefix}:${name}`, jsonData);
this.verbose(`Published message to ${name} channel, data: ${jsonData}
`);
}
/**
* Initializes and starts current queue routines
*
* @returns {Promise<RedisQueue>}
*/
async start() {
if (!this.name) {
throw new TypeError(`${this.name}: No queue name provided!`);
}
if (this.initialized) {
return this;
}
this.destroyed = false;
const connPromises = [];
// istanbul ignore next
if (!this.reader && this.isWorker()) {
this.verbose('Initializing reader...');
connPromises.push(this.connect('reader', this.options));
}
if (!this.writer) {
this.verbose('Initializing writer...');
connPromises.push(this.connect('writer', this.options));
}
await Promise.all(connPromises);
this.verbose('Connections initialized');
if (!this.signalsInitialized) {
this.verbose('Setting up OS signal handlers...');
// istanbul ignore next
const free = async () => {
let exitCode = 0;
setTimeout(() => {
this.verbose(`Shutting down after ${exports.IMQ_SHUTDOWN_TIMEOUT} timeout`);
process.exit(exitCode);
}, exports.IMQ_SHUTDOWN_TIMEOUT);
try {
if (this.watchOwner) {
this.verbose('Freeing watcher lock...');
await this.unlock();
}
}
catch (err) {
this.logger.error(err);
exitCode = 1;
}
};
process.on('SIGTERM', free);
process.on('SIGINT', free);
process.on('SIGABRT', free);
this.signalsInitialized = true;
this.verbose('OS signal handlers initialized!');
}
await this.initWatcher();
this.initialized = true;
return this;
}
/**
* Sends a given message to a given queue (by name)
*
* @param {string} toQueue
* @param {JsonObject} message
* @param {number} [delay]
* @param {(err: Error) => void} [errorHandler]
* @returns {Promise<RedisQueue>}
*/
async send(toQueue, message, delay, errorHandler) {
if (!this.isPublisher()) {
throw new TypeError('IMQ: Unable to publish in WORKER only mode!');
}
// istanbul ignore next
if (!this.writer) {
await this.start();
}
if (!this.writer) {
throw new TypeError('IMQ: unable to initialize queue!');
}
const id = (0, _1.uuid)();
const data = { id, message, from: this.name };
const key = `${this.options.prefix}:${toQueue}`;
const packet = this.pack(data);
const cb = (error, op) => {
// istanbul ignore next
if (error) {
this.verbose(`Writer ${op} error: ${error}`);
if (errorHandler) {
errorHandler(error);
}
}
};
if (delay) {
this.writer.zadd(`${key}:delayed`, Date.now() + delay, packet, (err) => {
// istanbul ignore next
if (err) {
cb(err, 'ZADD');
return;
}
this.writer.set(`${key}:${id}:ttl`, '', 'PX', delay, 'NX', (err) => {
// istanbul ignore next
if (err) {
cb(err, 'SET');
return;
}
}).catch((err) => cb(err, 'SET'));
});
}
else {
this.writer.lpush(key, packet, (err) => {
// istanbul ignore next
if (err) {
cb(err, 'LPUSH');
return;
}
});
}
return id;
}
/**
* Stops current queue routines
*
* @returns {Promise<RedisQueue>}
*/
async stop() {
this.verbose('Stopping queue...');
if (this.reader) {
this.verbose('Destroying reader...');
this.destroyChannel('reader', this);
delete this.reader;
}
this.initialized = false;
this.verbose('Queue stopped!');
return this;
}
/**
* Gracefully destroys this queue
*
* @returns {Promise<void>}
*/
async destroy() {
this.verbose('Destroying queue...');
this.destroyed = true;
this.removeAllListeners();
this.cleanSafeCheckInterval();
this.destroyWatcher();
await this.stop();
await this.clear();
this.destroyWriter();
await this.unsubscribe();
this.verbose('Queue destroyed!');
}
/**
* Clears queue data in redis
*
* @returns {Promise<void>}
*/
async clear() {
if (!this.writer) {
return this;
}
try {
this.verbose('Clearing expired queue keys...');
await Promise.all([
this.writer.del(this.key),
this.writer.del(`${this.key}:delayed`),
]);
this.verbose('Expired queue keys cleared!');
}
catch (err) {
// istanbul ignore next
if (this.initialized) {
this.logger.error(`${context.name}: error clearing the redis queue host ${this.redisKey} on writer, pid ${process.pid}:`, err);
}
}
return this;
}
/**
* Retrieves the current count of messages in the queue
*
* @returns {Promise<number>}
*/
async queueLength() {
if (!this.writer) {
return 0;
}
return this.writer.llen(this.key);
}
/**
* Returns true if publisher mode is enabled on this queue, false otherwise.
*
* @return {boolean}
*/
isPublisher() {
return this.mode === _1.IMQMode.BOTH || this.mode === _1.IMQMode.PUBLISHER;
}
/**
* Returns true if worker mode is enabled on this queue, false otherwise.
*
* @return {boolean}
*/
isWorker() {
return this.mode === _1.IMQMode.BOTH || this.mode === _1.IMQMode.WORKER;
}
// noinspection JSMethodCanBeStatic
/**
* Writer connection associated with this queue instance
*
* @type {IRedisClient}
*/
get writer() {
return RedisQueue.writers[this.redisKey];
}
// noinspection JSUnusedLocalSymbols
/**
* Writer connection setter.
*
* @param {IRedisClient} conn
*/
// noinspection JSUnusedLocalSymbols,JSUnusedLocalSymbols
set writer(conn) {
RedisQueue.writers[this.redisKey] = conn;
}
/**
* Watcher connection instance associated with this queue instance
*
* @type {IRedisClient}
*/
get watcher() {
return RedisQueue.watchers[this.redisKey];
}
// noinspection JSUnusedLocalSymbols
/**
* Watcher setter sets the watcher connection property for this
* queue instance
*
* @param {IRedisClient} conn
*/
// noinspection JSUnusedLocalSymbols
set watcher(conn) {
RedisQueue.watchers[this.redisKey] = conn;
}
/**
* Logger instance associated with the current queue instance
* @type {ILogger}
*/
get logger() {
// istanbul ignore next
return this.options.logger || console;
}
/**
* Return a lock key for watcher connection
*
* @access private
* @returns {string}
*/
get lockKey() {
return `${this.options.prefix}:watch:lock`;
}
/**
* Returns current queue key
*
* @access private
* @returns {string}
*/
get key() {
return `${this.options.prefix}:${this.name}`;
}
/**
* Destroys watcher channel
*
* @access private
*/
destroyWatcher() {
if (this.watcher) {
this.verbose('Destroying watcher...');
this.destroyChannel('watcher', this);
delete RedisQueue.watchers[this.redisKey];
this.verbose('Watcher destroyed!');
}
}
/**
* Destroys writer channel
*
* @access private
*/
destroyWriter() {
if (this.writer) {
this.verbose('Destroying writer...');
this.destroyChannel('writer', this);
delete RedisQueue.writers[this.redisKey];
this.verbose('Writer destroyed!');
}
}
/**
* Destroys any channel
*
* @access private
*/
destroyChannel(channel, context = this) {
const client = context[channel];
if (client) {
try {
client.removeAllListeners();
client.quit().then(() => {
client.disconnect(false);
}).catch(e => {
this.verbose(`Error quitting ${channel}: ${e}`);
});
}
catch (error) {
this.verbose(`Error destroying ${channel}: ${error}`);
}
}
}
/**
* Establishes a given connection channel by its name
*
* @access private
* @param {RedisConnectionChannel} channel
* @param {IMQOptions} options
* @param {any} context
* @returns {Promise<IRedisClient>}
*/
async connect(channel, options, context = this) {
this.verbose(`Connecting to ${channel} channel...`);
// istanbul ignore next
if (context[channel]) {
return context[channel];
}
const redis = new redis_1.default({
// istanbul ignore next
port: options.port || 6379,
// istanbul ignore next
host: options.host || 'localhost',
// istanbul ignore next
username: options.username,
// istanbul ignore next
password: options.password,
connectionName: this.getChannelName(context.name + '', options.prefix || '', channel),
retryStrategy: this.retryStrategy(),
autoResubscribe: true,
enableOfflineQueue: true,
autoResendUnfulfilledCommands: true,
offlineQueue: true,
maxRetriesPerRequest: null,
enableReadyCheck: channel !== 'subscription',
lazyConnect: true,
});
context[channel] = redis;
context[channel].__imq = true;
for (const event of [
'wait',
'reconnecting',
'connecting',
'connect',
'close',
]) {
redis.on(event, () => {
context.verbose(`Redis Event fired: ${event}`);
});
}
redis.setMaxListeners(IMQ_REDIS_MAX_LISTENERS_LIMIT);
redis.on('error', this.onErrorHandler(context, channel));
redis.on('end', this.onCloseHandler(context, channel));
await redis.connect();
this.logger.info('%s: %s channel connected, host %s, pid %s', context.name, channel, this.redisKey, process.pid);
switch (channel) {
case 'reader':
this.read();
break;
case 'writer':
await this.processDelayed(this.key);
break;
case 'watcher':
await this.initWatcher();
break;
}
return context[channel];
}
// istanbul ignore next
/**
* Builds and returns redis reconnection strategy
*
* @returns {() => (number | void | null)}
* @private
*/
retryStrategy() {
return () => {
return null;
};
}
/**
* Schedules custom reconnection for a given channel with capped
* exponential backoff
*
* @param {RedisConnectionChannel} channel
* @private
*/
scheduleReconnect(channel) {
if (this.destroyed) {
return;
}
if (this.reconnecting[channel]) {
return;
}
this.reconnecting[channel] = true;
const attempts = (this.reconnectAttempts[channel] || 0) + 1;
this.reconnectAttempts[channel] = attempts;
const base = Math.min(30000, 1000 * Math.pow(2, attempts - 1));
const jitter = Math.floor(base * 0.2 * Math.random());
const delay = base + jitter;
this.verbose(`Scheduling ${channel} reconnect in ${delay} ms (attempt ${attempts})`);
if (this.reconnectTimers[channel]) {
clearTimeout(this.reconnectTimers[channel]);
}
this.reconnectTimers[channel] = setTimeout(async () => {
if (this.destroyed) {
this.reconnecting[channel] = false;
return;
}
try {
switch (channel) {
case 'watcher':
this.destroyWatcher();
break;
case 'writer':
this.destroyWriter();
break;
case 'reader':
this.destroyChannel(channel, this);
this.reader = undefined;
break;
case 'subscription':
this.destroyChannel(channel, this);
this.subscription = undefined;
break;
}
await this.connect(channel, this.options);
this.reconnectAttempts[channel] = 0;
this.reconnecting[channel] = false;
if (this.reconnectTimers[channel]) {
clearTimeout(this.reconnectTimers[channel]);
this.reconnectTimers[channel] = undefined;
}
this.verbose(`Reconnected ${channel} channel`);
}
catch (err) {
this.reconnecting[channel] = false;
this.verbose(`Reconnect ${channel} failed: ${err}`);
this.scheduleReconnect(channel);
}
}, delay);
}
// noinspection JSMethodCanBeStatic
/**
* Generates channel name
*
* @param {string} contextName
* @param {string} prefix
* @param {RedisConnectionChannel} name
* @return {string}
*/
getChannelName(contextName, prefix, name) {
const uniqueSuffix = `pid:${process.pid}:host:${os.hostname()}`;
return `${prefix}:${contextName}:${name}:${uniqueSuffix}`;
}
/**
* Builds and returns connection error handler
*
* @access private
* @param {RedisQueue} context
* @param {RedisConnectionChannel} channel
* @return {(err: Error) => void}
*/
onErrorHandler(context, channel) {
// istanbul ignore next
return ((error) => {
var _a;
this.verbose(`Redis Error: ${error}`);
if (this.destroyed) {
return;
}
this.logger.error(`${context.name}: error connecting redis host ${this.redisKey} on ${channel}, pid ${process.pid}:`, error);
if (error.code === 'ECONNREFUSED' ||
error.code === 'ETIMEDOUT' ||
((_a = context[channel]) === null || _a === void 0 ? void 0 : _a.status) !== 'ready') {
this.scheduleReconnect(channel);
}
});
}
/**
* Builds and returns redis connection close handler
*
* @access private
* @param {RedisQueue} context
* @param {RedisConnectionChannel} channel
* @return {(...args: any[]) => any}
*/
onCloseHandler(context, channel) {
this.verbose(`Redis ${channel} is closing...`);
// istanbul ignore next
return (() => {
this.initialized = false;
this.logger.warn('%s: redis connection %s closed on host %s, pid %s!', context.name, channel, this.redisKey, process.pid);
if (!this.destroyed) {
this.scheduleReconnect(channel);
}
});
}
/**
* Processes given redis-queue message
*
* @access private
* @param {[any, any]} msg
* @returns {RedisQueue}
*/
process(msg) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const [queue, data] = msg;
// istanbul ignore next
if (!queue || queue !== this.key) {
return this;
}
try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-argument
const { id, message, from } = this.unpack(data);
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
this.emit('message', message, id, from);
}
catch (err) {
// istanbul ignore next
this.emitError('OnMessage', 'process error - message is invalid', err);
}
return this;
}
/**
* Returns the number of established watcher connections on redis
*
* @access private
* @returns {Promise<number>}
*/
// istanbul ignore next
async watcherCount() {
if (!this.writer) {
return 0;
}
const rx = new RegExp(`\\bname=${this.options.prefix}:[\\S]+?:watcher:`);
const list = await this.writer.client('LIST');
if (!list || !list.split) {
return 0;
}
return list.split(/\r?\n/).filter(client => rx.test(client)).length;
}
/**
* Processes delayed a message by its given redis key
*
* @access private
* @param {string} key
* @returns {Promise<void>}
*/
async processDelayed(key) {
try {
if (this.scripts.moveDelayed.checksum) {
await this.writer.evalsha(this.scripts.moveDelayed.checksum, 2, `${key}:delayed`, key, Date.now());
}
}
catch (err) {
this.emitError('OnProcessDelayed', 'error processing delayed queue', err);
}
}
// istanbul ignore next
/**
* Watch routine
*
* @access private
* @return {Promise<void>}
*/
async processWatch() {
const now = Date.now();
let cursor = '0';
while (true) {
try {
const data = await this.writer.scan(cursor, 'MATCH', `${this.options.prefix}:*:worker:*`, 'COUNT', '1000');
cursor = data.shift();
const keys = data.shift() || [];
await this.processKeys(keys, now);
if (cursor === '0') {
return;
}
}
catch (err) {
this.emitError('OnSafeDelivery', 'safe queue message delivery problem', err);
this.cleanSafeCheckInterval();
return;
}
}
}
// istanbul ignore next
/**
* Process given keys from a message queue
*
* @access private
* @param {string[]} keys
* @param {number} now
* @return {Promise<void>}
*/
async processKeys(keys, now) {
if (!keys.length) {
return;
}
this.verbose(`Watching ${keys.length} keys: ${keys.map(key => `"${key}"`).join(', ')}`);
for (const key of keys) {
const kp = key.split(':');
if (Number(kp.pop()) < now) {
continue;
}
await this.writer.rpoplpush(key, `${kp.shift()}:${kp.shift()}`);
}
}
// istanbul ignore next
/**
* Watch message processor
*
* @access private
* @param {...any[]} args
* @return {Promise<void>}
*/
async onWatchMessage(...args) {
try {
const key = (args.pop() || '').split(':');
if (key.pop() !== 'ttl') {
return;
}
key.pop(); // msg id
await this.processDelayed(key.join(':'));
}
catch (err) {
this.emitError('OnWatch', 'watch error', err);
}
}
// istanbul ignore next
/**
* Clears safe check interval
*
* @access private
*/
cleanSafeCheckInterval() {
if (this.safeCheckInterval) {
clearInterval(this.safeCheckInterval);
delete this.safeCheckInterval;
}
}
/**
* Setups watch a process on delayed messages
*
* @access private
* @returns {RedisQueue}
*/
// istanbul ignore next
watch() {
if (!this.writer || !this.watcher || this.watcher.__ready__) {
return this;
}
try {
this.writer.config('SET', 'notify-keyspace-events', 'Ex').catch(err => {
this.emitError('OnConfig', 'events config error', err);
});
}
catch (err) {
this.emitError('OnConfig', 'events config error', err);
}
this.watcher.on('pmessage', this.onWatchMessage.bind(this));
this.watcher.psubscribe('__keyevent@0__:expired', `${this.options.prefix}:delayed:*`).catch(err => {
this.verbose(`Error subscribing to watcher channel: ${err}`);
});
// watch for expired unhandled safe queues
if (!this.safeCheckInterval) {
if (this.options.safeDeliveryTtl != null) {
this.safeCheckInterval = setInterval((async () => {
if (!this.writer) {
this.cleanSafeCheckInterval();
return;
}
if (this.options.safeDelivery) {
await this.processWatch();
}
await this.processCleanup();
}), this.options.safeDeliveryTtl);
}
}
this.watcher.__ready__ = true;
return this;
}
/**
* Cleans up orphaned keys from redis
*
* @access private
* @returns {Promise<RedisQueue | undefined>}
*/
async processCleanup() {
this.verbose('Cleaning up orphaned keys...');
try {
if (!this.options.cleanup) {
return;
}
const filter = new RegExp(this.options.prefix + ':' +
(this.options.cleanupFilter || '*').replace(/\*/g, '.*'), 'i');
this.verbose(`Cleaning up keys matching ${filter}`);
const clients = (await this.writer.client('LIST')).toString() || '';
const connectedKeys = (clients.match(RX_CLIENT_NAME) || [])
.filter((name) => RX_CLIENT_TEST.test(name) && filter.test(name))
.map((name) => name
.replace(/^name=/, '')
.replace(RX_CLIENT_CLEAN, ''))
.filter((name, i, a) => a.indexOf(name) === i);
const keysToRemove = [];
let cursor = '0';
this.verbose(`Found connected keys: ${connectedKeys.map(k => `"${k}"`).join(', ')}`);
while (true) {
const data = await this.writer.scan(cursor, 'MATCH', `${this.options.prefix}:${this.options.cleanupFilter || '*'}`, 'COUNT', '1000');
cursor = data.shift();
const keys = data.shift() || [];
keysToRemove.push(...keys.filter(key => key !== this.lockKey &&
connectedKeys.every(connectedKey => key.indexOf(connectedKey) === -1)));
if (cursor === '0') {
break;
}
}
if (keysToRemove.length) {
await this.writer.del(...keysToRemove);
this.verbose(`Keys ${keysToRemove.map(k => `"${k}"`).join(', ')} were successfully removed!`);
}
}
catch (err) {
this.logger.warn('Clean-up error occurred:', err);
}
return this;
}
// noinspection JSUnusedLocalSymbols
/**
* Unreliable but fast way of message handling by the queue
*/
async readUnsafe() {
try {
const key = this.key;
while (true) {
if (!this.reader) {
break;
}
try {
const msg = await this.reader.brpop(key, 0);
if (msg) {
this.process(msg);
}
}
catch (err) {
// istanbul ignore next
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
if (err.message.match(/Stream connection ended/)) {
break;
}
// istanbul ignore next
// noinspection ExceptionCaughtLocallyJS
throw err;
}
}
}
catch (err) {
// istanbul ignore next
this.emitError('OnReadUnsafe', 'unsafe reader failed', err);
}
}
// noinspection JSUnusedLocalSymbols
/**
* Reliable but slow method of message handling by message queue
*/
async readSafe() {
try {
const key = this.key;
while (true) {
const expire = Date.now() +
Number(this.options.safeDeliveryTtl);
const workerKey = `${key}:worker:${(0, _1.uuid)()}:${expire}`;
if (!this.reader || !this.writer) {
break;
}
try {
await this.reader.brpoplpush(this.key, workerKey, 0);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
}
catch (_) {
// istanbul ignore next
break;
}
const msgArr = await this.writer.lrange(workerKey, -1, 1);
if (!msgArr || (msgArr === null || msgArr === void 0 ? void 0 : msgArr.length) !== 1) {
// noinspection ExceptionCaughtLocallyJS
throw new Error('Wrong messages count');
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const [msg] = msgArr;
this.process([key, msg]);
this.writer.del(workerKey).catch(e => this.logger.warn('OnReadSafe: del error', e));
}
}
catch (err) {
// istanbul ignore next
this.emitError('OnReadSafe', 'safe reader failed', err);
}
}
/**
* Initializes a read process on the redis message queue
*
* @returns {RedisQueue}
*/
read() {
// istanbul ignore next
if (!this.reader) {
this.logger.error(`${this.name}: reader connection is not initialized, pid ${process.pid} on redis host ${this.redisKey}!`);
return this;
}
const readMethod = this.options.safeDelivery
? 'readSafe'
: 'readUnsafe';
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
process.nextTick(this[readMethod].bind(this));
return this;
}
/**
* Checks if the watcher connection is locked
*
* @access private
* @returns {Promise<boolean>}
*/
async isLocked() {
if (this.writer) {
return Boolean(Number(await this.writer.exists(this.lockKey)));
}
return false;
}
/**
* Locks watcher connection
*
* @access private
* @returns {Promise<boolean>}
*/
async lock() {
if (this.writer) {
return Boolean(Number(await this.writer.setnx(this.lockKey, '')));
}
return false;
}
/**
* Unlocks watcher connection
*
* @access private
* @returns {Promise<boolean>}
*/
async unlock() {
if (this.writer) {
return Boolean(Number(await this.writer.del(this.lockKey)));
}
return false;
}
// istanbul ignore next
/**
* Emits error
*
* @access private
* @param {string} eventName
* @param {string} message
* @param {Error} err
*/
emitError(eventName, message, err) {
this.emit('error', err, eventName);
this.logger.error(`${this.name}: ${message}, pid ${process.pid} on redis host ${this.redisKey}:`, err);
this.verbose(`Error in event ${eventName}: ${message}, pid ${process.pid} on redis host ${this.redisKey}: ${err}`);
}
/**
* Acquires an owner for watcher connection to this instance of the queue
*
* @returns {Promise<void>}
*/
// istanbul ignore next
async ownWatch() {
const owned = await this.lock();
if (owned) {
this.verbose('Watcher connection lock acquired!');
for (const script of Object.keys(this.scripts)) {
try {
const checksum = sha1(this.scripts[script].code);
this.scripts[script].checksum = checksum;
const scriptExists = await this.writer.script('EXISTS', checksum);
const loaded = (scriptExists || []).shift();
if (!loaded) {
await this.writer.script('LOAD', this.scripts[script].code);
}
}
catch (err) {
this.emitError('OnScriptLoad', 'script load error', err);
}
}
this.watchOwner = true;
await this.connect('watcher', this.options);
this.watch();
}
}
// istanbul ignore next
/**
* This method returns a watcher lock resolver function
*
* @access private
* @param {(...args: any[]) => void} resolve
* @param {(...args: any[]) => void} reject
* @return {() => Promise<any>}
*/
watchLockResolver(resolve, reject) {
return (async () => {
try {
const noWatcher = !await this.watcherCount();
if (await this.isLocked() && noWatcher) {
await this.unlock();
await this.ownWatch();
}
resolve();
}
catch (err) {
reject(err);
}
});
}
/**
* Initializes a single watcher connection across all queues with the same
* prefix.
*
* @returns {Promise<void>}
*/
// istanbul ignore next
async initWatcher() {
return new Promise((async (resolve, reject) => {
try {
if (!await this.watcherCount()) {
this.verbose('Initializing watcher...');
await this.ownWatch();
if (this.watchOwner && this.watcher) {
resolve();
}
else {
// check for possible deadlock to resolve
setTimeout(this.watchLockResolver(resolve, reject), intrand(1, 50));
}
}
else {
resolve();
}
}
catch (err) {
this.logger.error(`${this.name}: error initializing watcher, pid ${process.pid} on redis host ${this.redisKey}`, err);
reject(err);
}
}));
}
}
exports.RedisQueue = RedisQueue;
/**
* Writer connections collection
*
* @type {{}}
*/
RedisQueue.writers = {};
/**
* Watcher connections collection
*
* @type {{}}
*/
RedisQueue.watchers = {};
__decorate([
(0, _1.profile)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], RedisQueue.prototype, "stop", null);
__decorate([
(0, _1.profile)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], RedisQueue.prototype, "destroy", null);
__decorate([
(0, _1.profile)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], RedisQueue.prototype, "clear", null);
__decorate([
(0, _1.profile)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], RedisQueue.prototype, "queueLength", null);
__decorate([
(0, _1.profile)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", void 0)
], RedisQueue.prototype, "destroyWatcher", null);
__decorate([
(0, _1.profile)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", void 0)
], RedisQueue.prototype, "destroyWriter", null);
__decorate([
(0, _1.profile)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [String, RedisQueue]),
__metadata("design:returntype", void 0)
], RedisQueue.prototype, "destroyChannel", null);
//# sourceMappingURL=RedisQueue.js.map