redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
399 lines • 22.8 kB
JavaScript
import { async, CallbackEmptyReplyError, createLogger, withWatchTransaction, } from 'redis-smq-common';
import { withSharedPoolConnection } from '../../common/redis/redis-connection-pool/with-shared-pool-connection.js';
import { EQueueType } from '../../queue-manager/index.js';
import { _saveExchange } from '../_/_save-exchange.js';
import { _validateQueueBinding } from '../_/_validate-queue-binding.js';
import { EExchangeProperty, EExchangeQueuePolicy, EExchangeType, } from '../index.js';
import { Configuration } from '../../config-manager/configuration.js';
import { _getRoutingPatterns } from './_/_get-routing-patterns.js';
import { _getRoutingPatternBoundQueues } from './_/_get-routing-pattern-bound-queues.js';
import { _parseExchangeParams } from '../_/_parse-exchange-params.js';
import { redisKeys } from '../../common/redis/redis-keys/redis-keys.js';
import { _parseQueueParams } from '../../queue-manager/_/_parse-queue-params.js';
import { ExchangeHasBoundQueuesError, InvalidExchangeRoutingKeyError, InvalidTopicBindingPatternError, InvalidTopicExchangeParamsError, NamespaceMismatchError, QueueAlreadyBound, QueueNotBoundError, } from '../../errors/index.js';
import { _validateRoutingPattern } from './_/_validate-routing-pattern.js';
import { _matchRoutingKey } from './_/_match-routing-key.js';
import { _validateExchange } from '../_/_validate-exchange.js';
import { _validateOperation } from '../../queue-operation-validator/_/_validate-operation.js';
import { EQueueOperation } from '../../queue-operation-validator/index.js';
export class ExchangeTopic {
type = EExchangeType.TOPIC;
logger;
constructor() {
this.logger = createLogger(Configuration.getConfig().logger, this.constructor.name);
}
matchQueues(exchange, routingKey, cb) {
return async.withOptionalCallback(cb, (callback) => {
const topicParams = _parseExchangeParams(exchange, this.type);
if (topicParams instanceof Error) {
return callback(topicParams);
}
const validatedRoutingKey = routingKey.trim();
if (!validatedRoutingKey) {
return callback(new InvalidExchangeRoutingKeyError({
metadata: {
exchange: topicParams,
routingKey: routingKey,
},
}));
}
withSharedPoolConnection((client, topCb) => {
_getRoutingPatterns(client, topicParams, (err, patterns) => {
if (err)
return topCb(err);
const allPatterns = patterns ?? [];
if (allPatterns.length === 0)
return topCb(null, []);
const matched = allPatterns.filter((p) => _matchRoutingKey(validatedRoutingKey, p));
if (matched.length === 0)
return topCb(null, []);
const union = new Map();
const tasks = matched.map((p) => (tcb) => {
_getRoutingPatternBoundQueues(client, p, topicParams, (e, qs) => {
if (e)
return tcb(e);
(qs ?? []).forEach((q) => union.set(`${q.name}@${q.ns}`, q));
tcb();
});
});
async.parallel(tasks, (e) => {
if (e)
return topCb(e);
topCb(null, Array.from(union.values()));
});
});
}, callback);
});
}
getRoutingPatterns(exchange, cb) {
return async.withOptionalCallback(cb, (callback) => {
withSharedPoolConnection((client, cb) => _getRoutingPatterns(client, exchange, cb), callback);
});
}
getRoutingPatternBoundQueues(exchange, bindingPattern, cb) {
return async.withOptionalCallback(cb, (callback) => {
withSharedPoolConnection((client, cb) => _getRoutingPatternBoundQueues(client, bindingPattern, exchange, cb), callback);
});
}
bindQueue(queue, exchange, routingPattern, cb) {
return async.withOptionalCallback(cb, (callback) => {
const queueParams = _parseQueueParams(queue);
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (queueParams instanceof Error)
return callback(queueParams);
if (exchangeParams instanceof Error)
return callback(exchangeParams);
if (queueParams.ns !== exchangeParams.ns) {
return callback(new NamespaceMismatchError());
}
if (!_validateRoutingPattern(routingPattern)) {
return callback(new InvalidTopicBindingPatternError({
metadata: { pattern: routingPattern },
}));
}
const { keyQueueProperties, keyQueueExchangeBindings } = redisKeys.getQueueKeys(queueParams.ns, queueParams.name, null);
const { keyExchange } = redisKeys.getExchangeKeys(exchangeParams.ns, exchangeParams.name);
const { keyExchangeBindingPatterns } = redisKeys.getExchangeTopicKeys(exchangeParams.ns, exchangeParams.name);
const { keyBindingPatternQueues } = redisKeys.getExchangeTopicBindingPatternKeys(exchangeParams.ns, exchangeParams.name, routingPattern);
const { keyExchanges } = redisKeys.getMainKeys();
const { keyNamespaceExchanges } = redisKeys.getNamespaceKeys(queueParams.ns);
const queueStr = JSON.stringify(queueParams);
const exchangeStr = JSON.stringify(exchangeParams);
withSharedPoolConnection((client, outerCb) => {
async.series([
(cb) => _validateOperation(client, queueParams, EQueueOperation.BIND_EXCHANGE, cb),
(cb) => withWatchTransaction(client, (c, watch, done) => {
let exchangeQueuePolicy = null;
async.waterfall([
(cb1) => watch([
keyExchange,
keyQueueProperties,
keyExchangeBindingPatterns,
keyBindingPatternQueues,
keyQueueExchangeBindings,
keyExchanges,
keyNamespaceExchanges,
], cb1),
(_, cb1) => _validateQueueBinding(c, exchangeParams, queueParams, (err, reply) => {
if (err)
return cb1(err);
if (!reply)
return cb1(new CallbackEmptyReplyError());
const [queueProperties] = reply;
exchangeQueuePolicy =
queueProperties.queueType ===
EQueueType.PRIORITY_QUEUE
? EExchangeQueuePolicy.PRIORITY
: EExchangeQueuePolicy.STANDARD;
cb1();
}),
(_, cb1) => c.sismember(keyBindingPatternQueues, queueStr, (err, reply) => {
if (err)
return cb1(err);
if (reply === 1) {
this.logger.debug('bindQueue: already bound');
return cb1(new QueueAlreadyBound());
}
cb1();
}),
(_, cb1) => {
const typeField = String(EExchangeProperty.TYPE);
const queuePolicyField = String(EExchangeProperty.QUEUE_POLICY);
const multi = c.multi();
multi.hset(keyExchange, typeField, EExchangeType.TOPIC);
multi.hset(keyExchange, queuePolicyField, Number(exchangeQueuePolicy));
multi.sadd(keyExchanges, exchangeStr);
multi.sadd(keyNamespaceExchanges, exchangeStr);
multi.sadd(keyExchangeBindingPatterns, routingPattern);
multi.sadd(keyBindingPatternQueues, queueStr);
multi.sadd(keyQueueExchangeBindings, exchangeStr);
cb1(null, { multi });
},
], done);
}, (err) => {
if (err) {
if (err instanceof QueueAlreadyBound)
return cb();
return cb(err);
}
this.logger.info(`bindQueue: bound queue=${queueParams.name}@${queueParams.ns} -> ex=${exchangeParams.name}@${exchangeParams.ns} pat=${routingPattern}`);
cb();
}, {
maxAttempts: 5,
onRetry: (attemptNo, maxAttempts) => this.logger.warn(`bindQueue: concurrent modification, retrying attempt=${attemptNo}/${maxAttempts}`),
}),
], (err) => outerCb(err));
}, callback);
});
}
unbindQueue(queue, exchange, routingPattern, cb) {
return async.withOptionalCallback(cb, (callback) => {
const queueParams = _parseQueueParams(queue);
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (queueParams instanceof Error)
return callback(queueParams);
if (exchangeParams instanceof Error)
return callback(exchangeParams);
if (queueParams.ns !== exchangeParams.ns) {
return callback(new NamespaceMismatchError());
}
if (!_validateRoutingPattern(routingPattern)) {
return callback(new InvalidTopicBindingPatternError({
metadata: { pattern: routingPattern },
}));
}
const { keyQueueExchangeBindings } = redisKeys.getQueueKeys(queueParams.ns, queueParams.name, null);
const { keyExchange } = redisKeys.getExchangeKeys(exchangeParams.ns, exchangeParams.name);
const { keyExchangeBindingPatterns } = redisKeys.getExchangeTopicKeys(exchangeParams.ns, exchangeParams.name);
const { keyBindingPatternQueues } = redisKeys.getExchangeTopicBindingPatternKeys(exchangeParams.ns, exchangeParams.name, routingPattern);
const queueStr = JSON.stringify(queueParams);
const exchangeStr = JSON.stringify(exchangeParams);
withSharedPoolConnection((client, outerCb) => {
async.series([
(cb) => _validateOperation(client, queueParams, EQueueOperation.UNBIND_EXCHANGE, cb),
(cb) => withWatchTransaction(client, (c, watch, done) => {
let allPatterns = [];
let currentPatternCount = 0;
let stillBoundViaOtherPattern = false;
async.waterfall([
(cb1) => watch([
keyExchange,
keyExchangeBindingPatterns,
keyBindingPatternQueues,
keyQueueExchangeBindings,
], cb1),
(_, cb1) => _validateExchange(c, exchangeParams, true, cb1),
(_, cb1) => c.sismember(keyBindingPatternQueues, queueStr, (err, reply) => {
if (err)
return cb1(err);
if (reply !== 1)
return cb1(new QueueNotBoundError());
cb1();
}),
(_, cb1) => c.smembers(keyExchangeBindingPatterns, (err, pats) => {
if (err)
return cb1(err);
allPatterns = (pats ?? []).filter((p) => p && p.length);
cb1();
}),
(_, cb1) => {
const otherPatterns = allPatterns.filter((p) => p !== routingPattern);
const otherPatternSets = otherPatterns.map((p) => {
const { keyBindingPatternQueues: k } = redisKeys.getExchangeTopicBindingPatternKeys(exchangeParams.ns, exchangeParams.name, p);
return k;
});
const doWatch = (next) => otherPatternSets.length
? watch(otherPatternSets, next)
: next();
doWatch((err) => {
if (err)
return cb1(err);
async.series([
(cbx) => c.scard(keyBindingPatternQueues, (e, count) => {
if (e)
return cbx(e);
currentPatternCount = count || 0;
cbx();
}),
(cbx) => {
if (otherPatternSets.length === 0)
return cbx();
async.eachOf(otherPatternSets, (setKey, _i, next) => {
if (stillBoundViaOtherPattern)
return next();
c.sismember(setKey, queueStr, (e2, rep) => {
if (e2)
return next(e2);
if (rep === 1)
stillBoundViaOtherPattern = true;
next();
});
}, (e3) => cbx(e3 || null));
},
], (err) => cb1(err));
});
},
(_, cb1) => {
const multi = c.multi();
multi.srem(keyBindingPatternQueues, queueStr);
if (currentPatternCount === 1) {
multi.srem(keyExchangeBindingPatterns, routingPattern);
}
if (!stillBoundViaOtherPattern) {
multi.srem(keyQueueExchangeBindings, exchangeStr);
}
cb1(null, { multi });
},
], done);
}, (err) => {
if (err)
return cb(err);
this.logger.info(`unbindQueue: unbound queue=${queueParams.name}@${queueParams.ns} from ex=${exchangeParams.name}@${exchangeParams.ns} pat=${routingPattern}`);
cb();
}, {
maxAttempts: 5,
onRetry: (attemptNo, maxAttempts) => this.logger.warn(`unbindQueue: concurrent modification, retrying attempt=${attemptNo}/${maxAttempts}`),
}),
], (err) => outerCb(err));
}, callback);
});
}
create(exchange, queuePolicy, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (exchangeParams instanceof Error)
return callback(new InvalidTopicExchangeParamsError());
withSharedPoolConnection((client, cb) => _saveExchange(client, exchangeParams, queuePolicy, cb), callback);
});
}
delete(exchange, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (exchangeParams instanceof Error)
return callback(exchangeParams);
const { keyExchanges } = redisKeys.getMainKeys();
const { keyNamespaceExchanges } = redisKeys.getNamespaceKeys(exchangeParams.ns);
const { keyExchange } = redisKeys.getExchangeKeys(exchangeParams.ns, exchangeParams.name);
const { keyExchangeBindingPatterns } = redisKeys.getExchangeTopicKeys(exchangeParams.ns, exchangeParams.name);
const exchangeStr = JSON.stringify(exchangeParams);
withSharedPoolConnection((client, outerCb) => {
withWatchTransaction(client, (c, watch, done) => {
let patterns = [];
async.waterfall([
(cb1) => watch([
keyExchange,
keyExchangeBindingPatterns,
keyExchanges,
keyNamespaceExchanges,
], cb1),
(_, cb1) => _validateExchange(c, exchangeParams, true, cb1),
(_, cb1) => c.smembers(keyExchangeBindingPatterns, (err, pats) => {
if (err)
return cb1(err);
patterns = (pats ?? []).filter((p) => p && p.length);
cb1();
}),
(_, cb1) => {
if (patterns.length === 0)
return cb1();
const derivedKeys = patterns.map((p) => {
const { keyBindingPatternQueues } = redisKeys.getExchangeTopicBindingPatternKeys(exchangeParams.ns, exchangeParams.name, p);
return keyBindingPatternQueues;
});
watch(derivedKeys, cb1);
},
(_, cb1) => {
if (patterns.length === 0)
return cb1();
let hasBoundQueues = false;
async.eachOf(patterns, (p, _idx, next) => {
const { keyBindingPatternQueues } = redisKeys.getExchangeTopicBindingPatternKeys(exchangeParams.ns, exchangeParams.name, p);
c.scard(keyBindingPatternQueues, (err, count) => {
if (!err && (count || 0) > 0) {
hasBoundQueues = true;
this.logger.debug(`delete: pattern "${p}" has ${count} bound queue(s)`);
}
next(err || null);
});
}, (err) => {
if (err)
return cb1(err);
if (hasBoundQueues)
return cb1(new ExchangeHasBoundQueuesError());
cb1();
});
},
(_, cb1) => {
const multi = c.multi();
multi.del(keyExchange);
multi.del(keyExchangeBindingPatterns);
multi.srem(keyExchanges, exchangeStr);
multi.srem(keyNamespaceExchanges, exchangeStr);
for (const p of patterns) {
const { keyBindingPatternQueues } = redisKeys.getExchangeTopicBindingPatternKeys(exchangeParams.ns, exchangeParams.name, p);
multi.del(keyBindingPatternQueues);
}
cb1(null, { multi });
},
], done);
}, (err) => {
if (err)
return outerCb(err);
this.logger.info(`delete: exchange ${exchangeParams.name}@${exchangeParams.ns} deleted`);
outerCb();
});
}, callback);
});
}
getBindings(exchange, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (exchangeParams instanceof Error)
return callback(exchangeParams);
withSharedPoolConnection((client, done) => {
async.waterfall([
(cb) => {
_getRoutingPatterns(client, exchangeParams, cb);
},
(bindingPatterns, done) => {
const bindings = {};
async.eachOf(bindingPatterns, (bindingPattern, _, done) => {
bindings[bindingPattern] = [];
_getRoutingPatternBoundQueues(client, bindingPattern, exchangeParams, (err, queues) => {
if (err)
return done(err);
bindings[bindingPattern].push(...(queues ?? []));
done();
});
}, (err) => {
if (err)
return done(err);
done(null, bindings);
});
},
], done);
}, callback);
});
}
}
//# sourceMappingURL=exchange-topic.js.map