redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
436 lines • 25.3 kB
JavaScript
import { async, CallbackEmptyReplyError, createLogger, withWatchTransaction, } from 'redis-smq-common';
import { withSharedPoolConnection } from '../../common/redis/redis-connection-pool/with-shared-pool-connection.js';
import { redisKeys } from '../../common/redis/redis-keys/redis-keys.js';
import { Configuration } from '../../config-manager/configuration.js';
import { ExchangeHasBoundQueuesError, InvalidDirectExchangeParametersError, InvalidExchangeRoutingKeyError, NamespaceMismatchError, QueueAlreadyBound, QueueNotBoundError, } from '../../errors/index.js';
import { _parseQueueParams } from '../../queue-manager/_/_parse-queue-params.js';
import { EQueueType } from '../../queue-manager/index.js';
import { _parseExchangeParams } from '../_/_parse-exchange-params.js';
import { _saveExchange } from '../_/_save-exchange.js';
import { _validateQueueBinding } from '../_/_validate-queue-binding.js';
import { _validateExchange } from '../_/_validate-exchange.js';
import { EExchangeProperty, EExchangeQueuePolicy, EExchangeType, } from '../types/index.js';
import { _getRoutingKeyBoundQueues } from './_/_get-routing-key-bound-queues.js';
import { _getRoutingKeys } from './_/_get-routing-keys.js';
import { _validateOperation } from '../../queue-operation-validator/_/_validate-operation.js';
import { EQueueOperation } from '../../queue-operation-validator/index.js';
import { validateRedisKey } from '../../common/redis/redis-keys/validator.js';
export class ExchangeDirect {
type = EExchangeType.DIRECT;
logger;
constructor() {
this.logger = createLogger(Configuration.getConfig().logger, this.constructor.name);
this.logger.debug('ExchangeDirect initialized');
}
getRoutingKeys(exchange, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, EExchangeType.DIRECT);
if (exchangeParams instanceof Error)
return callback(exchangeParams);
withSharedPoolConnection((client, done) => _getRoutingKeys(client, exchangeParams, done), callback);
});
}
getRoutingKeyBoundQueues(exchange, routingKey, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, EExchangeType.DIRECT);
if (exchangeParams instanceof Error)
return callback(exchangeParams);
const validatedRoutingKey = validateRedisKey(routingKey);
if (validatedRoutingKey instanceof Error)
return callback(new InvalidDirectExchangeParametersError());
withSharedPoolConnection((client, done) => {
_getRoutingKeyBoundQueues(client, exchangeParams, validatedRoutingKey, done);
}, callback);
});
}
matchQueues(exchange, routingKey, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (exchangeParams instanceof Error) {
this.logger.error('matchQueues: invalid exchange params');
return callback(exchangeParams);
}
const validatedRoutingKey = validateRedisKey(routingKey);
if (validatedRoutingKey instanceof Error) {
this.logger.error(`matchQueues: invalid routing key "${routingKey}"`);
return callback(new InvalidExchangeRoutingKeyError({
metadata: {
exchange: exchangeParams,
routingKey: routingKey,
},
}));
}
this.logger.debug(`matchQueues: resolving queues ns=${exchangeParams.ns} ex=${exchangeParams.name} rk=${validatedRoutingKey}`);
withSharedPoolConnection((client, done) => {
const result = [];
async.series([
(cb1) => _validateExchange(client, exchangeParams, true, cb1),
(cb1) => _getRoutingKeyBoundQueues(client, exchangeParams, validatedRoutingKey, (err, reply) => {
if (err)
return cb1(err);
if (reply && reply.length)
result.push(...reply);
cb1();
}),
], (err) => {
if (err)
return done(err);
this.logger.debug(`matchQueues: matched ${result.length} queue(s) ns=${exchangeParams.ns} ex=${exchangeParams.name} rk=${validatedRoutingKey}`);
done(null, result);
});
}, callback);
});
}
create(exchange, queuePolicy, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (exchangeParams instanceof Error)
return callback(new InvalidDirectExchangeParametersError());
withSharedPoolConnection((client, cb) => _saveExchange(client, exchangeParams, queuePolicy, cb), callback);
});
}
bindQueue(queue, exchange, routingKey, cb) {
return async.withOptionalCallback(cb, (callback) => {
const queueParams = _parseQueueParams(queue);
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (queueParams instanceof Error) {
this.logger.error('bindQueue: invalid queue params');
return callback(queueParams);
}
if (exchangeParams instanceof Error) {
this.logger.error('bindQueue: invalid exchange params');
return callback(exchangeParams);
}
if (queueParams.ns !== exchangeParams.ns) {
this.logger.error(`bindQueue: namespace mismatch q.ns=${queueParams.ns} ex.ns=${exchangeParams.ns}`);
return callback(new NamespaceMismatchError());
}
const validatedRoutingKey = validateRedisKey(routingKey);
if (validatedRoutingKey instanceof Error) {
this.logger.error(`bindQueue: invalid routing key "${routingKey}"`);
return callback(new InvalidDirectExchangeParametersError());
}
const { keyQueueProperties, keyQueueExchangeBindings } = redisKeys.getQueueKeys(queueParams.ns, queueParams.name, null);
const { keyExchange } = redisKeys.getExchangeKeys(exchangeParams.ns, exchangeParams.name);
const { keyExchangeRoutingKeys } = redisKeys.getExchangeDirectKeys(exchangeParams.ns, exchangeParams.name);
const { keyRoutingKeyQueues } = redisKeys.getExchangeDirectRoutingKeyKeys(exchangeParams.ns, exchangeParams.name, validatedRoutingKey);
const { keyExchanges } = redisKeys.getMainKeys();
const { keyNamespaceExchanges } = redisKeys.getNamespaceKeys(queueParams.ns);
const queueStr = JSON.stringify(queueParams);
const exchangeStr = JSON.stringify(exchangeParams);
this.logger.debug(`bindQueue: bind q=${queueParams.name} ns=${queueParams.ns} -> ex=${exchangeParams.name} ns=${exchangeParams.ns} rk=${validatedRoutingKey}`);
withSharedPoolConnection((client, outerCb) => {
async.series([
(cb) => _validateOperation(client, queueParams, EQueueOperation.BIND_EXCHANGE, cb),
(cb) => withWatchTransaction(client, (client, watch, done) => {
let exchangeQueuePolicy = null;
async.waterfall([
(cb1) => watch([
keyExchange,
keyQueueProperties,
keyExchangeRoutingKeys,
keyRoutingKeyQueues,
keyQueueExchangeBindings,
keyExchanges,
keyNamespaceExchanges,
], cb1),
(_, cb1) => _validateQueueBinding(client, exchangeParams, queueParams, (err, reply) => {
if (err)
return cb1(err);
if (!reply)
return cb1(new CallbackEmptyReplyError());
const [queueProperties] = reply;
exchangeQueuePolicy =
queueProperties.queueType ===
EQueueType.PRIORITY_QUEUE
? EExchangeQueuePolicy.PRIORITY
: EExchangeQueuePolicy.STANDARD;
cb1();
}),
(_, cb1) => client.sismember(keyRoutingKeyQueues, queueStr, (err, reply) => {
if (err)
return cb1(err);
const already = reply === 1;
if (already) {
this.logger.debug('bindQueue: already bound');
return cb1(new QueueAlreadyBound());
}
cb1();
}),
(_, cb1) => {
const typeField = String(EExchangeProperty.TYPE);
const queuePolicyField = String(EExchangeProperty.QUEUE_POLICY);
const multi = client.multi();
multi.hset(keyExchange, typeField, EExchangeType.DIRECT);
multi.hset(keyExchange, queuePolicyField, Number(exchangeQueuePolicy));
multi.sadd(keyExchanges, exchangeStr);
multi.sadd(keyNamespaceExchanges, exchangeStr);
multi.sadd(keyExchangeRoutingKeys, validatedRoutingKey);
multi.sadd(keyRoutingKeyQueues, queueStr);
multi.sadd(keyQueueExchangeBindings, exchangeStr);
cb1(null, { multi });
},
], done);
}, (err) => {
if (err) {
if (err instanceof QueueAlreadyBound)
return cb();
return cb(err);
}
this.logger.info(`bindQueue: bound queue=${queueParams.name}@${queueParams.ns} -> ex=${exchangeParams.name}@${exchangeParams.ns} rk=${validatedRoutingKey}`);
cb();
}),
], (err) => outerCb(err));
}, callback);
});
}
unbindQueue(queue, exchange, routingKey, cb) {
return async.withOptionalCallback(cb, (callback) => {
const queueParams = _parseQueueParams(queue);
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (queueParams instanceof Error) {
this.logger.error('unbindQueue: invalid queue params');
return callback(queueParams);
}
if (exchangeParams instanceof Error) {
this.logger.error('unbindQueue: invalid exchange params');
return callback(exchangeParams);
}
if (queueParams.ns !== exchangeParams.ns) {
this.logger.error('unbindQueue: namespace mismatch');
return callback(new NamespaceMismatchError());
}
const validatedRoutingKey = validateRedisKey(routingKey);
if (validatedRoutingKey instanceof Error) {
this.logger.error(`unbindQueue: invalid routing key "${routingKey}"`);
return callback(new InvalidDirectExchangeParametersError());
}
const { keyQueueExchangeBindings } = redisKeys.getQueueKeys(queueParams.ns, queueParams.name, null);
const { keyExchange } = redisKeys.getExchangeKeys(exchangeParams.ns, exchangeParams.name);
const { keyExchangeRoutingKeys } = redisKeys.getExchangeDirectKeys(exchangeParams.ns, exchangeParams.name);
const { keyRoutingKeyQueues } = redisKeys.getExchangeDirectRoutingKeyKeys(exchangeParams.ns, exchangeParams.name, validatedRoutingKey);
const queueStr = JSON.stringify(queueParams);
const exchangeStr = JSON.stringify(exchangeParams);
this.logger.debug(`unbindQueue: unbinding q=${queueParams.name} ns=${queueParams.ns} from ex=${exchangeParams.name} ns=${exchangeParams.ns} rk=${validatedRoutingKey}`);
withSharedPoolConnection((client, outerCb) => {
async.series([
(cb) => _validateOperation(client, queueParams, EQueueOperation.UNBIND_EXCHANGE, cb),
(cb) => withWatchTransaction(client, (client, watch, done) => {
let routingKeysAll = [];
let otherRoutingKeySets = [];
let currentCount = 0;
let stillBoundViaOtherRK = false;
async.waterfall([
(cb1) => {
const baseWatchKeys = [
keyExchange,
keyExchangeRoutingKeys,
keyRoutingKeyQueues,
keyQueueExchangeBindings,
];
watch(baseWatchKeys, cb1);
},
(_, cb1) => _validateExchange(client, exchangeParams, true, cb1),
(_, cb1) => {
client.sismember(keyRoutingKeyQueues, queueStr, (err, reply) => {
if (err)
return cb1(err);
if (reply !== 1) {
this.logger.warn('unbindQueue: queue not bound');
return cb1(new QueueNotBoundError());
}
cb1();
});
},
(_, cb1) => {
client.smembers(keyExchangeRoutingKeys, (err, keys) => {
if (err)
return cb1(err);
routingKeysAll = keys ?? [];
cb1();
});
},
(_, cb1) => {
const others = routingKeysAll.filter((rk) => rk !== validatedRoutingKey);
otherRoutingKeySets = others.map((rk) => {
const { keyRoutingKeyQueues: k } = redisKeys.getExchangeDirectRoutingKeyKeys(exchangeParams.ns, exchangeParams.name, rk);
return k;
});
if (otherRoutingKeySets.length === 0)
return cb1();
watch(otherRoutingKeySets, cb1);
},
(_, cb1) => {
async.series([
(cbx) => client.scard(keyRoutingKeyQueues, (err, count) => {
if (err)
return cbx(err);
currentCount = count || 0;
cbx();
}),
(cbx) => {
if (otherRoutingKeySets.length === 0)
return cbx();
async.eachOf(otherRoutingKeySets, (k, _i, next) => {
if (stillBoundViaOtherRK)
return next();
client.sismember(k, queueStr, (e, rep) => {
if (e)
return next(e);
if (rep === 1)
stillBoundViaOtherRK = true;
next();
});
}, (e) => cbx(e || null));
},
], (err) => cb1(err));
},
(_, cb1) => {
const multi = client.multi();
multi.srem(keyRoutingKeyQueues, queueStr);
if (currentCount === 1) {
multi.srem(keyExchangeRoutingKeys, validatedRoutingKey);
}
if (!stillBoundViaOtherRK) {
multi.srem(keyQueueExchangeBindings, exchangeStr);
}
cb1(null, { multi });
},
], done);
}, (err) => {
if (err)
return cb(err);
this.logger.info(`unbindQueue: unbound q=${queueParams.name} ns=${queueParams.ns} from ex=${exchangeParams.name} ns=${exchangeParams.ns} rk=${validatedRoutingKey}`);
cb();
}),
], (err) => outerCb(err));
}, callback);
});
}
delete(exchange, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, this.type);
if (exchangeParams instanceof Error) {
this.logger.error('delete: invalid exchange params');
return callback(exchangeParams);
}
const { keyExchanges } = redisKeys.getMainKeys();
const { keyNamespaceExchanges } = redisKeys.getNamespaceKeys(exchangeParams.ns);
const { keyExchange } = redisKeys.getExchangeKeys(exchangeParams.ns, exchangeParams.name);
const { keyExchangeRoutingKeys } = redisKeys.getExchangeDirectKeys(exchangeParams.ns, exchangeParams.name);
const exchangeStr = JSON.stringify(exchangeParams);
this.logger.debug(`delete: direct exchange ex=${exchangeParams.name}@${exchangeParams.ns}`);
withSharedPoolConnection((client, outerCb) => {
withWatchTransaction(client, (client, watch, done) => {
let routingKeysAll = [];
async.waterfall([
(cb1) => {
const baseWatchKeys = [
keyExchange,
keyExchangeRoutingKeys,
keyExchanges,
keyNamespaceExchanges,
];
watch(baseWatchKeys, cb1);
},
(_, cb1) => _validateExchange(client, exchangeParams, true, cb1),
(_, cb1) => {
client.smembers(keyExchangeRoutingKeys, (err, keys) => {
if (err)
return cb1(err);
routingKeysAll = keys ?? [];
this.logger.debug(`delete: routingKeys=${routingKeysAll.length} ns=${exchangeParams.ns} ex=${exchangeParams.name}`);
cb1(null, routingKeysAll);
});
},
(routingKeys, cb1) => {
if (!routingKeys.length)
return cb1(null, routingKeys);
const derivedKeys = routingKeys.map((rk) => {
const { keyRoutingKeyQueues } = redisKeys.getExchangeDirectRoutingKeyKeys(exchangeParams.ns, exchangeParams.name, rk);
return keyRoutingKeyQueues;
});
watch(derivedKeys, (err) => cb1(err || null, routingKeys));
},
(routingKeys, cb1) => {
if (!routingKeys.length)
return cb1(null, routingKeys);
let hasBoundQueues = false;
async.eachOf(routingKeys, (rk, _idx, next) => {
const { keyRoutingKeyQueues } = redisKeys.getExchangeDirectRoutingKeyKeys(exchangeParams.ns, exchangeParams.name, rk);
client.scard(keyRoutingKeyQueues, (err, count) => {
if (!err && count && count > 0) {
hasBoundQueues = true;
this.logger.debug(`delete: rk has bound queues rk=${rk}`);
}
next(err || null);
});
}, (err) => {
if (err)
return cb1(err);
if (hasBoundQueues) {
this.logger.warn('delete: exchange has bound queues, aborting');
return cb1(new ExchangeHasBoundQueuesError());
}
cb1(null, routingKeys);
});
},
(routingKeys, cb1) => {
const multi = client.multi();
multi.del(keyExchange);
multi.del(keyExchangeRoutingKeys);
multi.srem(keyExchanges, exchangeStr);
multi.srem(keyNamespaceExchanges, exchangeStr);
for (const rk of routingKeys) {
const { keyRoutingKeyQueues } = redisKeys.getExchangeDirectRoutingKeyKeys(exchangeParams.ns, exchangeParams.name, rk);
multi.del(keyRoutingKeyQueues);
}
cb1(null, { multi });
},
], done);
}, (err) => {
if (err)
return outerCb(err);
this.logger.info(`delete: deleted direct exchange ex=${exchangeParams.name}@${exchangeParams.ns}`);
outerCb();
}, {
maxAttempts: 5,
onRetry: (attemptNo, maxAttempts) => {
this.logger.warn(`delete: concurrent modification detected, retrying attempt=${attemptNo}/${maxAttempts}`);
},
});
}, callback);
});
}
getBindings(exchange, cb) {
return async.withOptionalCallback(cb, (callback) => {
const exchangeParams = _parseExchangeParams(exchange, EExchangeType.DIRECT);
if (exchangeParams instanceof Error)
return callback(exchangeParams);
withSharedPoolConnection((client, done) => {
async.waterfall([
(cb) => {
_getRoutingKeys(client, exchangeParams, cb);
},
(routingKeys, done) => {
const bindings = {};
async.eachOf(routingKeys, (routingKey, _, done) => {
bindings[routingKey] = [];
_getRoutingKeyBoundQueues(client, exchangeParams, routingKey, (err, queues) => {
if (err)
return done(err);
bindings[routingKey].push(...(queues ?? []));
done();
});
}, (err) => {
if (err)
return done(err);
done(null, bindings);
});
},
], done);
}, callback);
});
}
}
//# sourceMappingURL=exchange-direct.js.map