relief-valve
Version:
This is a simple library for Redis Streams data type, which is used to accumulate messages until a specified threshold is reached, post which the same is available to consumer stream.
208 lines (207 loc) • 11.3 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ReliefValve = void 0;
const path_1 = __importDefault(require("path"));
class ReliefValve {
/** Used to contruct and instance of the class.
* @param redisPool Connector through which this instance will talk to redis.
* @param name A unique name for the Queue/Stream for the consumers to subscribe on.
* @param countThreshold A positive number which acts as a setpoint for the relief valve(pressure release point), negative numbers will be converted to positive and zero to 1.
* @param timeThresholdInSeconds A positive number in seconds which acts as elapsed time in future post which the valve will be opened even if count threshold is not reached from the last time of write, negative numbers will be converted to positive and zero to 1.
*
*/
constructor(redisPool, name, countThreshold, timeThresholdInSeconds, groupName, clientName, indexKey = name + "Idx", accumalatorKey = (data) => Promise.resolve(name + "Acc"), cappedStreamLength = -1, systemIdPropName = "_id_", releaseCount = -1) {
this.redisPool = redisPool;
this.name = name;
this.countThreshold = countThreshold;
this.timeThresholdInSeconds = timeThresholdInSeconds;
this.groupName = groupName;
this.clientName = clientName;
this.indexKey = indexKey;
this.accumalatorKey = accumalatorKey;
this.cappedStreamLength = cappedStreamLength;
this.systemIdPropName = systemIdPropName;
this.releaseCount = releaseCount;
this.writeScript = path_1.default.join(__dirname, "write_count_purge.lua");
this.timePurgeScript = path_1.default.join(__dirname, "time_purge.lua");
this.groupsCreated = new Set();
if (this.timeThresholdInSeconds < 0) {
this.timeThresholdInSeconds *= -1;
}
if (this.countThreshold < 0) {
this.countThreshold *= -1;
}
if (this.timeThresholdInSeconds === 0) {
this.timeThresholdInSeconds = 1;
}
if (this.countThreshold === 0) {
this.countThreshold = 1;
}
if (releaseCount < -1 || releaseCount === 0) {
releaseCount = -1;
}
}
publish(data, id = "*") {
return __awaiter(this, void 0, void 0, function* () {
const token = this.redisPool.generateUniqueToken("publish");
const accKey = yield this.accumalatorKey(data);
const keys = [this.indexKey, accKey, this.name];
if (keys.length != Array.from((new Set(keys)).values()).length) {
throw new Error("Name, IndexKey, AccumalatorKey and AccumalatorPurgedKey cannot be same.");
}
const values = Array.from(Object.entries(data)).flat();
const allStrings = values.reduce((acc, e) => acc && typeof (e) === "string", true);
if (allStrings === false) {
throw new Error("Publish only support objects having strings as their values.");
}
values.unshift(this.releaseCount);
values.unshift(this.cappedStreamLength);
values.unshift(this.systemIdPropName);
values.unshift(id);
values.unshift(this.countThreshold);
yield this.redisPool.acquire(token);
try {
const response = yield this.redisPool.script(token, this.writeScript, keys, values);
return response[0];
}
finally {
yield this.redisPool.release(token);
}
});
}
recheckTimeThreshold(refreshTimeOnSucessfullPurge = -1) {
return __awaiter(this, void 0, void 0, function* () {
const token = this.redisPool.generateUniqueToken("recheckTimeThreshold");
yield this.redisPool.acquire(token);
try {
const redisTimeResponse = yield this.redisPool.run(token, ["TIME"]);
const redisTime = parseInt(redisTimeResponse[0]);
const accumulatorKeysWithScore = yield this.redisPool.run(token, ["ZRANGE", this.indexKey, "-inf", (redisTime - this.timeThresholdInSeconds).toString(), "BYSCORE", "WITHSCORES"]);
const asyncHandles = [];
for (let counter = 0; counter < accumulatorKeysWithScore.length; counter += 2) {
const accKey = accumulatorKeysWithScore[counter];
const accKeyScore = accumulatorKeysWithScore[counter + 1];
asyncHandles.push((() => __awaiter(this, void 0, void 0, function* () {
const keys = [this.indexKey, accKey, this.name];
if (keys.length != Array.from((new Set(keys)).values()).length) {
throw new Error("Name, IndexKey, AccumalatorKey and AccumalatorPurgedKey cannot be same.");
}
yield this.redisPool.script(token, this.timePurgeScript, keys, [accKeyScore, this.systemIdPropName, this.releaseCount.toString(), refreshTimeOnSucessfullPurge.toString()]); // Script is used to ensure serializable transactions and score is passed to make it thread safe with other instances.
}))());
}
yield Promise.allSettled(asyncHandles);
}
finally {
yield this.redisPool.release(token);
}
});
}
consumeFreshOrStale(batchIdealThresholdInSeconds) {
return __awaiter(this, void 0, void 0, function* () {
const token = this.redisPool.generateUniqueToken("consumeFreshOrPending");
let returnValue = undefined;
yield this.redisPool.acquire(token);
try {
yield this.createStreamGroupIfNotExists(this.name, this.groupName, this.redisPool, token);
const staleResponse = yield this.redisPool.run(token, ["XAUTOCLAIM", this.name, this.groupName, this.clientName, (batchIdealThresholdInSeconds * 1000).toString(), "0-0", "COUNT", "1"]);
let itemToProcess = undefined;
if (Array.isArray(staleResponse) && staleResponse.length >= 2 && Array.isArray(staleResponse[1]) && staleResponse[1].length > 0) {
//We have a stale response
itemToProcess = staleResponse[1][0];
}
else {
//We need to pluck fresh ones.
const freshResponse = yield this.redisPool.run(token, ["XREADGROUP", "GROUP", this.groupName, this.clientName, "COUNT", "1", "STREAMS", this.name, ">"]);
if (Array.isArray(freshResponse) && freshResponse.length >= 1 && Array.isArray(freshResponse[0]) && freshResponse[0].length >= 2 && freshResponse[0][1].length > 0) {
itemToProcess = freshResponse[0][1][0];
}
}
if (itemToProcess != undefined) {
returnValue = {
"id": itemToProcess[0],
"readsInCurrentGroup": -1,
"payload": new Map()
};
const retrivalCountResponse = yield this.redisPool.run(token, ["XPENDING", this.name, this.groupName, returnValue.id, returnValue.id, "1"]);
if (Array.isArray(retrivalCountResponse) && retrivalCountResponse.length >= 1) {
returnValue.readsInCurrentGroup = parseInt(retrivalCountResponse[0][3]);
}
let currentMessageId = "";
const serializedPayload = itemToProcess[1];
for (let propCounter = 0; propCounter < serializedPayload.length; propCounter += 2) {
const propName = serializedPayload[propCounter];
const propValue = serializedPayload[propCounter + 1];
if (propName === this.systemIdPropName) {
currentMessageId = propValue;
returnValue.payload.set(currentMessageId, {});
}
else {
const payloadObject = returnValue.payload.get(currentMessageId) || {};
payloadObject[propName] = propValue;
returnValue.payload.set(currentMessageId, payloadObject);
}
}
}
return returnValue;
}
finally {
yield this.redisPool.release(token);
}
});
}
acknowledge(batch, dropBatch = true) {
return __awaiter(this, void 0, void 0, function* () {
const token = this.redisPool.generateUniqueToken("acknowledge");
yield this.redisPool.acquire(token);
try {
yield this.createStreamGroupIfNotExists(this.name, this.groupName, this.redisPool, token);
const response = yield this.redisPool.run(token, ["XACK", this.name, this.groupName, batch.id]);
if (response === 1) {
if (dropBatch === true) {
yield this.redisPool.run(token, ["XDEL", this.name, batch.id]);
}
return true;
}
else {
return false;
}
}
finally {
yield this.redisPool.release(token);
}
});
}
//-------------------------------------------------------------------------Private Methods------------------------------------------------------------------------------------
createStreamGroupIfNotExists(streamName, groupName, acquiredClient, token, cache = this.groupsCreated) {
return __awaiter(this, void 0, void 0, function* () {
if (cache.has(groupName)) {
return;
}
try {
yield acquiredClient.run(token, ["XGROUP", "CREATE", streamName, groupName, "0", "MKSTREAM"]);
cache.add(groupName);
}
catch (err) {
if (err.message === 'BUSYGROUP Consumer Group name already exists') {
cache.add(groupName);
}
else {
throw err;
}
}
});
}
}
exports.ReliefValve = ReliefValve;