redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
484 lines • 24.4 kB
JavaScript
"use strict";
var __rest = (this && this.__rest) || function (s, e) {
var t = {};
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
t[p] = s[p];
if (s != null && typeof Object.getOwnPropertySymbols === "function")
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
t[p[i]] = s[p[i]];
}
return t;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BackgroundJobManagerAbstract = void 0;
const redis_smq_common_1 = require("redis-smq-common");
const node_crypto_1 = require("node:crypto");
const index_js_1 = require("../../../errors/index.js");
const index_js_2 = require("./types/index.js");
const path_1 = require("path");
const redis_keys_js_1 = require("../../redis/redis-keys/redis-keys.js");
const _is_background_job_worker_alive_js_1 = require("./helpers/_is-background-job-worker-alive.js");
const batchSize = 1000;
const delay = 5000;
var ELuaScript;
(function (ELuaScript) {
ELuaScript["CREATE_JOB"] = "CREATE_JOB";
ELuaScript["CANCEL_JOB"] = "CANCEL_JOB";
ELuaScript["COMPLETE_JOB"] = "COMPLETE_JOB";
ELuaScript["FAIL_JOB"] = "FAIL_JOB";
ELuaScript["START_JOB"] = "START_JOB";
ELuaScript["RECOVER_STUCK_JOB"] = "RECOVER_STUCK_JOB";
})(ELuaScript || (ELuaScript = {}));
const curDir = redis_smq_common_1.env.getCurrentDir();
const luaScriptMap = {
[ELuaScript.CREATE_JOB]: (0, path_1.resolve)(curDir, './redis/scripts/create-job.lua'),
[ELuaScript.CANCEL_JOB]: (0, path_1.resolve)(curDir, './redis/scripts/cancel-job.lua'),
[ELuaScript.COMPLETE_JOB]: (0, path_1.resolve)(curDir, './redis/scripts/complete-job.lua'),
[ELuaScript.FAIL_JOB]: (0, path_1.resolve)(curDir, './redis/scripts/fail-job.lua'),
[ELuaScript.START_JOB]: (0, path_1.resolve)(curDir, './redis/scripts/start-job.lua'),
[ELuaScript.RECOVER_STUCK_JOB]: (0, path_1.resolve)(curDir, './redis/scripts/recover-stuck-job.lua'),
};
class BackgroundJobManagerAbstract {
constructor(redisClient, config, logger) {
this.redisClient = redisClient;
this.logger = logger.createLogger(this.constructor.name);
this.config = config;
}
applyPartialUpdate(job, partial) {
const { meta: partialMeta } = partial, partialFields = __rest(partial, ["meta"]);
const result = Object.assign(Object.assign({}, job), partialFields);
if (partialMeta) {
result.meta = Object.assign(Object.assign({}, job.meta), partialMeta);
}
return result;
}
update(jobId, updates, cb) {
this.get(jobId, (err, backgroundJob) => {
if (err)
return cb(err);
if (!backgroundJob)
return cb(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
const updatedJob = this.applyPartialUpdate(backgroundJob, Object.assign(Object.assign({}, updates), { updatedAt: Date.now() }));
this.redisClient.hset(this.config.keyBackgroundJobs, jobId, JSON.stringify(updatedJob), (setErr) => cb(setErr));
});
}
removeFromProcessing(jobId, cb) {
this.redisClient.lrem(this.config.keyBackgroundJobsProcessing, 0, jobId, cb);
}
initialize(cb) {
this.logger.debug('Loading Redis Lua scripts');
this.redisClient.loadScriptFiles(luaScriptMap, (err) => {
if (err) {
this.logger.error(`Failed to load Redis Lua scripts: ${err.message}`, err);
}
else {
this.logger.debug('Redis Lua scripts loaded successfully');
}
cb(err);
});
}
create(payload, options = {}, cb) {
var _a;
const jobId = (_a = options.id) !== null && _a !== void 0 ? _a : (0, node_crypto_1.randomUUID)();
const now = Date.now();
const backgroundJob = Object.assign(Object.assign({ batchSize,
delay }, options), { id: jobId, payload, status: index_js_2.EBackgroundJobStatus.PENDING, createdAt: now, updatedAt: now });
this.redisClient.runScript(ELuaScript.CREATE_JOB, [this.config.keyBackgroundJobs, this.config.keyBackgroundJobsPending], [jobId, JSON.stringify(backgroundJob), jobId], (err, reply) => {
if (err)
return cb(err);
switch (reply) {
case -1:
return cb(new index_js_1.BackgroundJobAlreadyExistsError({ metadata: { jobId } }));
case 1:
this.logger.debug(`Created job ${jobId} for target "${JSON.stringify(payload)}"`);
return cb(null, backgroundJob);
default:
return cb(new index_js_1.UnexpectedScriptReplyError({ metadata: { reply } }));
}
});
}
get(jobId, cb) {
this.redisClient.hget(this.config.keyBackgroundJobs, jobId, (err, data) => {
if (err)
return cb(err);
if (!data)
return cb(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
cb(null, JSON.parse(data));
});
}
acquireNextJob(cb) {
this.redisClient.brpoplpush(this.config.keyBackgroundJobsPending, this.config.keyBackgroundJobsProcessing, 0, cb);
}
cancel(jobId, cb) {
redis_smq_common_1.async.waterfall([
(next) => {
this.get(jobId, (err, backgroundJob) => {
if (err)
return next(err);
if (!backgroundJob)
return next(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
next(null, backgroundJob);
});
},
(backgroundJob, next) => {
const { keyJobWorker } = redis_keys_js_1.redisKeys.getJobKeys(jobId);
const updatedJob = Object.assign(Object.assign({}, backgroundJob), { status: index_js_2.EBackgroundJobStatus.CANCELED, updatedAt: Date.now() });
this.redisClient.runScript(ELuaScript.CANCEL_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsPending,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
JSON.stringify(updatedJob),
index_js_2.EBackgroundJobStatus.PENDING.toString(),
index_js_2.EBackgroundJobStatus.PROCESSING.toString(),
index_js_2.EBackgroundJobStatus.COMPLETED.toString(),
index_js_2.EBackgroundJobStatus.FAILED.toString(),
index_js_2.EBackgroundJobStatus.CANCELED.toString(),
], (err, result) => {
if (err)
return next(err);
switch (result) {
case 1:
this.logger.info(`Cancelled job ${jobId}`);
return next(null, updatedJob);
case 2:
this.logger.warn(`Job ${jobId} was already cancelled`);
return next(null);
case 0:
return next(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
case -1:
return next(new index_js_1.BackgroundJobNotCancellableError({
metadata: { jobId, reason: 'already completed' },
}));
case -2:
return next(new index_js_1.BackgroundJobNotCancellableError({
metadata: { jobId, reason: 'already failed' },
}));
default:
return next(new Error(`Unexpected result from cancel script: ${result}`));
}
});
},
], cb);
}
start(jobId, workerId, cb) {
this.get(jobId, (err, backgroundJob) => {
if (err)
return cb(err);
if (!backgroundJob)
return cb(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
const { keyJobWorker } = redis_keys_js_1.redisKeys.getJobKeys(jobId);
const updatedJob = Object.assign(Object.assign({}, backgroundJob), { status: index_js_2.EBackgroundJobStatus.PROCESSING, startedAt: Date.now(), updatedAt: Date.now() });
this.redisClient.runScript(ELuaScript.START_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
workerId,
JSON.stringify(updatedJob),
index_js_2.EBackgroundJobStatus.PENDING.toString(),
index_js_2.EBackgroundJobStatus.PROCESSING.toString(),
index_js_2.EBackgroundJobStatus.COMPLETED.toString(),
index_js_2.EBackgroundJobStatus.FAILED.toString(),
index_js_2.EBackgroundJobStatus.CANCELED.toString(),
], (err, reply) => {
if (err)
return cb(err);
switch (reply) {
case 1:
this.logger.info(`Started processing job ${jobId}`);
return cb(null, updatedJob);
case 2:
this.logger.warn(`Job ${jobId} was already processing`);
return cb(null);
case 0:
return cb(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
case -1:
return cb(new index_js_1.BackgroundJobNotStartableError({
metadata: {
jobId,
reason: "Job cannot be started because it's already completed",
},
}));
case -2:
return cb(new index_js_1.BackgroundJobNotStartableError({
metadata: {
jobId,
reason: "Job cannot be started because it's already failed",
},
}));
case -3:
return cb(new index_js_1.BackgroundJobNotStartableError({
metadata: {
jobId,
reason: "Job cannot be started because it's already cancelled",
},
}));
default:
return cb(new index_js_1.UnexpectedScriptReplyError({ metadata: { reply } }));
}
});
});
}
complete(jobId, options = {}, cb) {
redis_smq_common_1.async.waterfall([
(next) => {
this.get(jobId, (err, backgroundJob) => {
if (err)
return next(err);
if (!backgroundJob)
return next(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
next(null, backgroundJob);
});
},
(backgroundJob, next) => {
const { keyJobWorker } = redis_keys_js_1.redisKeys.getJobKeys(jobId);
const updatedJob = this.applyPartialUpdate(backgroundJob, Object.assign(Object.assign({}, options), { status: index_js_2.EBackgroundJobStatus.COMPLETED, completedAt: Date.now(), updatedAt: Date.now() }));
this.redisClient.runScript(ELuaScript.COMPLETE_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
JSON.stringify(updatedJob),
index_js_2.EBackgroundJobStatus.PROCESSING.toString(),
index_js_2.EBackgroundJobStatus.COMPLETED.toString(),
index_js_2.EBackgroundJobStatus.FAILED.toString(),
index_js_2.EBackgroundJobStatus.CANCELED.toString(),
], (err, reply) => {
if (err)
return next(err);
switch (reply) {
case 1:
this.logger.info(`Completed job ${jobId}`);
return next(null, updatedJob);
case 2:
this.logger.warn(`Job ${jobId} was already completed`);
return next(null);
case 0:
return next(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
case -1:
return next(new index_js_1.BackgroundJobNotCompletableError({
metadata: { jobId, reason: 'already failed' },
}));
case -2:
return next(new index_js_1.BackgroundJobNotCompletableError({
metadata: { jobId, reason: 'already cancelled' },
}));
default:
return next(new index_js_1.UnexpectedScriptReplyError({ metadata: { reply } }));
}
});
},
], cb);
}
fail(jobId, error, cb) {
redis_smq_common_1.async.waterfall([
(next) => {
this.get(jobId, (err, backgroundJob) => {
if (err)
return next(err);
if (!backgroundJob)
return next(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
next(null, backgroundJob);
});
},
(backgroundJob, next) => {
const { keyJobWorker } = redis_keys_js_1.redisKeys.getJobKeys(jobId);
const updatedJob = Object.assign(Object.assign({}, backgroundJob), { status: index_js_2.EBackgroundJobStatus.FAILED, updatedAt: Date.now(), error });
this.redisClient.runScript(ELuaScript.FAIL_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
JSON.stringify(updatedJob),
index_js_2.EBackgroundJobStatus.PROCESSING.toString(),
index_js_2.EBackgroundJobStatus.COMPLETED.toString(),
index_js_2.EBackgroundJobStatus.FAILED.toString(),
index_js_2.EBackgroundJobStatus.CANCELED.toString(),
], (err, reply) => {
if (err)
return next(err);
switch (reply) {
case 1:
this.logger.error(`Job ${jobId} failed: ${error}`);
return next(null, updatedJob);
case 2:
this.logger.warn(`Job ${jobId} was already failed`);
return next(null);
case 0:
return next(new index_js_1.BackgroundJobNotFoundError({ metadata: { jobId } }));
case -1:
return next(new index_js_1.BackgroundJobNotFailableError({
metadata: {
jobId,
reason: "Job cannot be marked as failed because it's already completed",
},
}));
case -2:
return next(new index_js_1.BackgroundJobNotFailableError({
metadata: {
jobId,
reason: "Job cannot be marked as failed because it's already cancelled",
},
}));
default:
return next(new index_js_1.UnexpectedScriptReplyError({ metadata: { reply } }));
}
});
},
], cb);
}
list(filter, cb) {
const limit = (filter === null || filter === void 0 ? void 0 : filter.limit) || 50;
this.redisClient.hkeys(this.config.keyBackgroundJobs, (err, jobIds) => {
if (err)
return cb(err);
if (!jobIds)
return cb(null, []);
const jobsToGet = jobIds.slice(0, limit * 2);
const jobs = [];
let processed = 0;
const processNext = () => {
if (processed >= jobsToGet.length || jobs.length >= limit) {
return cb(null, jobs);
}
const jobId = jobsToGet[processed];
processed++;
this.get(jobId, (err, backgroundJob) => {
if (err) {
this.logger.warn(`Failed to get job ${jobId}:`, err);
return processNext();
}
if (backgroundJob) {
if ((filter === null || filter === void 0 ? void 0 : filter.status) && backgroundJob.status !== filter.status) {
return processNext();
}
if ((filter === null || filter === void 0 ? void 0 : filter.target) && backgroundJob.payload !== filter.target) {
return processNext();
}
jobs.push(backgroundJob);
}
processNext();
});
};
processNext();
});
}
recoverStuckJobs(cb) {
this.logger.debug(`Checking for stuck jobs...`);
this.redisClient.lrange(this.config.keyBackgroundJobsProcessing, 0, -1, (err, jobIds) => {
if (err)
return cb(err);
if (!jobIds || jobIds.length === 0) {
this.logger.debug('No stuck jobs found');
return cb(null);
}
this.logger.debug(`Found ${jobIds.length} potentially stuck jobs`);
let recovered = 0;
let skipped = 0;
let failed = 0;
redis_smq_common_1.async.eachOf(jobIds, (jobId, _, next) => {
redis_smq_common_1.async.waterfall([
(cb) => {
(0, _is_background_job_worker_alive_js_1._isBackgroundJobWorkerAlive)(jobId, (err, isAlive) => {
if (err) {
this.logger.error(`Error checking worker liveness for job ${jobId}:`, err);
return cb(err);
}
if (isAlive) {
this.logger.debug(`Worker for job ${jobId} is still alive, skipping recovery`);
return cb(null, false);
}
cb(null, true);
});
},
(shouldRecover, cb) => {
if (!shouldRecover) {
skipped++;
return cb(null);
}
this.get(jobId, (err, job) => {
if (err) {
this.logger.error(`Error getting job ${jobId} for recovery:`, err);
failed++;
return cb(err);
}
if (!job) {
this.logger.debug(`Job ${jobId} not found, removing from processing list`);
this.removeFromProcessing(jobId, () => {
skipped++;
cb(null);
});
return;
}
const { keyJobWorker } = redis_keys_js_1.redisKeys.getJobKeys(jobId);
const recoveryMessage = 'Recovered from worker crash';
const updatedJob = Object.assign(Object.assign({}, job), { status: index_js_2.EBackgroundJobStatus.PENDING, error: recoveryMessage, updatedAt: Date.now() });
this.redisClient.runScript(ELuaScript.RECOVER_STUCK_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsPending,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
JSON.stringify(updatedJob),
index_js_2.EBackgroundJobStatus.PROCESSING.toString(),
index_js_2.EBackgroundJobStatus.COMPLETED.toString(),
index_js_2.EBackgroundJobStatus.FAILED.toString(),
index_js_2.EBackgroundJobStatus.CANCELED.toString(),
recoveryMessage,
], (err, reply) => {
if (err) {
this.logger.error(`Error recovering job ${jobId}:`, err);
failed++;
return cb(err);
}
switch (reply) {
case 1:
this.logger.info(`Successfully recovered stuck job ${jobId}`);
recovered++;
break;
case 0:
this.logger.debug(`Job ${jobId} not found or not in recoverable state`);
skipped++;
break;
case -1:
this.logger.debug(`Job ${jobId} already completed`);
skipped++;
break;
case -2:
this.logger.debug(`Job ${jobId} already failed`);
skipped++;
break;
case -3:
this.logger.debug(`Job ${jobId} already cancelled`);
skipped++;
break;
default:
this.logger.warn(`Unexpected reply for job ${jobId}: ${reply}`);
failed++;
}
cb(null);
});
});
},
], (err) => next(err));
}, (err) => {
if (err) {
this.logger.error('Error during stuck job recovery:', err);
return cb(err);
}
this.logger.debug(`Stuck job recovery complete: ${recovered} recovered, ${skipped} skipped, ${failed} failed`);
cb(null);
});
});
}
}
exports.BackgroundJobManagerAbstract = BackgroundJobManagerAbstract;
//# sourceMappingURL=background-job-manager-abstract.js.map