redis-smq
Version:
A high-performance, reliable, and scalable message queue for Node.js.
512 lines • 23.3 kB
JavaScript
import { async, env } from 'redis-smq-common';
import { randomUUID } from 'node:crypto';
import { BackgroundJobAlreadyExistsError, BackgroundJobNotCancellableError, BackgroundJobNotCompletableError, BackgroundJobNotFailableError, BackgroundJobNotFoundError, BackgroundJobNotStartableError, UnexpectedScriptReplyError, } from '../../../errors/index.js';
import { EBackgroundJobStatus, } from './types/index.js';
import { resolve } from 'path';
import { redisKeys } from '../../redis/redis-keys/redis-keys.js';
import { _isBackgroundJobWorkerAlive } from './helpers/_is-background-job-worker-alive.js';
const batchSize = 1000;
const delay = 5000;
var ELuaScript;
(function (ELuaScript) {
ELuaScript["CREATE_JOB"] = "CREATE_JOB";
ELuaScript["CANCEL_JOB"] = "CANCEL_JOB";
ELuaScript["COMPLETE_JOB"] = "COMPLETE_JOB";
ELuaScript["FAIL_JOB"] = "FAIL_JOB";
ELuaScript["START_JOB"] = "START_JOB";
ELuaScript["RECOVER_STUCK_JOB"] = "RECOVER_STUCK_JOB";
})(ELuaScript || (ELuaScript = {}));
const curDir = env.getCurrentDir();
const luaScriptMap = {
[ELuaScript.CREATE_JOB]: resolve(curDir, './redis/scripts/create-job.lua'),
[ELuaScript.CANCEL_JOB]: resolve(curDir, './redis/scripts/cancel-job.lua'),
[ELuaScript.COMPLETE_JOB]: resolve(curDir, './redis/scripts/complete-job.lua'),
[ELuaScript.FAIL_JOB]: resolve(curDir, './redis/scripts/fail-job.lua'),
[ELuaScript.START_JOB]: resolve(curDir, './redis/scripts/start-job.lua'),
[ELuaScript.RECOVER_STUCK_JOB]: resolve(curDir, './redis/scripts/recover-stuck-job.lua'),
};
export class BackgroundJobManagerAbstract {
config;
redisClient;
logger;
constructor(redisClient, config, logger) {
this.redisClient = redisClient;
this.logger = logger.createLogger(this.constructor.name);
this.config = config;
}
applyPartialUpdate(job, partial) {
const { meta: partialMeta, ...partialFields } = partial;
const result = {
...job,
...partialFields,
};
if (partialMeta) {
result.meta = {
...job.meta,
...partialMeta,
};
}
return result;
}
update(jobId, updates, cb) {
this.get(jobId, (err, backgroundJob) => {
if (err)
return cb(err);
if (!backgroundJob)
return cb(new BackgroundJobNotFoundError({ metadata: { jobId } }));
const updatedJob = this.applyPartialUpdate(backgroundJob, {
...updates,
updatedAt: Date.now(),
});
this.redisClient.hset(this.config.keyBackgroundJobs, jobId, JSON.stringify(updatedJob), (setErr) => cb(setErr));
});
}
removeFromProcessing(jobId, cb) {
this.redisClient.lrem(this.config.keyBackgroundJobsProcessing, 0, jobId, cb);
}
initialize(cb) {
this.logger.debug('Loading Redis Lua scripts');
this.redisClient.loadScriptFiles(luaScriptMap, (err) => {
if (err) {
this.logger.error(`Failed to load Redis Lua scripts: ${err.message}`, err);
}
else {
this.logger.debug('Redis Lua scripts loaded successfully');
}
cb(err);
});
}
create(payload, options = {}, cb) {
const jobId = options.id ?? randomUUID();
const now = Date.now();
const backgroundJob = {
batchSize,
delay,
...options,
id: jobId,
payload,
status: EBackgroundJobStatus.PENDING,
createdAt: now,
updatedAt: now,
};
this.redisClient.runScript(ELuaScript.CREATE_JOB, [this.config.keyBackgroundJobs, this.config.keyBackgroundJobsPending], [jobId, JSON.stringify(backgroundJob), jobId], (err, reply) => {
if (err)
return cb(err);
switch (reply) {
case -1:
return cb(new BackgroundJobAlreadyExistsError({ metadata: { jobId } }));
case 1:
this.logger.debug(`Created job ${jobId} for target "${JSON.stringify(payload)}"`);
return cb(null, backgroundJob);
default:
return cb(new UnexpectedScriptReplyError({ metadata: { reply } }));
}
});
}
get(jobId, cb) {
this.redisClient.hget(this.config.keyBackgroundJobs, jobId, (err, data) => {
if (err)
return cb(err);
if (!data)
return cb(new BackgroundJobNotFoundError({ metadata: { jobId } }));
cb(null, JSON.parse(data));
});
}
acquireNextJob(cb) {
this.redisClient.brpoplpush(this.config.keyBackgroundJobsPending, this.config.keyBackgroundJobsProcessing, 0, cb);
}
cancel(jobId, cb) {
async.waterfall([
(next) => {
this.get(jobId, (err, backgroundJob) => {
if (err)
return next(err);
if (!backgroundJob)
return next(new BackgroundJobNotFoundError({ metadata: { jobId } }));
next(null, backgroundJob);
});
},
(backgroundJob, next) => {
const { keyJobWorker } = redisKeys.getJobKeys(jobId);
const updatedJob = {
...backgroundJob,
status: EBackgroundJobStatus.CANCELED,
updatedAt: Date.now(),
};
this.redisClient.runScript(ELuaScript.CANCEL_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsPending,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
JSON.stringify(updatedJob),
EBackgroundJobStatus.PENDING.toString(),
EBackgroundJobStatus.PROCESSING.toString(),
EBackgroundJobStatus.COMPLETED.toString(),
EBackgroundJobStatus.FAILED.toString(),
EBackgroundJobStatus.CANCELED.toString(),
], (err, result) => {
if (err)
return next(err);
switch (result) {
case 1:
this.logger.info(`Cancelled job ${jobId}`);
return next(null, updatedJob);
case 2:
this.logger.warn(`Job ${jobId} was already cancelled`);
return next(null);
case 0:
return next(new BackgroundJobNotFoundError({ metadata: { jobId } }));
case -1:
return next(new BackgroundJobNotCancellableError({
metadata: { jobId, reason: 'already completed' },
}));
case -2:
return next(new BackgroundJobNotCancellableError({
metadata: { jobId, reason: 'already failed' },
}));
default:
return next(new Error(`Unexpected result from cancel script: ${result}`));
}
});
},
], cb);
}
start(jobId, workerId, cb) {
this.get(jobId, (err, backgroundJob) => {
if (err)
return cb(err);
if (!backgroundJob)
return cb(new BackgroundJobNotFoundError({ metadata: { jobId } }));
const { keyJobWorker } = redisKeys.getJobKeys(jobId);
const updatedJob = {
...backgroundJob,
status: EBackgroundJobStatus.PROCESSING,
startedAt: Date.now(),
updatedAt: Date.now(),
};
this.redisClient.runScript(ELuaScript.START_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
workerId,
JSON.stringify(updatedJob),
EBackgroundJobStatus.PENDING.toString(),
EBackgroundJobStatus.PROCESSING.toString(),
EBackgroundJobStatus.COMPLETED.toString(),
EBackgroundJobStatus.FAILED.toString(),
EBackgroundJobStatus.CANCELED.toString(),
], (err, reply) => {
if (err)
return cb(err);
switch (reply) {
case 1:
this.logger.info(`Started processing job ${jobId}`);
return cb(null, updatedJob);
case 2:
this.logger.warn(`Job ${jobId} was already processing`);
return cb(null);
case 0:
return cb(new BackgroundJobNotFoundError({ metadata: { jobId } }));
case -1:
return cb(new BackgroundJobNotStartableError({
metadata: {
jobId,
reason: "Job cannot be started because it's already completed",
},
}));
case -2:
return cb(new BackgroundJobNotStartableError({
metadata: {
jobId,
reason: "Job cannot be started because it's already failed",
},
}));
case -3:
return cb(new BackgroundJobNotStartableError({
metadata: {
jobId,
reason: "Job cannot be started because it's already cancelled",
},
}));
default:
return cb(new UnexpectedScriptReplyError({ metadata: { reply } }));
}
});
});
}
complete(jobId, options = {}, cb) {
async.waterfall([
(next) => {
this.get(jobId, (err, backgroundJob) => {
if (err)
return next(err);
if (!backgroundJob)
return next(new BackgroundJobNotFoundError({ metadata: { jobId } }));
next(null, backgroundJob);
});
},
(backgroundJob, next) => {
const { keyJobWorker } = redisKeys.getJobKeys(jobId);
const updatedJob = this.applyPartialUpdate(backgroundJob, {
...options,
status: EBackgroundJobStatus.COMPLETED,
completedAt: Date.now(),
updatedAt: Date.now(),
});
this.redisClient.runScript(ELuaScript.COMPLETE_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
JSON.stringify(updatedJob),
EBackgroundJobStatus.PROCESSING.toString(),
EBackgroundJobStatus.COMPLETED.toString(),
EBackgroundJobStatus.FAILED.toString(),
EBackgroundJobStatus.CANCELED.toString(),
], (err, reply) => {
if (err)
return next(err);
switch (reply) {
case 1:
this.logger.info(`Completed job ${jobId}`);
return next(null, updatedJob);
case 2:
this.logger.warn(`Job ${jobId} was already completed`);
return next(null);
case 0:
return next(new BackgroundJobNotFoundError({ metadata: { jobId } }));
case -1:
return next(new BackgroundJobNotCompletableError({
metadata: { jobId, reason: 'already failed' },
}));
case -2:
return next(new BackgroundJobNotCompletableError({
metadata: { jobId, reason: 'already cancelled' },
}));
default:
return next(new UnexpectedScriptReplyError({ metadata: { reply } }));
}
});
},
], cb);
}
fail(jobId, error, cb) {
async.waterfall([
(next) => {
this.get(jobId, (err, backgroundJob) => {
if (err)
return next(err);
if (!backgroundJob)
return next(new BackgroundJobNotFoundError({ metadata: { jobId } }));
next(null, backgroundJob);
});
},
(backgroundJob, next) => {
const { keyJobWorker } = redisKeys.getJobKeys(jobId);
const updatedJob = {
...backgroundJob,
status: EBackgroundJobStatus.FAILED,
updatedAt: Date.now(),
error,
};
this.redisClient.runScript(ELuaScript.FAIL_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
JSON.stringify(updatedJob),
EBackgroundJobStatus.PROCESSING.toString(),
EBackgroundJobStatus.COMPLETED.toString(),
EBackgroundJobStatus.FAILED.toString(),
EBackgroundJobStatus.CANCELED.toString(),
], (err, reply) => {
if (err)
return next(err);
switch (reply) {
case 1:
this.logger.error(`Job ${jobId} failed: ${error}`);
return next(null, updatedJob);
case 2:
this.logger.warn(`Job ${jobId} was already failed`);
return next(null);
case 0:
return next(new BackgroundJobNotFoundError({ metadata: { jobId } }));
case -1:
return next(new BackgroundJobNotFailableError({
metadata: {
jobId,
reason: "Job cannot be marked as failed because it's already completed",
},
}));
case -2:
return next(new BackgroundJobNotFailableError({
metadata: {
jobId,
reason: "Job cannot be marked as failed because it's already cancelled",
},
}));
default:
return next(new UnexpectedScriptReplyError({ metadata: { reply } }));
}
});
},
], cb);
}
list(filter, cb) {
const limit = filter?.limit || 50;
this.redisClient.hkeys(this.config.keyBackgroundJobs, (err, jobIds) => {
if (err)
return cb(err);
if (!jobIds)
return cb(null, []);
const jobsToGet = jobIds.slice(0, limit * 2);
const jobs = [];
let processed = 0;
const processNext = () => {
if (processed >= jobsToGet.length || jobs.length >= limit) {
return cb(null, jobs);
}
const jobId = jobsToGet[processed];
processed++;
this.get(jobId, (err, backgroundJob) => {
if (err) {
this.logger.warn(`Failed to get job ${jobId}:`, err);
return processNext();
}
if (backgroundJob) {
if (filter?.status && backgroundJob.status !== filter.status) {
return processNext();
}
if (filter?.target && backgroundJob.payload !== filter.target) {
return processNext();
}
jobs.push(backgroundJob);
}
processNext();
});
};
processNext();
});
}
recoverStuckJobs(cb) {
this.logger.debug(`Checking for stuck jobs...`);
this.redisClient.lrange(this.config.keyBackgroundJobsProcessing, 0, -1, (err, jobIds) => {
if (err)
return cb(err);
if (!jobIds || jobIds.length === 0) {
this.logger.debug('No stuck jobs found');
return cb(null);
}
this.logger.debug(`Found ${jobIds.length} potentially stuck jobs`);
let recovered = 0;
let skipped = 0;
let failed = 0;
async.eachOf(jobIds, (jobId, _, next) => {
async.waterfall([
(cb) => {
_isBackgroundJobWorkerAlive(jobId, (err, isAlive) => {
if (err) {
this.logger.error(`Error checking worker liveness for job ${jobId}:`, err);
return cb(err);
}
if (isAlive) {
this.logger.debug(`Worker for job ${jobId} is still alive, skipping recovery`);
return cb(null, false);
}
cb(null, true);
});
},
(shouldRecover, cb) => {
if (!shouldRecover) {
skipped++;
return cb(null);
}
this.get(jobId, (err, job) => {
if (err) {
this.logger.error(`Error getting job ${jobId} for recovery:`, err);
failed++;
return cb(err);
}
if (!job) {
this.logger.debug(`Job ${jobId} not found, removing from processing list`);
this.removeFromProcessing(jobId, () => {
skipped++;
cb(null);
});
return;
}
const { keyJobWorker } = redisKeys.getJobKeys(jobId);
const recoveryMessage = 'Recovered from worker crash';
const updatedJob = {
...job,
status: EBackgroundJobStatus.PENDING,
error: recoveryMessage,
updatedAt: Date.now(),
};
this.redisClient.runScript(ELuaScript.RECOVER_STUCK_JOB, [
this.config.keyBackgroundJobs,
this.config.keyBackgroundJobsPending,
this.config.keyBackgroundJobsProcessing,
keyJobWorker,
], [
jobId,
JSON.stringify(updatedJob),
EBackgroundJobStatus.PROCESSING.toString(),
EBackgroundJobStatus.COMPLETED.toString(),
EBackgroundJobStatus.FAILED.toString(),
EBackgroundJobStatus.CANCELED.toString(),
recoveryMessage,
], (err, reply) => {
if (err) {
this.logger.error(`Error recovering job ${jobId}:`, err);
failed++;
return cb(err);
}
switch (reply) {
case 1:
this.logger.info(`Successfully recovered stuck job ${jobId}`);
recovered++;
break;
case 0:
this.logger.debug(`Job ${jobId} not found or not in recoverable state`);
skipped++;
break;
case -1:
this.logger.debug(`Job ${jobId} already completed`);
skipped++;
break;
case -2:
this.logger.debug(`Job ${jobId} already failed`);
skipped++;
break;
case -3:
this.logger.debug(`Job ${jobId} already cancelled`);
skipped++;
break;
default:
this.logger.warn(`Unexpected reply for job ${jobId}: ${reply}`);
failed++;
}
cb(null);
});
});
},
], (err) => next(err));
}, (err) => {
if (err) {
this.logger.error('Error during stuck job recovery:', err);
return cb(err);
}
this.logger.debug(`Stuck job recovery complete: ${recovered} recovered, ${skipped} skipped, ${failed} failed`);
cb(null);
});
});
}
}
//# sourceMappingURL=background-job-manager-abstract.js.map