@lexamica-modules/job-queue
Version:
The package for the Lexamica Job Queue SDK powered by Redis and BullMQ
548 lines (547 loc) • 22.2 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.CommonQueueManagement = void 0;
/* eslint-disable max-lines */
const errors_1 = require("../util/errors");
const bullmq_1 = require("bullmq");
const queues_1 = require("../queues");
const types_1 = require("../types");
const workers_1 = require("../workers");
const jobs_1 = __importDefault(require("../jobs"));
const config_1 = require("../util/config");
const ioredis_1 = __importDefault(require("ioredis"));
const { READ, WRITE, EXECUTE } = types_1.QueuePermissions;
/**
* This is the common queue management class for the Lexamica Message Queue package. This class provides a protected API for using the Lexamica message queues that provents the erroneous mutiliating, processing, or deleting of message queues and their jobs. Create an instance of this class, and then call the .init() function to start.
*/
class CommonQueueManagement {
// <Queue Name, Queue>
queues;
// <Queue Name, Workers>
workers;
// <Job Id, Job>
jobs;
// <Job Id, Job>
delayed_jobs;
// <Queue Name, Queue Events>
listeners;
// Queue Names []
init_names;
// either the mainframe or an integration API
context;
// access control
access_control = config_1.QUEUES_INIT;
// track init status
initialized = false;
QUEUES;
WORKERS;
JOBS;
/**
* The constructor that accepts the config props necessary to create a class instance
* @param {string[]} initQueues the list of queue names to initialize. This will likely be managed by the specific instance sub-class you should be using.
* @param {Consumers} context one of the available instances defined. Your instance sub-class will automatically provide this.
* @param {string} connection_url the connection url to a Redis DB. If no connection url is given, a local Redis instance will be used.
*/
constructor(initQueues, context, connection_url) {
this.queues = {};
this.workers = {};
this.jobs = {};
this.delayed_jobs = {};
this.listeners = {};
this.init_names = initQueues;
this.context = context;
if (!connection_url) {
console.log("WARNING: Remote Redis connection string was not given. Using local instance instead. If you are on a local environment, disregard.");
}
const redis = !!connection_url
? new ioredis_1.default(connection_url)
: new ioredis_1.default();
this.QUEUES = new queues_1.Queues(redis);
this.WORKERS = new workers_1.Workers(redis);
this.JOBS = new jobs_1.default();
}
/**
* Create the default queues and start tracking them in this class instance. Each queue will also either be created in Redis, or hooked into an existing instance of that queue set up previously.
* @param {{ [name: string]: JobExecuter }} executors A object with the keys set to queue names, and the values set to the default worker execution functions that will be assigned to the workers in this queue. Additional workers with different execution functions can be added later.
* @param {number} workers How many workers to create by default for this queue. If you are unsure, pick 1.
*/
init = async (executors, workers) => {
// init queues based on names given
for (const name of this.init_names) {
try {
await this.addQueue(name, workers, executors[name]);
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: `Failed to create an initializer queue. This is required`,
job: "Initialize Queues",
error: err,
handle: false,
});
}
}
console.log(`Initialized ${this.init_names.length} queues: ${this.init_names} with ${workers * this.init_names.length} workers assigned ${workers} to each queue`);
this.initialized = true;
};
/**
* Utility function that throws an error if queues have not been initialized before trying to call class functions
* @throws {Error} Queues have not been initialized. Call the .init() method.
*/
_checkInit = () => {
if (!this.initialized) {
throw new Error("Instance has not been initialized. Call .init() before calling other class methods");
}
};
/**
* Utility function that throws an error if the queue specified has not been created in this instance
* @param {string} queue_name the name of the queue
* @throws {Error} Queue has likely not been created in this class instance.
*/
_verifyQueue = (queue_name) => {
if (!this.queues[queue_name]) {
throw new Error("Queue has not been created");
}
};
/**
* Utility function that verifies that the current instance sub-class of this class is allowed to perform the queue action desired. This protects instances from process, deleting, etc. jobs not intended for them.
* @param {string} queue_name The name of the queue being checked
* @param {QueuePermissions} access one of the queue permissions like READ, WRITE, etc.
* @throws {Error} queue access is denied for the given access type
* @returns {boolean} returns true if access is allowed. Throws an error otherwise
*/
_verifyAccess = (queue_name, access) => {
this._verifyQueue(queue_name);
const config = this.access_control[queue_name];
if (!config) {
// no access control configured. Most likely a custom queue with access allowed
return true;
}
if (config[this.context].scopes.includes(access)) {
return true;
}
else {
throw new Error(`Access to the ${access} scope is on the queue: ${queue_name} is not allowed for a ${this.context}`);
}
};
/**
* Accepts a job ID in an object and removes it from the applicable tracked jobs list.
* @param {{ jobId: string }} param0
*/
_removeJob = ({ jobId }) => {
delete this.jobs[jobId];
// if this is a delayed job, delete it.
if (this.delayed_jobs[jobId]) {
delete this.delayed_jobs[jobId];
}
};
/**
* Gets the list of queues being tracked in this class instance
* @returns {Record<string, Queue>}
*/
getQueues = () => {
this._checkInit();
return this.queues;
};
/**
* Gets the list of workers currently being tracked against the queue given in this class instance
* @param {string} queue_name the name of the queue
* @returns {Worker[]}
*/
getWorkers(queue_name) {
this._checkInit();
this._verifyQueue(queue_name);
return this.workers[queue_name];
}
/**
* Gets the non-delayed jobs currently being tracked in this class instance. NOTE: this does not represent all jobs in the Redis queue, and may not even be completely consistent for this class instance. This is mainly to aid in finding a Job instance or ID if possible.
* @param {string} queue_name the name of the queue the job is in
* @param {boolean} withData set to true to get the job data payload as well
* @returns An object with the keys set to the job IDs and the values set to a job payload.
*/
getJobs(queue_name, withData) {
this._checkInit();
const queueJobs = Object.values(this.jobs).filter((job) => {
job.queueName === queue_name;
});
const cleanOutput = queueJobs.reduce((prev, job) => {
prev = {
...prev,
[job.id ?? "00"]: {
name: job.name,
queue: job.queueName,
attempts: job.attemptsMade,
status: job.progress,
data: withData ? job.data : null,
},
};
return prev;
}, {});
return cleanOutput;
}
/**
* Gets the delayed jobs currently being tracked in this class instance. NOTE: this does not represent all jobs in the Redis queue, and may not even be completely consistent for this class instance. This is mainly to aid in finding a Job instance or ID if possible.
* @param {string} queue_name the name of the queue the job is in
* @param {boolean} withData set to true to get the job data payload as well
* @returns An object with the keys set to the job IDs and the values set to a job payload.
*/
getDelayedJobs(queue_name, withData) {
this._checkInit();
const queueJobs = Object.values(this.delayed_jobs).filter((job) => {
job.queueName === queue_name;
});
const cleanOutput = queueJobs.reduce((prev, job) => {
prev = {
...prev,
[job.id ?? "00"]: {
name: job.name,
queue: job.queueName,
delay: job.delay,
status: job.progress,
data: withData ? job.data : null,
},
};
return prev;
}, {});
return cleanOutput;
}
/**
* Add a new queue instance to this class instance. This will create the queue in Redis if it does not exist, or will link to an existing queue in Redis with the same name.
* @param {string} queue_name the name of the queue
* @param {number} workers the number of workers to initialize with this queue
* @param {JobExecuter} executer the execution function to assign to this queue's workers
* @returns {Promise<{ queue: Queue | null; workers: Worker[] }>}
*/
addQueue = async (queue_name, workers, executer) => {
const workers_created = [];
let queue_created = null;
try {
// if this is an existing queue, this will check to see if this consumer can access it
this._verifyAccess(queue_name, types_1.QueuePermissions.READ);
const queue = this.QUEUES.create(queue_name);
if (queue) {
queue_created = queue;
}
else {
throw new Error(`Failed to create queue: ${queue_name}`);
}
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: `Failed to create a queue.`,
job: "Add Queue",
error: err,
handle: false,
});
}
for (let i = 0; i < workers; i++) {
try {
// create a worker and add it to the applicable queue array of workers
const worker = this.WORKERS.addWorker(queue_name, executer, {
concurrency: 1,
});
workers_created.push(worker);
}
catch (err) {
// delete queue after failure
this.QUEUES.delete(queue_created);
await Promise.all(workers_created.map((worker) => worker.close()));
(0, errors_1.handleErrorWithInfo)({
message: `Failed to create a worker for a queue`,
job: "Add Queue",
error: err,
handle: false,
});
}
}
if (queue_created) {
this.queues[queue_name] = queue_created;
this.workers[queue_name] = workers_created;
this.listeners[queue_name] = new bullmq_1.QueueEvents(queue_name);
this.listeners[queue_name].on("completed", this._removeJob);
// update access control
this.access_control = {
...this.access_control,
[queue_name]: {
[this.context]: {
scopes: [READ, WRITE, EXECUTE],
init: false,
},
},
};
}
return { queue: queue_created, workers: workers_created };
};
/**
* Removes all jobs associated with this queue
* @param {string} queue_name the name of the queue
* @returns {Promise<boolean>}
*/
clearQueue = async (queue_name) => {
this._checkInit();
try {
this._verifyAccess(queue_name, types_1.QueuePermissions.WRITE);
// check to make sure queue exists
this._verifyQueue(queue_name);
const drain = await this.QUEUES.drain(this.queues[queue_name]);
// remove all tracked jobs in this instance if queue was drained correctly
if (drain) {
for (const job of Object.values(this.jobs)) {
if (job.queueName === queue_name && job.id) {
delete this.jobs[job.id];
}
}
}
return drain;
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: "Failed to drain a queue",
job: "Clear Queue",
error: err,
});
return false;
}
};
/**
* Remove the queue from this class instance and from the Redis DB
* @param {string} queue_name the name of the queue
* @returns {Promise<boolean>}
*/
deleteQueue = async (queue_name) => {
this._checkInit();
try {
this._verifyAccess(queue_name, types_1.QueuePermissions.WRITE);
// check to make sure queue exists
this._verifyQueue(queue_name);
// delete queue from system
await this.QUEUES.delete(this.queues[queue_name]);
// delete queue from directory
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: "Failed to delete a queue from the system",
job: "Delete Queue",
error: err,
});
return false;
}
try {
delete this.queues[queue_name];
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: "Failed to delete a queue from the directory",
job: "Delete Queue",
error: err,
});
return false;
}
try {
// close all workers
await Promise.all(this.workers[queue_name].map((worker) => worker.close()));
// remove workers from directory
delete this.workers[queue_name];
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: "Failed to close deleted queue workers from the system",
job: "Delete Queue",
error: err,
});
// remove directory entry anyway
delete this.workers[queue_name];
return false;
}
return true;
};
/**
* Create a new worker in this instance
* @param {string} queue_name the queue to assign this worker to
* @param {JobExecuter} executer the function to execute for this worker's jobs
* @param {WorkerOptions} options the config options to configure the worker
* @returns {Worker | void}
*/
addWorker = (queue_name, executer, options) => {
this._checkInit();
try {
this._verifyAccess(queue_name, types_1.QueuePermissions.EXECUTE);
// check to make sure queue exists
this._verifyQueue(queue_name);
// create the worker
const worker = this.WORKERS.addWorker(queue_name, executer, options);
// store the worker in the directory
if (!this.workers[queue_name]) {
this.workers[queue_name] = [worker];
}
else {
this.workers[queue_name].push(worker);
}
return worker;
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: `Failed to create a worker for queue: ${queue_name}`,
job: "Create Worker",
error: err,
});
return;
}
};
/**
* Finished current jobs and close the worker from accepting any more jobs
* @param {Worker} worker the worker instance to close
* @returns {Promise<void>}
*/
closeWorker = async (queue_name, worker) => {
this._checkInit();
try {
this._verifyAccess(queue_name, types_1.QueuePermissions.EXECUTE);
// check to make sure queue exists
this._verifyQueue(queue_name);
await this.WORKERS.closeWorker(worker);
// find worker in directory
const location = this.workers[queue_name].findIndex((i_worker) => worker.id === i_worker.id);
// remove worker from directory
this.workers[queue_name].splice(location, 1);
return;
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: `Failed to close worker`,
job: "Close Worker",
error: err,
});
return;
}
};
/**
* Finished all jobs in a queue and close all of its workers from accepting new jobs
* @param {string} queue_name the name of the queue to close all workers in
* @returns {Promise<void>}
*/
closeAllWorkersInQueue = async (queue_name) => {
this._checkInit();
try {
this._verifyAccess(queue_name, types_1.QueuePermissions.EXECUTE);
// check to make sure queue exists
this._verifyQueue(queue_name);
await Promise.all(this.workers[queue_name].map((worker) => worker.close()));
return;
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: `Failed to close workers for queue: ${queue_name}`,
job: "Close All Workers In Queue",
error: err,
});
return;
}
};
/**
* Add a new job to a queue
* @param {string} queue_name name of the queue
* @param {string} job_name the name to assign this job
* @param {Record<string, unknown>} job_data data payload to attach to this job
* @param {DefaultJobOptions} job_options job options to config this job
* @param {number} delay add an optional processing delay to this job in milliseconds
* @returns {Promise<Job | void>}
*/
addJob = async (queue_name, job_name, job_data, job_options, delay) => {
this._checkInit();
try {
this._verifyAccess(queue_name, types_1.QueuePermissions.WRITE);
// check to make sure queue exists
this._verifyQueue(queue_name);
const queue = this.queues[queue_name];
if (delay) {
try {
const delayedJob = await this.JOBS.addDelayedJob(queue, job_name, job_data, delay, job_options);
// add the job to the directory
if (delayedJob.id) {
this.jobs[delayedJob.id] = delayedJob;
this.delayed_jobs[delayedJob.id] = delayedJob;
}
return delayedJob;
}
catch (err) {
throw new Error(`Delayed Job Creation Error: ${err}`);
}
}
else {
try {
const job = await this.JOBS.addJob(queue, job_name, job_data, job_options);
// add the job to the directory
if (job.id) {
this.jobs[job.id] = job;
}
return job;
}
catch (err) {
throw new Error(`Job Creation Error: ${err}`);
}
}
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: `Failed to add a new Job`,
job: "Add Job",
error: err,
});
}
};
/**
* Cancel a job and pervent it from processing.
* @param {Job} job instance of the job to cancel
* @returns {Promise<boolean>}
*/
cancelJob = async (job) => {
this._checkInit();
try {
this._verifyAccess(job.queueName, types_1.QueuePermissions.WRITE);
await this.JOBS.cancelJob(job);
if (job.id) {
this._removeJob({ jobId: job.id });
}
return true;
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: `Failed to cancel a job`,
job: "Cancel Job",
error: err,
});
return false;
}
};
/**
* Cancel all jobs being tracked in this instance in a given queue from executing
* @param {string} queue_name name of the queue
* @returns {Promise<number>}
*/
cancelAllJobsInQueue = async (queue_name) => {
this._checkInit();
try {
this._verifyAccess(queue_name, types_1.QueuePermissions.WRITE);
// check to make sure queue exists
this._verifyQueue(queue_name);
// find all jobs
const jobs = Object.values(this.jobs).filter((job) => {
return job.queueName === queue_name;
});
await Promise.all(jobs.map((job) => this.cancelJob(job)));
return jobs.length;
}
catch (err) {
(0, errors_1.handleErrorWithInfo)({
message: `Failed to cancel all jobs in queue: ${queue_name}`,
job: "Cancel All Jobs In Queue",
error: err,
});
return -1;
}
};
}
exports.CommonQueueManagement = CommonQueueManagement;