@lexamica-modules/job-queue/src/instances/common.ts

The package for the Lexamica Job Queue SDK powered by Redis and BullMQ

/* eslint-disable max-lines */
import { handleErrorWithInfo } from "../util/errors";
import { DefaultJobOptions, Job, Queue, Worker, QueueEvents } from "bullmq";
import { Queues } from "../queues";
import {
  Consumers,
  JobExecuter,
  QueuePermissions,
  WorkerOptions,
} from "../types";
import { Workers } from "../workers";
import Jobs from "../jobs";
import { QUEUES_INIT } from "../util/config";
import IORedis from "ioredis";

const { READ, WRITE, EXECUTE } = QueuePermissions;

/**
 * This is the common queue management class for the Lexamica Message Queue package. This class provides a protected API for using the Lexamica message queues that provents the erroneous mutiliating, processing, or deleting of message queues and their jobs. Create an instance of this class, and then call the .init() function to start.
 */
export class CommonQueueManagement {
  // <Queue Name, Queue>
  private queues: Record<string, Queue<unknown, unknown, string>>;
  // <Queue Name, Workers>
  private workers: Record<string, Worker[]>;
  // <Job Id, Job>
  private jobs: Record<string, Job>;
  // <Job Id, Job>
  private delayed_jobs: Record<string, Job>;
  // <Queue Name, Queue Events>
  private listeners: Record<string, QueueEvents>;
  // Queue Names []
  private init_names: string[];
  // either the mainframe or an integration API
  private context: Consumers;
  // access control
  private access_control = QUEUES_INIT;
  // track init status
  private initialized = false;

  private QUEUES: Queues;
  private WORKERS: Workers;
  private JOBS: Jobs;

  /**
   * The constructor that accepts the config props necessary to create a class instance
   * @param {string[]} initQueues the list of queue names to initialize. This will likely be managed by the specific instance sub-class you should be using.
   * @param {Consumers} context one of the available instances defined. Your instance sub-class will automatically provide this.
   * @param {string} connection_url the connection url to a Redis DB. If no connection url is given, a local Redis instance will be used.
   */
  constructor(
    initQueues: string[],
    context: Consumers,
    connection_url?: string,
  ) {
    this.queues = {};
    this.workers = {};
    this.jobs = {};
    this.delayed_jobs = {};
    this.listeners = {};
    this.init_names = initQueues;
    this.context = context;

    if (!connection_url) {
      console.log(
        "WARNING: Remote Redis connection string was not given. Using local instance instead. If you are on a local environment, disregard.",
      );
    }

    const redis = !!connection_url
      ? new IORedis(connection_url)
      : new IORedis();

    this.QUEUES = new Queues(redis);
    this.WORKERS = new Workers(redis);
    this.JOBS = new Jobs();
  }

  /**
   * Create the default queues and start tracking them in this class instance. Each queue will also either be created in Redis, or hooked into an existing instance of that queue set up previously.
   * @param {{ [name: string]: JobExecuter }} executors A object with the keys set to queue names, and the values set to the default worker execution functions that will be assigned to the workers in this queue. Additional workers with different execution functions can be added later.
   * @param {number} workers How many workers to create by default for this queue. If you are unsure, pick 1.
   */
  init = async (
    executors: {
      [name: string]: JobExecuter;
    },
    workers: number,
  ) => {
    // init queues based on names given
    for (const name of this.init_names) {
      try {
        await this.addQueue(name, workers, executors[name]);
      } catch (err) {
        handleErrorWithInfo({
          message: `Failed to create an initializer queue. This is required`,
          job: "Initialize Queues",
          error: err as Error,
          handle: false,
        });
      }
    }
    console.log(
      `Initialized ${this.init_names.length} queues: ${this.init_names} with ${
        workers * this.init_names.length
      } workers assigned ${workers} to each queue`,
    );
    this.initialized = true;
  };

  /**
   * Utility function that throws an error if queues have not been initialized before trying to call class functions
   * @throws {Error} Queues have not been initialized. Call the .init() method.
   */
  private _checkInit = (): void => {
    if (!this.initialized) {
      throw new Error(
        "Instance has not been initialized. Call .init() before calling other class methods",
      );
    }
  };

  /**
   * Utility function that throws an error if the queue specified has not been created in this instance
   * @param {string} queue_name the name of the queue
   * @throws {Error} Queue has likely not been created in this class instance.
   */
  private _verifyQueue = (queue_name: string): void => {
    if (!this.queues[queue_name]) {
      throw new Error("Queue has not been created");
    }
  };

  /**
   * Utility function that verifies that the current instance sub-class of this class is allowed to perform the queue action desired. This protects instances from process, deleting, etc. jobs not intended for them.
   * @param {string} queue_name The name of the queue being checked
   * @param {QueuePermissions} access one of the queue permissions like READ, WRITE, etc.
   * @throws {Error} queue access is denied for the given access type
   * @returns {boolean} returns true if access is allowed. Throws an error otherwise
   */
  private _verifyAccess = (
    queue_name: string,
    access: QueuePermissions,
  ): boolean => {
    this._verifyQueue(queue_name);

    const config = this.access_control[queue_name];

    if (!config) {
      // no access control configured. Most likely a custom queue with access allowed
      return true;
    }
    if (config[this.context].scopes.includes(access)) {
      return true;
    } else {
      throw new Error(
        `Access to the ${access} scope is on the queue: ${queue_name} is not allowed for a ${this.context}`,
      );
    }
  };

  /**
   * Accepts a job ID in an object and removes it from the applicable tracked jobs list.
   * @param {{ jobId: string }} param0
   */
  private _removeJob = ({ jobId }: { jobId: string }): void => {
    delete this.jobs[jobId];
    // if this is a delayed job, delete it.
    if (this.delayed_jobs[jobId]) {
      delete this.delayed_jobs[jobId];
    }
  };

  /**
   * Gets the list of queues being tracked in this class instance
   * @returns {Record<string, Queue>}
   */
  getQueues = (): Record<string, Queue> => {
    this._checkInit();
    return this.queues;
  };

  /**
   * Gets the list of workers currently being tracked against the queue given in this class instance
   * @param {string} queue_name the name of the queue
   * @returns {Worker[]}
   */
  getWorkers(queue_name: string): Worker[] {
    this._checkInit();
    this._verifyQueue(queue_name);
    return this.workers[queue_name];
  }

  /**
   * Gets the non-delayed jobs currently being tracked in this class instance. NOTE: this does not represent all jobs in the Redis queue, and may not even be completely consistent for this class instance. This is mainly to aid in finding a Job instance or ID if possible.
   * @param {string} queue_name the name of the queue the job is in
   * @param {boolean} withData set to true to get the job data payload as well
   * @returns An object with the keys set to the job IDs and the values set to a job payload.
   */
  getJobs(
    queue_name: string,
    withData?: boolean,
  ): Record<
    string,
    {
      queue: string;
      attempts: number;
      status: string;
      data: Record<string, unknown>;
    }
  > {
    this._checkInit();
    const queueJobs = Object.values(this.jobs).filter((job: Job) => {
      job.queueName === queue_name;
    });

    const cleanOutput = queueJobs.reduce((prev, job: Job) => {
      prev = {
        ...prev,
        [(job.id as string) ?? "00"]: {
          name: job.name,
          queue: job.queueName,
          attempts: job.attemptsMade,
          status: job.progress,
          data: withData ? job.data : null,
        },
      };
      return prev;
    }, {});

    return cleanOutput;
  }

  /**
   * Gets the delayed jobs currently being tracked in this class instance. NOTE: this does not represent all jobs in the Redis queue, and may not even be completely consistent for this class instance. This is mainly to aid in finding a Job instance or ID if possible.
   * @param {string} queue_name the name of the queue the job is in
   * @param {boolean} withData set to true to get the job data payload as well
   * @returns An object with the keys set to the job IDs and the values set to a job payload.
   */
  getDelayedJobs(
    queue_name: string,
    withData?: boolean,
  ): Record<
    string,
    {
      queue: string;
      attempts: number;
      status: string;
      data: Record<string, unknown>;
    }
  > {
    this._checkInit();
    const queueJobs = Object.values(this.delayed_jobs).filter((job: Job) => {
      job.queueName === queue_name;
    });

    const cleanOutput = queueJobs.reduce((prev, job: Job) => {
      prev = {
        ...prev,
        [(job.id as string) ?? "00"]: {
          name: job.name,
          queue: job.queueName,
          delay: job.delay,
          status: job.progress,
          data: withData ? job.data : null,
        },
      };
      return prev;
    }, {});

    return cleanOutput;
  }

  /**
   * Add a new queue instance to this class instance. This will create the queue in Redis if it does not exist, or will link to an existing queue in Redis with the same name.
   * @param {string} queue_name the name of the queue
   * @param {number} workers the number of workers to initialize with this queue
   * @param {JobExecuter} executer the execution function to assign to this queue's workers
   * @returns {Promise<{ queue: Queue | null; workers: Worker[] }>}
   */
  addQueue = async (
    queue_name: string,
    workers: number,
    executer: JobExecuter,
  ): Promise<{ queue: Queue | null; workers: Worker[] }> => {
    const workers_created: Worker[] = [];
    let queue_created: Queue | null = null;

    try {
      // if this is an existing queue, this will check to see if this consumer can access it
      this._verifyAccess(queue_name, QueuePermissions.READ);

      const queue = this.QUEUES.create(queue_name);
      if (queue) {
        queue_created = queue;
      } else {
        throw new Error(`Failed to create queue: ${queue_name}`);
      }
    } catch (err) {
      handleErrorWithInfo({
        message: `Failed to create a queue.`,
        job: "Add Queue",
        error: err as Error,
        handle: false,
      });
    }

    for (let i = 0; i < workers; i++) {
      try {
        // create a worker and add it to the applicable queue array of workers
        const worker = this.WORKERS.addWorker(queue_name, executer, {
          concurrency: 1,
        });
        workers_created.push(worker);
      } catch (err) {
        // delete queue after failure
        this.QUEUES.delete(queue_created as Queue);

        await Promise.all(workers_created.map((worker) => worker.close()));

        handleErrorWithInfo({
          message: `Failed to create a worker for a queue`,
          job: "Add Queue",
          error: err as Error,
          handle: false,
        });
      }
    }

    if (queue_created) {
      this.queues[queue_name] = queue_created;
      this.workers[queue_name] = workers_created;
      this.listeners[queue_name] = new QueueEvents(queue_name);
      this.listeners[queue_name].on("completed", this._removeJob);
      // update access control
      this.access_control = {
        ...this.access_control,
        [queue_name]: {
          [this.context]: {
            scopes: [READ, WRITE, EXECUTE],
            init: false,
          },
        },
      };
    }

    return { queue: queue_created, workers: workers_created };
  };

  /**
   * Removes all jobs associated with this queue
   * @param {string} queue_name the name of the queue
   * @returns {Promise<boolean>}
   */
  clearQueue = async (queue_name: string): Promise<boolean> => {
    this._checkInit();
    try {
      this._verifyAccess(queue_name, QueuePermissions.WRITE);

      // check to make sure queue exists
      this._verifyQueue(queue_name);

      const drain = await this.QUEUES.drain(this.queues[queue_name]);

      // remove all tracked jobs in this instance if queue was drained correctly
      if (drain) {
        for (const job of Object.values(this.jobs)) {
          if (job.queueName === queue_name && job.id) {
            delete this.jobs[job.id];
          }
        }
      }

      return drain;
    } catch (err) {
      handleErrorWithInfo({
        message: "Failed to drain a queue",
        job: "Clear Queue",
        error: err as Error,
      });
      return false;
    }
  };

  /**
   * Remove the queue from this class instance and from the Redis DB
   * @param {string} queue_name the name of the queue
   * @returns {Promise<boolean>}
   */
  deleteQueue = async (queue_name: string): Promise<boolean> => {
    this._checkInit();
    try {
      this._verifyAccess(queue_name, QueuePermissions.WRITE);
      // check to make sure queue exists
      this._verifyQueue(queue_name);
      // delete queue from system
      await this.QUEUES.delete(this.queues[queue_name]);
      // delete queue from directory
    } catch (err) {
      handleErrorWithInfo({
        message: "Failed to delete a queue from the system",
        job: "Delete Queue",
        error: err as Error,
      });
      return false;
    }
    try {
      delete this.queues[queue_name];
    } catch (err) {
      handleErrorWithInfo({
        message: "Failed to delete a queue from the directory",
        job: "Delete Queue",
        error: err as Error,
      });
      return false;
    }
    try {
      // close all workers
      await Promise.all(
        this.workers[queue_name].map((worker) => worker.close()),
      );
      // remove workers from directory
      delete this.workers[queue_name];
    } catch (err) {
      handleErrorWithInfo({
        message: "Failed to close deleted queue workers from the system",
        job: "Delete Queue",
        error: err as Error,
      });
      // remove directory entry anyway
      delete this.workers[queue_name];
      return false;
    }
    return true;
  };

  /**
   * Create a new worker in this instance
   * @param {string} queue_name the queue to assign this worker to
   * @param {JobExecuter} executer the function to execute for this worker's jobs
   * @param {WorkerOptions} options the config options to configure the worker
   * @returns {Worker | void}
   */
  addWorker = (
    queue_name: string,
    executer: JobExecuter,
    options: WorkerOptions,
  ): Worker | void => {
    this._checkInit();
    try {
      this._verifyAccess(queue_name, QueuePermissions.EXECUTE);
      // check to make sure queue exists
      this._verifyQueue(queue_name);

      // create the worker
      const worker = this.WORKERS.addWorker(queue_name, executer, options);

      // store the worker in the directory
      if (!this.workers[queue_name]) {
        this.workers[queue_name] = [worker];
      } else {
        this.workers[queue_name].push(worker);
      }

      return worker;
    } catch (err) {
      handleErrorWithInfo({
        message: `Failed to create a worker for queue: ${queue_name}`,
        job: "Create Worker",
        error: err as Error,
      });
      return;
    }
  };

  /**
   * Finished current jobs and close the worker from accepting any more jobs
   * @param {Worker} worker the worker instance to close
   * @returns {Promise<void>}
   */
  closeWorker = async (queue_name: string, worker: Worker): Promise<void> => {
    this._checkInit();
    try {
      this._verifyAccess(queue_name, QueuePermissions.EXECUTE);
      // check to make sure queue exists
      this._verifyQueue(queue_name);

      await this.WORKERS.closeWorker(worker);

      // find worker in directory
      const location = this.workers[queue_name].findIndex(
        (i_worker) => worker.id === i_worker.id,
      );

      // remove worker from directory
      this.workers[queue_name].splice(location, 1);

      return;
    } catch (err) {
      handleErrorWithInfo({
        message: `Failed to close worker`,
        job: "Close Worker",
        error: err as Error,
      });
      return;
    }
  };

  /**
   * Finished all jobs in a queue and close all of its workers from accepting new jobs
   * @param {string} queue_name the name of the queue to close all workers in
   * @returns {Promise<void>}
   */
  closeAllWorkersInQueue = async (queue_name: string): Promise<void> => {
    this._checkInit();
    try {
      this._verifyAccess(queue_name, QueuePermissions.EXECUTE);
      // check to make sure queue exists
      this._verifyQueue(queue_name);

      await Promise.all(
        this.workers[queue_name].map((worker) => worker.close()),
      );

      return;
    } catch (err) {
      handleErrorWithInfo({
        message: `Failed to close workers for queue: ${queue_name}`,
        job: "Close All Workers In Queue",
        error: err as Error,
      });
      return;
    }
  };

  /**
   * Add a new job to a queue
   * @param {string} queue_name name of the queue
   * @param {string} job_name the name to assign this job
   * @param {Record<string, unknown>} job_data data payload to attach to this job
   * @param {DefaultJobOptions} job_options job options to config this job
   * @param {number} delay add an optional processing delay to this job in milliseconds
   * @returns {Promise<Job | void>}
   */
  addJob = async (
    queue_name: string,
    job_name: string,
    job_data: Record<string, unknown>,
    job_options: DefaultJobOptions,
    delay?: number,
  ): Promise<Job | void> => {
    this._checkInit();
    try {
      this._verifyAccess(queue_name, QueuePermissions.WRITE);
      // check to make sure queue exists
      this._verifyQueue(queue_name);

      const queue = this.queues[queue_name];

      if (delay) {
        try {
          const delayedJob = await this.JOBS.addDelayedJob(
            queue,
            job_name,
            job_data,
            delay,
            job_options,
          );

          // add the job to the directory
          if (delayedJob.id) {
            this.jobs[delayedJob.id] = delayedJob;
            this.delayed_jobs[delayedJob.id] = delayedJob;
          }

          return delayedJob;
        } catch (err) {
          throw new Error(`Delayed Job Creation Error: ${err}`);
        }
      } else {
        try {
          const job = await this.JOBS.addJob(
            queue,
            job_name,
            job_data,
            job_options,
          );

          // add the job to the directory
          if (job.id) {
            this.jobs[job.id] = job;
          }

          return job;
        } catch (err) {
          throw new Error(`Job Creation Error: ${err}`);
        }
      }
    } catch (err) {
      handleErrorWithInfo({
        message: `Failed to add a new Job`,
        job: "Add Job",
        error: err as Error,
      });
    }
  };

  /**
   * Cancel a job and pervent it from processing.
   * @param {Job} job instance of the job to cancel
   * @returns {Promise<boolean>}
   */
  cancelJob = async (job: Job): Promise<boolean> => {
    this._checkInit();
    try {
      this._verifyAccess(job.queueName, QueuePermissions.WRITE);

      await this.JOBS.cancelJob(job);
      if (job.id) {
        this._removeJob({ jobId: job.id });
      }
      return true;
    } catch (err) {
      handleErrorWithInfo({
        message: `Failed to cancel a job`,
        job: "Cancel Job",
        error: err as Error,
      });
      return false;
    }
  };

  /**
   * Cancel all jobs being tracked in this instance in a given queue from executing
   * @param {string} queue_name name of the queue
   * @returns {Promise<number>}
   */
  cancelAllJobsInQueue = async (queue_name: string): Promise<number> => {
    this._checkInit();
    try {
      this._verifyAccess(queue_name, QueuePermissions.WRITE);

      // check to make sure queue exists
      this._verifyQueue(queue_name);

      // find all jobs
      const jobs = Object.values(this.jobs).filter((job) => {
        return job.queueName === queue_name;
      });

      await Promise.all(jobs.map((job) => this.cancelJob(job)));

      return jobs.length;
    } catch (err) {
      handleErrorWithInfo({
        message: `Failed to cancel all jobs in queue: ${queue_name}`,
        job: "Cancel All Jobs In Queue",
        error: err as Error,
      });
      return -1;
    }
  };
}