cliche-scheduler/lib/scheduler.js

Job queue and scheduler for cliche.js

const fs = require('fs');
const kue = require('kue');
const createLogger = require('cliche-logger');
const path = require('path');
const Promise = require('bluebird');
const s = require('string');

const ID = Symbol('id');
const QUEUE = Symbol('queue');
const JOBS = Symbol('jobs');
const NAMESPACE = Symbol('namespace');
const LOGGER = Symbol('logger');

let queueId = 1;

class Scheduler {
  constructor(ns = 'DEFAULT') {
    this[ID] = [process.pid, queueId++].join(':');

    const namespace = ns.trim();
    const logger = createLogger(`cliche-scheduler:${this[ID]}:${namespace}`);
    const queue = kue.createQueue({ prefix: 'cliche' });

    logger.debug(`Initializing cliche-scheduler ${this[ID]} with namespace '${namespace}'`);

    queue.on('job enqueue', id => {
      kue.Job.get(id, (err, job) => {
        if (err) { return; }

        if (job.data.__QUEUE__ === this[ID]) {
          logger.debug(`Queued job '${job.type}:${job.id}'`);
        }
      });
    });

    queue.on('job complete', (id, result) => {
      kue.Job.get(id, (err, job) => {
        if (err) { return; }

        if (job.data.__QUEUE__ === this[ID]) {
          logger.debug(`Job '${job.type}:${job.id}' completed.`);
          logger.debug(`Result: ${result}`);

          // job.remove(() => {
          //   logger.debug(`Job '${job.type}:${job.id}' removed.`);
          // });
        }
      });
    });

    queue.on('job failed', (id, error) => {
      kue.Job.get(id, (err, job) => {
        if (err) { return; }

        if (job.data.__QUEUE__ === this[ID]) {
          logger.debug(`Job '${job.type}:${job.id}' failed.`);
          logger.debug(`Error message: ${error}`);
        }
      });
    });

    this[NAMESPACE] = namespace;
    this[LOGGER] = logger;
    this[QUEUE] = queue;
    this[JOBS] = new Set();
  }

  loadJobs(dir, ns) {
    const logger = this[LOGGER];
    const directory = path.resolve(dir);
    const namespace = s(ns || this[NAMESPACE]).camelize().s;

    logger.debug(`Loading jobs recursively from ${directory}`);

    try {
      fs.statSync(directory);
    } catch (ex) {
      if (ex && ex.code === 'ENOENT') {
        logger.debug(`Directory ${directory} does not exist. Skipping.`);
        logger.info('No jobs. Exiting.');
        return this;
      }

      throw ex;
    }

    fs.readdirSync(directory)
      .filter(f => !f.startsWith('.') && !f.startsWith('_'))
      .forEach(file => {
        const filepath = path.resolve(directory, file);
        const prefix = namespace ? `${namespace}:` : '';

        if (fs.statSync(filepath).isDirectory()) {
          this.loadJobs(filepath, prefix + path.relative(directory, filepath));
        } else {
          const job = require(filepath);
          const name = prefix + s(job.name || file).chompRight('.js').camelize().s;
          const concurency = job.concurency || 1;

          this[QUEUE].process(name, concurency, job);
          this[JOBS].add(name);

          logger.debug(`Loaded job '${name}'`);
        }
      });

    return this;
  }

  schedule(name, data = {}, delay) {
    const jobName = `${this[NAMESPACE]}:${name}`;
    const jobData = Object.assign({ __QUEUE__: this[ID] }, data);
    const job = this[QUEUE].create(jobName, jobData);

    job.set('queueId', this[ID]);

    if (delay) {
      job.delay(delay);
    }

    return Promise.fromCallback(job.save.bind(job)).return(job);
  }
}

module.exports = Scheduler;