virtual-worker-pool/WorkerPool.js

Virtual async workers

// # Worker Pool manager
// Reponsible for pushing task into a queue
// Also responsible for pushing boiler data into queues

var util = require('util');
var EventEmitter = require('events').EventEmitter;
var os = require('os');
var when = require('when');



var WorkerPool = function(config, WorkerProvider, providers) {

  EventEmitter.call(this, {});

  this.workers = [];
  this.maxWorkers = config.maxWorkers || 10;
  this.taskQueueName = config.taskQueueName || 'WorkerPool:TaskQueue';
  this.targetQueueName = config.targetQueueName || 'WorkerTargets';
  this.workerRegistryQueue = config.workerRegistryQueue || 'WorkerTask:inProgressTask';
  this.redisClient = providers.redisClient;
  this.WorkerProvider = WorkerProvider;
  this.workersCreated = false;
  this.workerCounter = 0;
  this.providers = providers;
  this.taskQueue = [];
  this.inProgressTask = {};
};

util.inherits(WorkerPool, EventEmitter);

WorkerPool.prototype.getWorkerId = function() {
  return 'WRK:' + os.hostname() + (this.workerCounter);
};

WorkerPool.prototype.getTaskForWorker = function() {
  var self = this;
  var newTask = this.taskQueue.pop();
  if (this.inProgressTask['Task_' + newTask]) {
    //this task in progress
  } else {
    return newTask;
  }
};


WorkerPool.prototype.createWorkers = function() {
  for (var i = 0; i < this.maxWorkers; i++) {
    this.workerCounter++;
    var newWorker = new this.WorkerProvider(this.getWorkerId(), this, this.providers);
    this.workers.push(newWorker);

  }
  this.workersCreated = true;
};


WorkerPool.prototype.startAllWorkers = function() {
  this.workers.forEach(function(worker) {
    //console.log('worker :', worker);
    worker.beforeStart();
    worker.cycle();
  });
};

WorkerPool.prototype.stopAllWorkers = function() {
  var self = this;
  self.workers.forEach(function(worker) {
    worker.stop();
    worker.afterStop();
  });
};

WorkerPool.prototype.checkIfTaskTargetHasWorker = function(targetId) {
  var self = this;
  // check Redis to see if Worker is currently on a task
  return when.promise(function(resolve, reject) {

    self.redisClient.get('WorkerTask:inProgressTask:' + targetId, function(inProgressError, taskResult) {
      if (inProgressError) {
        reject(inProgressError);
      } else {
        if (taskResult) {
          // -- found a Worker working on this task
          resolve(true, taskResult);
        } else {
          // -- no Worker found working on this task
          resolve(false, {});
        }
      }
    });

  });
};

WorkerPool.prototype.addTaskToQueueForTarget = function(targetId) {
  this.taskQueue = this.taskQueue || [];

  if (this.taskQueue.some(function(task) {
      return task === targetId;
    })) {
    // this target already scheduled so do nothing
    return false;
  } else {

    this.taskQueue.push(targetId);
  }
};


// These left here since because they implement a redis queue. Some details to
// work out however.
// WorkerPool.prototype.getTaskForWorker = function() {
//   var self = this;
//   return when.promise(function(resolve, reject) {
//     // lets return a task to this worker
//     self.redisClient.lpop(self.taskQueueName, function(getNewTaskError, newTaskForWorker) {
//       if (getNewTaskError) {
//         reject(getNewTaskError);
//       } else {
//         // see if this task already has a worker assigned to it
//         self.checkIfTaskTargetHasWorker(newTaskForWorker)
//           .then(function(answer) {
//             if (!answer) {
//               resolve(newTaskForWorker);
//             }
//           })
//           .catch(function(taskhasWorkerError) {
//             reject(taskhasWorkerError);
//           });
//       }
//     });
//   });
// };
// WorkerPool.prototype.addTaskToQueueForTarget = function(targetId) {
//   var self = this;
//
//   // Workers will be responsible for removing task when they "POP" them
//   // off the Task Queue
//   console.log('WorkerPool : start the process of adding a task to queue');
//   self.checkIfTaskTargetHasWorker(targetId)
//     .then(function doesTaskHaveWorkerTarget(answer, worker) {
//       if (!answer) {
//         console.log('WorkerPool : task target doesnt have worker');
//
//         // TODO : Assign a worker?
//
//         // -- no worker on this target already so add to queue
//         self.redisClient.lpush(self.taskQueueName, targetId,
//           function addTaskToQueueHandler(addTaskToQueueError, addTaskResponse) {
//             if (addTaskToQueueError) {
//               self.errorJustHappened(addTaskToQueueError);
//             }
//           });
//       } else {
//         console.log('WorkerPool : task target does have worker');
//         // TODO: if a worker finishes but it hasn't removed
//         //       itself from the targetId, then its possible
//         //       that this task may need to be added, but not
//         //       added.
//       }
//     })
//     .catch(function errorCheckingTaskHasTarget(error) {
//       console.log('WorkerPool : error just happened ', error);
//       self.errorJustHappened(error);
//     });
// };

WorkerPool.prototype.errorJustHappened = function(errorThatJustHappened) {
  this.emit('error', errorThatJustHappened);
};

module.exports = WorkerPool;