manticore/dist/pool.js

Mythical multi-process worker pool

'use strict';
var __extends = this.__extends || function (d, b) {
    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
    function __() { this.constructor = d; }
    __.prototype = b.prototype;
    d.prototype = new __();
};
var os = require('os');
var path = require('path');
var events = require('events');
var Promise = require('bluebird');
var lib = require('./lib');
var _worker = require('./worker');
var Worker = _worker.Worker;
var Job = _worker.Job;
function createPool(options) {
    return new Pool(options);
}
exports.createPool = createPool;
var Pool = (function (_super) {
    __extends(Pool, _super);
    function Pool(options) {
        _super.call(this);
        this.queuedJobs = [];
        this.workers = Object.create(null);
        this.checksNext = false;
        this.options = options;
        this.options.worker = path.resolve(this.options.worker);
        this.options.concurrent = lib.optValue(this.options.concurrent, os.cpus().length);
        this.options.paralel = lib.optValue(this.options.paralel, 1);
        this.options.attempts = lib.optValue(this.options.attempts, 3);
        this.options.idleTimeout = lib.optValue(this.options.idleTimeout, 500);
        this.options.log = lib.optValue(this.options.log, false);
        this.options.emit = lib.optValue(this.options.emit, false || this.options.log);
        if (this.options.log) {
            this.on(lib.STATUS, function (msg) {
                console.log(msg);
            });
        }
    }
    Pool.prototype.run = function (task, params) {
        var _this = this;
        return new Promise(function (resolve, reject) {
            var job = new Job(task, params, function (err, res) {
                if (err) {
                    reject(err);
                } else {
                    resolve(res);
                }
            });
            _this.queuedJobs.push(job);
            _this.status(job, 'added');
            _this.checkQueue();
        });
    };
    Pool.prototype.curried = function (task) {
        var _this = this;
        return function (params) {
            return _this.run(task, params);
        };
    };
    Pool.prototype.status = function () {
        var message = [];
        for (var _i = 0; _i < (arguments.length - 0); _i++) {
            message[_i] = arguments[_i + 0];
        }
        if (this.options.emit) {
            this.emit(lib.STATUS, message.join('; '));
        }
    };
    Pool.prototype.checkQueue = function () {
        var _this = this;
        // debounce
        if (!this.checksNext) {
            this.checksNext = true;
            process.nextTick(function () {
                _this.checksNext = false;
                _this.procQueue();
            });
        }
    };
    Pool.prototype.procQueue = function () {
        while (this.queuedJobs.length > 0) {
            var best = null;
            var count = 0;
            for (var id in this.workers) {
                var worker = this.workers[id];
                count++;
                if (worker.activeCount < this.options.paralel) {
                    if (!best) {
                        best = worker;
                    } else if (worker.activeCount < best.activeCount) {
                        best = worker;
                    }
                }
            }
            if (count < this.options.concurrent && (!best || best.activeCount > 0)) {
                best = this.spawnWorker();
            }
            if (best) {
                best.run(this.queuedJobs.shift());
            } else {
                break;
            }
        }
    };
    Pool.prototype.removeWorker = function (worker) {
        delete this.workers[worker.id];
    };
    Pool.prototype.spawnWorker = function () {
        var _this = this;
        var worker = new Worker(this.options);
        this.workers[worker.id] = worker;
        worker.on(lib.ERROR, function (err) {
            _this.status(worker, 'pool error', err);
            _this.removeWorker(worker);
            worker.kill();
        });
        worker.on(lib.TASK_RESULT, function (job) {
            _this.checkQueue();
        });
        worker.on(lib.TASK_ABORT, function (job) {
            if (job.attempts < _this.options.attempts) {
                job.attempts++;
                _this.queuedJobs.push(job);
                _this.status(worker, 'requeue', job, job.attempts + ' of ' + _this.options.attempts);
            } else {
                _this.status(worker, 'abort', job);
                job.callback(new Error('job failed ' + job.attempts + ' attempts'), null);
            }
            _this.checkQueue();
        });
        worker.on(lib.WORKER_DOWN, function () {
            _this.status(worker, 'down');
            _this.removeWorker(worker);
            _this.checkQueue();
        });
        worker.on(lib.STATUS, function (msg) {
            _this.emit(lib.STATUS, msg);
        });
        this.status(worker, 'spawn <' + this.workerCount + '/' + this.options.concurrent + '>');
        return worker;
    };
    Object.defineProperty(Pool.prototype, "workerCount", {
        get: function () {
            var num = 0;
            for (var id in this.workers) {
                num++;
            }
            return num;
        },
        enumerable: true,
        configurable: true
    });
    return Pool;
})(events.EventEmitter);