nyx_schedule/lib/NyxSchedule.js

nyx任务调度

/* global process */

var Job = require('./Job.js'),
    utils = require('util'),
    Emitter = require('events').EventEmitter;
var db = require("./db")
var Promise = require("bluebird");
var http = require("http");
var url = require('url');

module.exports = NyxSchedule;

function NyxSchedule(config) {

    config = config ? config : {};
    this._name = config.name;
    this._processEvery = 5000; //5秒
    this._defaultConcurrency = config.defaultConcurrency || 1;  //job并发执行数
    this._maxConcurrency = config.maxConcurrency || 300;         //job的最大并发数
    this._definitions = {};
    this._runningJobs = [];
    this._lockedJobs = [];
    this._jobQueue = [];
    this._defaultLockLifetime = config.defaultLockLifetime || 10 * 60 * 1000; //10 minute default lockLifetime

    this._isLockingOnTheFly = false;
    this._jobsToLock = [];

};

utils.inherits(NyxSchedule, Emitter);    // Job uses emit() to fire job events client can use.


NyxSchedule.prototype.name = function (name) {
    this._name = name;
    return this;
};

NyxSchedule.prototype.processEvery = function (time) {
    this._processEvery = time;
    return this;
};

NyxSchedule.prototype.maxConcurrency = function (num) {
    this._maxConcurrency = num;
    return this;
};

NyxSchedule.prototype.defaultConcurrency = function (num) {
    this._defaultConcurrency = num;
    return this;
};


NyxSchedule.prototype.defaultLockLifetime = function (ms) {
    this._defaultLockLifetime = ms;
    return this;
};

/** 
 * 查询任务
 * @param query 查询语句
 */
NyxSchedule.prototype.jobs = function (pageInfo) {
    var self = this;
    return db.findJobs(pageInfo).map(function(result){
                return createJob(self , result);
           });
};


NyxSchedule.prototype.job = function (name) {
    var self = this;
    return db.findJob(name).map(function(result){
                return createJob(self , result);
           });
};


NyxSchedule.prototype.initDefinitions = function () {
    var _self = this;
    return this.jobs().map(function(job){
        _self._definitions[job.attrs.name] = {
            concurrency:  job.attrs.concurrency || 1,
            priority:     job.attrs.priority || 0,
            lockLifetime: job.attrs.lockLifetime || _self._defaultLockLifetime,
            running: 0,
            locked: 0
        };
        console.log("initDefinitions" , _self._definitions)
    }).then(function(){
        _self.emit('ready');
    })
};

/**
 * 定义一个周期执行的job
 * @param start  开始开始时间
 * @param end    结束时间
 * @param interval 执行间隔(秒)
 * @param name job名称
 */
NyxSchedule.prototype.define = function (attrs , options) {
    options = options || {}
    var self = this;
    var job = new Job(this , attrs);
    this._definitions[attrs.name] = {
         concurrency: options.concurrency || 1,
         lockLimit: options.lockLimit || 1,
         priority: options.priority || 0,
         lockLifetime: options.lockLifetime || this._defaultLockLifetime,
         running: 0,
         locked: 0
    };
    job.computeNextRunAt();
    return job.save().then(function(){
        processJobs.bind(self)(job);
    })
};


/**
 * 保存job
 * @param job
 */
NyxSchedule.prototype.saveJob = function (job) {
    var jobData = jobattrsTojobData(job);
    return db.saveOrUpdate(jobData).then(function(result){
         if(result.type == "insert"){
            job.attrs.id = result.id; 
         }
    }).catch(function(err){
        console.log(err);
    });
};


NyxSchedule.prototype.findJob = function(name){
    var _self = this;
    return db.findJob(name).then(function (jobData) {
         if(!jobData){
             throw new Error("没有指定的Job")
         }
         return createJob(_self , jobData);
    })
            
}


/**
 * 启动任务调度
 */
NyxSchedule.prototype.start = function () {
    if (!this._processInterval) {
        this._processInterval = setInterval(processJobs.bind(this), this._processEvery);
        process.nextTick(processJobs.bind(this));
    }
};

/**
 * 停止任务调度
 */
NyxSchedule.prototype.stop = function (cb) {
    cb = cb || function () { };
    clearInterval(this._processInterval);
    this._processInterval = undefined;
    this._unlockJobs(cb);
};


/**
 * 解锁一个任务
 */
NyxSchedule.prototype._unlockJobs = function () {
    var jobIds = this._lockedJobs.map(function (job) {
        return job.attrs.id;
    });
    return db.unlock(jobIds);
}

/**
 * 从数据库中获取指定的job配置，并且修改数据空中lockAt字段，防止其他进程并行执行
 * @param jobName
 * @param lockLifetime 
 * @param cb 查询完后的回调时间 
 */
NyxSchedule.prototype._findAndLockNextJob = function (jobName, lockLifetime) {
    var self = this;
     return db.findAndLockNextJob(jobName , lockLifetime , this).then(function(jobData){
          if(!jobData){
              return null;
          }
          return createJob(self , jobData);
     })
};

function jobDataToJobAttrs(jobData){
    return {
                 name : jobData.name,
                 start : jobData.start_time,
                 end   : jobData.end_time,
                 id : jobData.id,
                 lockAt : jobData.locked_time,
                 nextRunAt : jobData.next_run_time,
                 lastRunAt : jobData.last_run_time,
                 serviceUrl : jobData.server_url,
                 project : jobData.project,
                 disabled : jobData.disabled,
                 priority : jobData.priority,
                 lastFinishedAt : jobData.last_finished_time,
                 failedAt : jobData.failed_time,
                 failedCount : jobData.fail_count,
                 failReason : jobData.fail_reason,
                 interval : jobData.run_interval
           }
}

function jobattrsTojobData(job){
    return {
             name : job.attrs.name,
             start_time : job.attrs.start,
             end_time : job.attrs.end,
             id : job.attrs.id,
             locked_time : job.attrs.lockAt,
             next_run_time : job.attrs.nextRunAt,
             last_run_time : job.attrs.lastRunAt,
             server_url : job.attrs.serviceUrl,
             project : job.attrs.project,
             disabled : job.attrs.disabled,
             priority : job.attrs.priority,
             last_finished_time : job.attrs.lastFinishedAt,
             failed_time : job.attrs.failedAt,
             fail_count : job.attrs.failedCount,
             fail_reason : job.attrs.failReason,
             run_interval : job.attrs.interval
           }
}

/**
 * 创建job
 * @param schedule 当前调度器
 * @param jobData 
 */
function createJob(schedule, jobData) {
    var attrs = jobDataToJobAttrs(jobData); 
    return new Job(schedule , attrs);
}


NyxSchedule.prototype.execJob = function (job) {
    var self = this;
    console.log("execJob", job.attrs.name)
    var serviceUrl = job.attrs.serviceUrl;
    return sendHttp(serviceUrl).then(function () {
                self.emit('success', job);
                self.emit('success:' + job.attrs.name, job);
                self.emit('complete', job);
                self.emit('complete:' + job.attrs.name, job);
           }).catch(function (err) {
                self.emit('fail', err, job);
                self.emit('fail:' + job.attrs.name, err, job);
                self.emit('complete', job);
                self.emit('complete:' + job.attrs.name, job);
                throw err;
           })
}


function sendHttp(httpurl) {
    try {
        var urlobj = url.parse(httpurl)
        var options = {
            hostname: urlobj.hostname,
            port: urlobj.port,
            path: urlobj.path,
            method: 'post'
        };
        return new Promise(function (resolve, reject) {
            var _req = http.request(options, (_res) => {
                _res.setEncoding('utf8');
                var ret = "";
                _res.on('data', (chunk) => {
                    ret += chunk;
                });
                _res.on('end', () => {
                    var s = JSON.parse(ret);
                    var success = s.success;
                    if (success) {
                        resolve();
                    } else {
                        reject(new Error(s.message));
                    }
                });
            });
            _req.end();
            _req.on("error", function (err) {
                reject(err);
            })
        })
    } catch (err) {
        return Promise.reject(err);
    }
}

/**
 * 处理job
 * @param extraJob  额外指定的job
 */
function processJobs(extraJob) {
    
    if (!this._processInterval) {
        return;
    }
    
    var definitions = this._definitions,
        jobName,
        jobQueue = this._jobQueue,
        self = this;

    if (!extraJob) {  //如果没有指定的job则执行所有的job
        for (jobName in definitions) {
            jobQueueFilling(jobName);
        }
    }else{
        jobQueueFilling(extraJob.attrs.name);
    }

    /**
     * 查询数据库中的任务，如果有效则加入队列
     * @param name job名称
     */
    function jobQueueFilling(name) {
        
        var now = new Date();
        self._nextScanAt = new Date(now.valueOf() + self._processEvery); //计算下次扫描时间
        //查找并枷锁任务
        self._findAndLockNextJob(name, definitions[name].lockLifetime).then(function(job){
            if (job) {
                enqueueJobs(job);      //排序进入执行队列
                jobProcessing();       //处理任务
            }
        }).catch(function(err){
           console.log("_findAndLockNextJob err" , err);
        });
    }
    //按优先级，排序任务。
    function enqueueJobs(jobs) {
        if (!Array.isArray(jobs)) {
            jobs = [jobs];
        }

        jobs.forEach(function (job) {
            var job_index = 0;
            jobQueue.forEach(function (queued_job) {
                if (queued_job.attrs.priority < job.attrs.priority) {
                    job_index++;
                }
            });
            //将指定的job插入到指定的队列位置
            jobQueue.splice(job_index, 0, job);
        });
    }


    function jobProcessing() {
        if (!jobQueue.length) {
            return;
        }
        var now = new Date();
        var job = jobQueue.pop(),
            name = job.attrs.name,
            jobDefinition = definitions[name];

        if (job.attrs.nextRunAt < now) {  //下次执行时间小于当前时间则直接执行
            runOrRetry();
        } else {
            setTimeout(runOrRetry, job.attrs.nextRunAt - now); //延时到指定时间执行
        }

        function runOrRetry() {
            if (self._processInterval) {
                if (jobDefinition.concurrency > jobDefinition.running && //单个任务执行的总数不能大于单个任务定义的并发数
                    self._runningJobs.length < self._maxConcurrency) { //进程运行的job数小于进程最大并发执行job数
                    self._runningJobs.push(job);
                    jobDefinition.running++;
                    job.run(processJobResult); //执行任务
                    jobProcessing();
                } else {
                    jobQueue.push(job);
                }
            }
        }
    }

    function processJobResult(err, job) {
        var name = job.attrs.name;
        self._runningJobs.splice(self._runningJobs.indexOf(job), 1);
        definitions[name].running--;
        jobProcessing();
    }
}