bottleneck
Version:
Distributed task scheduler and rate limiter
519 lines (441 loc) • 18.2 kB
JavaScript
"use strict";
var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();
function _toArray(arr) { return Array.isArray(arr) ? arr : Array.from(arr); }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
// Generated by CoffeeScript 2.2.4
(function () {
var Bottleneck,
DEFAULT_PRIORITY,
DLList,
Events,
Local,
NUM_PRIORITIES,
RedisStorage,
States,
Sync,
packagejson,
parser,
splice = [].splice;
NUM_PRIORITIES = 10;
DEFAULT_PRIORITY = 5;
parser = require("./parser");
Local = require("./Local");
RedisStorage = require("./RedisStorage");
Events = require("./Events");
States = require("./States");
DLList = require("./DLList");
Sync = require("./Sync");
packagejson = require("../package.json");
Bottleneck = function () {
class Bottleneck {
constructor(options = {}, ...invalid) {
var sDefaults;
this.ready = this.ready.bind(this);
this.clients = this.clients.bind(this);
this.disconnect = this.disconnect.bind(this);
this.chain = this.chain.bind(this);
this.queued = this.queued.bind(this);
this.running = this.running.bind(this);
this.check = this.check.bind(this);
this._drainOne = this._drainOne.bind(this);
this.submit = this.submit.bind(this);
this.schedule = this.schedule.bind(this);
this.wrap = this.wrap.bind(this);
this.updateSettings = this.updateSettings.bind(this);
this.currentReservoir = this.currentReservoir.bind(this);
this.incrementReservoir = this.incrementReservoir.bind(this);
if (!(options != null && typeof options === "object" && invalid.length === 0)) {
throw new Bottleneck.prototype.BottleneckError("Bottleneck v2 takes a single object argument. Refer to https://github.com/SGrondin/bottleneck#upgrading-to-v2 if you're upgrading from Bottleneck v1.");
}
parser.load(options, this.instanceDefaults, this);
this._queues = this._makeQueues();
this._scheduled = {};
this._states = new States(["RECEIVED", "QUEUED", "RUNNING", "EXECUTING"].concat(this.trackDoneStatus ? ["DONE"] : []));
this._limiter = null;
this.Events = new Events(this);
this._submitLock = new Sync("submit");
this._registerLock = new Sync("register");
sDefaults = parser.load(options, this.storeDefaults, {});
this._store = function () {
if (this.datastore === "local") {
return new Local(parser.load(options, this.storeInstanceDefaults, sDefaults));
} else if (this.datastore === "redis") {
return new RedisStorage(this, sDefaults, parser.load(options, this.storeInstanceDefaults, {}));
} else {
throw new Bottleneck.prototype.BottleneckError(`Invalid datastore type: ${this.datastore}`);
}
}.call(this);
}
ready() {
return this._store.ready;
}
clients() {
return this._store.clients;
}
disconnect(flush = true) {
var _this = this;
return _asyncToGenerator(function* () {
return yield _this._store.disconnect(flush);
})();
}
chain(_limiter) {
this._limiter = _limiter;
return this;
}
queued(priority) {
if (priority != null) {
return this._queues[priority].length;
} else {
return this._queues.reduce(function (a, b) {
return a + b.length;
}, 0);
}
}
empty() {
return this.queued() === 0 && this._submitLock.isEmpty();
}
running() {
var _this2 = this;
return _asyncToGenerator(function* () {
return yield _this2._store.__running__();
})();
}
jobStatus(id) {
return this._states.jobStatus(id);
}
counts() {
return this._states.statusCounts();
}
_makeQueues() {
var i, j, ref, results;
results = [];
for (i = j = 1, ref = NUM_PRIORITIES; 1 <= ref ? j <= ref : j >= ref; i = 1 <= ref ? ++j : --j) {
results.push(new DLList());
}
return results;
}
_sanitizePriority(priority) {
var sProperty;
sProperty = ~~priority !== priority ? DEFAULT_PRIORITY : priority;
if (sProperty < 0) {
return 0;
} else if (sProperty > NUM_PRIORITIES - 1) {
return NUM_PRIORITIES - 1;
} else {
return sProperty;
}
}
_find(arr, fn) {
var ref;
return (ref = function () {
var i, j, len, x;
for (i = j = 0, len = arr.length; j < len; i = ++j) {
x = arr[i];
if (fn(x)) {
return x;
}
}
}()) != null ? ref : [];
}
_getFirst(arr) {
return this._find(arr, function (x) {
return x.length > 0;
});
}
_randomIndex() {
return Math.random().toString(36).slice(2);
}
check(weight = 1) {
var _this3 = this;
return _asyncToGenerator(function* () {
return yield _this3._store.__check__(weight);
})();
}
_run(next, wait, index) {
var _this4 = this;
var completed, done;
this.Events.trigger("debug", [`Scheduling ${next.options.id}`, {
args: next.args,
options: next.options
}]);
done = false;
completed = (() => {
var _ref = _asyncToGenerator(function* (...args) {
var e, ref, running;
if (!done) {
try {
done = true;
_this4._states.next(next.options.id); // DONE
clearTimeout(_this4._scheduled[index].expiration);
delete _this4._scheduled[index];
_this4.Events.trigger("debug", [`Completed ${next.options.id}`, {
args: next.args,
options: next.options
}]);
var _ref2 = yield _this4._store.__free__(index, next.options.weight);
running = _ref2.running;
_this4.Events.trigger("debug", [`Freed ${next.options.id}`, {
args: next.args,
options: next.options
}]);
_this4._drainAll().catch(function (e) {
return _this4.Events.trigger("error", [e]);
});
if (running === 0 && _this4.empty()) {
_this4.Events.trigger("idle", []);
}
return (ref = next.cb) != null ? ref.apply({}, args) : void 0;
} catch (error) {
e = error;
return _this4.Events.trigger("error", [e]);
}
}
});
return function completed() {
return _ref.apply(this, arguments);
};
})();
this._states.next(next.options.id); // RUNNING
return this._scheduled[index] = {
timeout: setTimeout(() => {
this.Events.trigger("debug", [`Executing ${next.options.id}`, {
args: next.args,
options: next.options
}]);
this._states.next(next.options.id); // EXECUTING
if (this._limiter != null) {
return this._limiter.submit.apply(this._limiter, Array.prototype.concat(next.options, next.task, next.args, completed));
} else {
return next.task.apply({}, next.args.concat(completed));
}
}, wait),
expiration: next.options.expiration != null ? setTimeout(() => {
return completed(new Bottleneck.prototype.BottleneckError(`This job timed out after ${next.options.expiration} ms.`));
}, wait + next.options.expiration) : void 0,
job: next
};
}
_drainOne(freed) {
return this._registerLock.schedule(() => {
var args, index, options, queue;
if (this.queued() === 0) {
return this.Promise.resolve(false);
}
queue = this._getFirst(this._queues);
var _queue$first = queue.first();
options = _queue$first.options;
args = _queue$first.args;
if (freed != null && options.weight > freed) {
return this.Promise.resolve(false);
}
this.Events.trigger("debug", [`Draining ${options.id}`, { args, options }]);
index = this._randomIndex();
return this._store.__register__(index, options.weight, options.expiration).then(({ success, wait, reservoir }) => {
var empty, next;
this.Events.trigger("debug", [`Drained ${options.id}`, { success, args, options }]);
if (success) {
next = queue.shift();
empty = this.empty();
if (empty) {
this.Events.trigger("empty", []);
}
if (reservoir === 0) {
this.Events.trigger("depleted", [empty]);
}
this._run(next, wait, index);
}
return this.Promise.resolve(success);
});
});
}
_drainAll(freed) {
return this._drainOne(freed).then(success => {
if (success) {
return this._drainAll();
} else {
return this.Promise.resolve(success);
}
}).catch(e => {
return this.Events.trigger("error", [e]);
});
}
_drop(job) {
this._states.remove(job.options.id);
if (this.rejectOnDrop) {
job.cb.apply({}, [new Bottleneck.prototype.BottleneckError("This job has been dropped by Bottleneck")]);
}
return this.Events.trigger("dropped", [job]);
}
submit(...args) {
var _this5 = this;
var cb, job, options, ref, ref1, task;
if (typeof args[0] === "function") {
var _ref3, _ref4, _splice$call, _splice$call2;
ref = args, (_ref3 = ref, _ref4 = _toArray(_ref3), task = _ref4[0], args = _ref4.slice(1), _ref3), (_splice$call = splice.call(args, -1), _splice$call2 = _slicedToArray(_splice$call, 1), cb = _splice$call2[0], _splice$call);
options = this.jobDefaults;
} else {
var _ref5, _ref6, _splice$call3, _splice$call4;
ref1 = args, (_ref5 = ref1, _ref6 = _toArray(_ref5), options = _ref6[0], task = _ref6[1], args = _ref6.slice(2), _ref5), (_splice$call3 = splice.call(args, -1), _splice$call4 = _slicedToArray(_splice$call3, 1), cb = _splice$call4[0], _splice$call3);
options = parser.load(options, this.jobDefaults);
}
job = { options, task, args, cb };
options.priority = this._sanitizePriority(options.priority);
if (options.id === this.jobDefaults.id) {
options.id = `${options.id}-${this._randomIndex()}`;
}
if (this.jobStatus(options.id) != null) {
job.cb(new Bottleneck.prototype.BottleneckError(`A job with the same id already exists (id=${options.id})`));
return false;
}
this._states.start(options.id); // RECEIVED
this.Events.trigger("debug", [`Queueing ${options.id}`, { args, options }]);
return this._submitLock.schedule(_asyncToGenerator(function* () {
var blocked, e, reachedHWM, shifted, strategy;
try {
var _ref8 = yield _this5._store.__submit__(_this5.queued(), options.weight);
reachedHWM = _ref8.reachedHWM;
blocked = _ref8.blocked;
strategy = _ref8.strategy;
_this5.Events.trigger("debug", [`Queued ${options.id}`, { args, options, reachedHWM, blocked }]);
} catch (error) {
e = error;
_this5._states.remove(options.id);
_this5.Events.trigger("debug", [`Could not queue ${options.id}`, {
args,
options,
error: e
}]);
job.cb(e);
return false;
}
if (blocked) {
_this5._queues = _this5._makeQueues();
_this5._drop(job);
return true;
} else if (reachedHWM) {
shifted = strategy === Bottleneck.prototype.strategy.LEAK ? _this5._getFirst(_this5._queues.slice(options.priority).reverse()).shift() : strategy === Bottleneck.prototype.strategy.OVERFLOW_PRIORITY ? _this5._getFirst(_this5._queues.slice(options.priority + 1).reverse()).shift() : strategy === Bottleneck.prototype.strategy.OVERFLOW ? job : void 0;
if (shifted != null) {
_this5._drop(shifted);
}
if (shifted == null || strategy === Bottleneck.prototype.strategy.OVERFLOW) {
if (shifted == null) {
_this5._drop(job);
}
return reachedHWM;
}
}
_this5._states.next(job.options.id); // QUEUED
_this5._queues[options.priority].push(job);
yield _this5._drainAll();
return reachedHWM;
}));
}
schedule(...args) {
var options, task, wrapped;
if (typeof args[0] === "function") {
var _args = args;
var _args2 = _toArray(_args);
task = _args2[0];
args = _args2.slice(1);
options = this.jobDefaults;
} else {
var _args3 = args;
var _args4 = _toArray(_args3);
options = _args4[0];
task = _args4[1];
args = _args4.slice(2);
options = parser.load(options, this.jobDefaults);
}
wrapped = function wrapped(...args) {
var _ref9, _ref10, _splice$call5, _splice$call6;
var cb, ref, returned;
ref = args, (_ref9 = ref, _ref10 = _toArray(_ref9), args = _ref10.slice(0), _ref9), (_splice$call5 = splice.call(args, -1), _splice$call6 = _slicedToArray(_splice$call5, 1), cb = _splice$call6[0], _splice$call5);
returned = task.apply({}, args);
return (!((returned != null ? returned.then : void 0) != null && typeof returned.then === "function") ? Promise.resolve(returned) : returned).then(function (...args) {
return cb.apply({}, Array.prototype.concat(null, args));
}).catch(function (...args) {
return cb.apply({}, args);
});
};
return new this.Promise((resolve, reject) => {
return this.submit.apply({}, Array.prototype.concat(options, wrapped, args, function (...args) {
return (args[0] != null ? reject : (args.shift(), resolve)).apply({}, args);
})).catch(e => {
return this.Events.trigger("error", [e]);
});
});
}
wrap(fn) {
return (...args) => {
return this.schedule.apply({}, Array.prototype.concat(fn, args));
};
}
updateSettings(options = {}) {
var _this6 = this;
return _asyncToGenerator(function* () {
yield _this6._store.__updateSettings__(parser.overwrite(options, _this6.storeDefaults));
parser.overwrite(options, _this6.instanceDefaults, _this6);
_this6._drainAll().catch(function (e) {
return _this6.Events.trigger("error", [e]);
});
return _this6;
})();
}
currentReservoir() {
var _this7 = this;
return _asyncToGenerator(function* () {
return yield _this7._store.__currentReservoir__();
})();
}
incrementReservoir(incr = 0) {
var _this8 = this;
return _asyncToGenerator(function* () {
yield _this8._store.__incrementReservoir__(incr);
_this8._drainAll().catch(function (e) {
return _this8.Events.trigger("error", [e]);
});
return _this8;
})();
}
};
Bottleneck.default = Bottleneck;
Bottleneck.version = Bottleneck.prototype.version = packagejson.version;
Bottleneck.strategy = Bottleneck.prototype.strategy = {
LEAK: 1,
OVERFLOW: 2,
OVERFLOW_PRIORITY: 4,
BLOCK: 3
};
Bottleneck.BottleneckError = Bottleneck.prototype.BottleneckError = require("./BottleneckError");
Bottleneck.Group = Bottleneck.prototype.Group = require("./Group");
Bottleneck.prototype.jobDefaults = {
priority: DEFAULT_PRIORITY,
weight: 1,
expiration: null,
id: "<no-id>"
};
Bottleneck.prototype.storeDefaults = {
maxConcurrent: null,
minTime: 0,
highWater: null,
strategy: Bottleneck.prototype.strategy.LEAK,
penalty: null,
reservoir: null
};
Bottleneck.prototype.storeInstanceDefaults = {
clientOptions: {},
clearDatastore: false,
Promise: Promise,
_groupTimeout: null
};
Bottleneck.prototype.instanceDefaults = {
datastore: "local",
id: "<no-id>",
rejectOnDrop: true,
trackDoneStatus: false,
Promise: Promise
};
return Bottleneck;
}.call(this);
module.exports = Bottleneck;
}).call(undefined);