vulpes
Version:
Job management framework
357 lines (293 loc) • 17.7 kB
JavaScript
;
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
var _get = function get(object, property, receiver) { if (object === null) object = Function.prototype; var desc = Object.getOwnPropertyDescriptor(object, property); if (desc === undefined) { var parent = Object.getPrototypeOf(object); if (parent === null) { return undefined; } else { return get(parent, property, receiver); } } else if ("value" in desc) { return desc.value; } else { var getter = desc.get; if (getter === undefined) { return undefined; } return getter.call(receiver); } };
function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; }
function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; }
var _require = require("stream"),
Readable = _require.Readable;
var Redis = require("ioredis");
var endOfStream = require("end-of-stream");
var sleep = require("sleep-promise");
var ChannelQueue = require("@buttercup/channel-queue");
var Storage = require("./Storage.js");
var NOOP = function NOOP() {};
var SHUTDOWN_GRACE_PRE = 100;
var SHUTDOWN_GRACE_POST = 150;
var STREAM_READ_COUNT = 2;
/**
* Redis storage adapter
* Stores items in a Redis database
* @augments Storage
* @memberof module:Vulpes
*/
var RedisStorage = function (_Storage) {
_inherits(RedisStorage, _Storage);
/**
* Create a new Redis storage instance
* @see https://www.npmjs.com/package/ioredis
* @param {Object=} redisOptions The options for ioredis
* @memberof RedisStorage
*/
function RedisStorage() {
var redisOptions = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
_classCallCheck(this, RedisStorage);
var _this = _possibleConstructorReturn(this, (RedisStorage.__proto__ || Object.getPrototypeOf(RedisStorage)).call(this));
_this.redis = new Redis(redisOptions);
_this._queue = new ChannelQueue();
return _this;
}
/**
* Get an item by its ID
* @returns {Promise.<Object|null>} The found item or null if not found
* @memberof RedisStorage
*/
_createClass(RedisStorage, [{
key: "getItem",
value: function () {
var _ref = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee(id) {
var json;
return regeneratorRuntime.wrap(function _callee$(_context) {
while (1) {
switch (_context.prev = _context.next) {
case 0:
_context.next = 2;
return this.redis.get(id);
case 2:
json = _context.sent;
return _context.abrupt("return", json ? JSON.parse(json) : null);
case 4:
case "end":
return _context.stop();
}
}
}, _callee, this);
}));
function getItem(_x2) {
return _ref.apply(this, arguments);
}
return getItem;
}()
/**
* Remove an item by its ID
* @returns {Promise}
* @memberof RedisStorage
*/
}, {
key: "removeItem",
value: function () {
var _ref2 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee2(id) {
return regeneratorRuntime.wrap(function _callee2$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
_context2.next = 2;
return this.redis.del(id);
case 2:
case "end":
return _context2.stop();
}
}
}, _callee2, this);
}));
function removeItem(_x3) {
return _ref2.apply(this, arguments);
}
return removeItem;
}()
/**
* Set an item for an ID
* @param {String} id The ID to set
* @param {Object} item The item to set
* @returns {Promise}
* @memberof RedisStorage
*/
}, {
key: "setItem",
value: function () {
var _ref3 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee3(id, item) {
return regeneratorRuntime.wrap(function _callee3$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
_context3.next = 2;
return this.redis.set(id, JSON.stringify(item));
case 2:
case "end":
return _context3.stop();
}
}
}, _callee3, this);
}));
function setItem(_x4, _x5) {
return _ref3.apply(this, arguments);
}
return setItem;
}()
/**
* Shutdown the adapter (and disconnect Redis)
* @memberof RedisStorage
* @returns {Promise} A promise that resolves when shutdown has completed
*/
}, {
key: "shutdown",
value: function () {
var _ref4 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee5() {
var _this2 = this;
return regeneratorRuntime.wrap(function _callee5$(_context5) {
while (1) {
switch (_context5.prev = _context5.next) {
case 0:
_context5.next = 2;
return _get(RedisStorage.prototype.__proto__ || Object.getPrototypeOf(RedisStorage.prototype), "shutdown", this).call(this);
case 2:
_context5.next = 4;
return sleep(SHUTDOWN_GRACE_PRE);
case 4:
_context5.next = 6;
return this._queue.channel("streamOut").enqueue(_asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee4() {
return regeneratorRuntime.wrap(function _callee4$(_context4) {
while (1) {
switch (_context4.prev = _context4.next) {
case 0:
_context4.next = 2;
return sleep(SHUTDOWN_GRACE_POST);
case 2:
_context4.next = 4;
return _this2.redis.quit();
case 4:
case "end":
return _context4.stop();
}
}
}, _callee4, _this2);
})));
case 6:
case "end":
return _context5.stop();
}
}
}, _callee5, this);
}));
function shutdown() {
return _ref4.apply(this, arguments);
}
return shutdown;
}()
/**
* Stream all items
* @returns {Promise.<ReadableStream>} A readable stream
* @memberof RedisStorage
*/
}, {
key: "streamItems",
value: function () {
var _ref6 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee8() {
var _this3 = this;
var outStream, inStream, usedKeys;
return regeneratorRuntime.wrap(function _callee8$(_context8) {
while (1) {
switch (_context8.prev = _context8.next) {
case 0:
// Create a readable stream for the consumer to read
// job items from
outStream = new Readable({ objectMode: true });
outStream._read = NOOP;
// Create a stream for the reading from Redis
inStream = this.redis.scanStream({
match: "*",
count: STREAM_READ_COUNT
});
usedKeys = [];
inStream.on("data", function (keys) {
return (
// We queue on portions of the stream to ensure that uniqueness
// is maintained while reading asynchronously
_this3._queue.channel("streamOut").enqueue(_asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee7() {
var uniqueKeys;
return regeneratorRuntime.wrap(function _callee7$(_context7) {
while (1) {
switch (_context7.prev = _context7.next) {
case 0:
// Pause the redis stream so we can process
inStream.pause();
// Track unique keys already used
uniqueKeys = keys.filter(function (key) {
return !usedKeys.includes(key);
});
usedKeys.push.apply(usedKeys, _toConsumableArray(uniqueKeys));
if (!(uniqueKeys.length > 0)) {
_context7.next = 8;
break;
}
_context7.next = 6;
return Promise.all(
// Push all new unique items into the out stream
uniqueKeys.map(function () {
var _ref8 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee6(itemKey) {
var json;
return regeneratorRuntime.wrap(function _callee6$(_context6) {
while (1) {
switch (_context6.prev = _context6.next) {
case 0:
_context6.next = 2;
return _this3.redis.get(itemKey);
case 2:
json = _context6.sent;
if (json) {
outStream.push(JSON.parse(json));
}
case 4:
case "end":
return _context6.stop();
}
}
}, _callee6, _this3);
}));
return function (_x6) {
return _ref8.apply(this, arguments);
};
}()));
case 6:
_context7.next = 10;
break;
case 8:
_context7.next = 10;
return sleep(50);
case 10:
// Continue processing
inStream.resume();
case 11:
case "end":
return _context7.stop();
}
}
}, _callee7, _this3);
})))
);
});
endOfStream(inStream, function () {
_this3._queue.channel("streamOut").enqueue(function () {
// Mark the stream as finished
outStream.push(null);
});
});
return _context8.abrupt("return", Promise.resolve(outStream));
case 7:
case "end":
return _context8.stop();
}
}
}, _callee8, this);
}));
function streamItems() {
return _ref6.apply(this, arguments);
}
return streamItems;
}()
}]);
return RedisStorage;
}(Storage);
module.exports = RedisStorage;