vulpes
Version:
Job management framework
421 lines (357 loc) • 22.1 kB
JavaScript
;
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
var _get = function get(object, property, receiver) { if (object === null) object = Function.prototype; var desc = Object.getOwnPropertyDescriptor(object, property); if (desc === undefined) { var parent = Object.getPrototypeOf(object); if (parent === null) { return undefined; } else { return get(parent, property, receiver); } } else if ("value" in desc) { return desc.value; } else { var getter = desc.get; if (getter === undefined) { return undefined; } return getter.call(receiver); } };
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; }
function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; }
var fs = require("fs");
var debounce = require("debounce");
var ChannelQueue = require("@buttercup/channel-queue");
var pify = require("pify");
var tmp = require("tmp");
var endOfStream = require("end-of-stream");
var pump = require("pump");
var JSONStream = require("JSONStream");
var fileExists = require("file-exists");
var objectStream = require("@kiosked/object-stream");
var Storage = require("./Storage.js");
var JSON_PARSE_ARGS = ["*"];
/**
* File storage adapter
* Stores and streams jobs in a local file (very inefficiently)
* @augments Storage
* @memberof module:Vulpes
*/
var FileStorage = function (_Storage) {
_inherits(FileStorage, _Storage);
/**
* Constructor for a new FileStorage instance
* @param {String} filename The file to store/stream jobs to and from
* @memberof FileStorage
*/
function FileStorage(filename) {
_classCallCheck(this, FileStorage);
var _this = _possibleConstructorReturn(this, (FileStorage.__proto__ || Object.getPrototypeOf(FileStorage)).call(this));
_this._filename = filename;
_this._queue = new ChannelQueue();
_this._unlinkFile = pify(fs.unlink);
_this._writeFile = pify(fs.writeFile);
return _this;
}
/**
* Get an item by its ID
* @param {String} id The item ID
* @returns {Promise.<Object|null>} A promise that resolves with the item or
* null if not found
* @memberof FileStorage
*/
_createClass(FileStorage, [{
key: "getItem",
value: function () {
var _ref = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee(id) {
var stream;
return regeneratorRuntime.wrap(function _callee$(_context) {
while (1) {
switch (_context.prev = _context.next) {
case 0:
_context.next = 2;
return this.streamItems();
case 2:
stream = _context.sent;
return _context.abrupt("return", new Promise(function (resolve, reject) {
var found = false;
stream.on("data", function (item) {
if (item.id === id) {
found = true;
stream.destroy();
resolve(item);
}
});
endOfStream(stream, function (err) {
if (err && !found) {
return reject(err);
}
if (!found) {
resolve(null);
}
});
}));
case 4:
case "end":
return _context.stop();
}
}
}, _callee, this);
}));
function getItem(_x) {
return _ref.apply(this, arguments);
}
return getItem;
}()
/**
* Remove an item by its ID
* @param {String} id The item ID
* @returns {Promise} A promise that resolves when the item's been removed
* @memberof FileStorage
*/
}, {
key: "removeItem",
value: function () {
var _ref2 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee2(id) {
return regeneratorRuntime.wrap(function _callee2$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
_context2.next = 2;
return this.setItem(id, null);
case 2:
case "end":
return _context2.stop();
}
}
}, _callee2, this);
}));
function removeItem(_x2) {
return _ref2.apply(this, arguments);
}
return removeItem;
}()
/**
* Set an item using its ID
* @param {String} id The item ID to set
* @param {Object|null} item The item to set (or null to remove)
* @returns {Promise} A promise that resolves when the operation has been
* completed
* @see setItems
* @memberof FileStorage
*/
}, {
key: "setItem",
value: function () {
var _ref3 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee3(id, item) {
return regeneratorRuntime.wrap(function _callee3$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
_context3.next = 2;
return this.setItems(_defineProperty({}, id, item));
case 2:
case "end":
return _context3.stop();
}
}
}, _callee3, this);
}));
function setItem(_x3, _x4) {
return _ref3.apply(this, arguments);
}
return setItem;
}()
/**
* Set many items by specifying a hash map
* @param {Object.<String, Object|null>} itemsInd The items hash map
* @param {String} method Method of writing. Defaults to "append".
* When set to "clear" all other jobs are wiped during this action.
* @returns {Promise}
* @memberof FileStorage
* @example
* await fileStorage.setItems({
* "abc123": {}, // Set/Create item
* "def456": null // Delete item
* });
*/
}, {
key: "setItems",
value: function () {
var _ref4 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee5(itemsInd) {
var _this2 = this;
var method = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : "append";
return regeneratorRuntime.wrap(function _callee5$(_context5) {
while (1) {
switch (_context5.prev = _context5.next) {
case 0:
_context5.next = 2;
return this._queue.channel("stream").enqueue(_asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee4() {
var originalExists, _ref6, tmpPath, cleanup, rs, ws, newItems;
return regeneratorRuntime.wrap(function _callee4$(_context4) {
while (1) {
switch (_context4.prev = _context4.next) {
case 0:
_context4.next = 2;
return fileExists(_this2._filename);
case 2:
originalExists = _context4.sent;
if (originalExists) {
_context4.next = 6;
break;
}
_context4.next = 6;
return _this2._writeFile(_this2._filename, "{}");
case 6:
_context4.next = 8;
return new Promise(function (resolve, reject) {
return tmp.file(function (err, tmpPath, fd, cleanup) {
if (err) {
return reject(err);
}
resolve({ tmpPath: tmpPath, cleanup: cleanup });
});
});
case 8:
_ref6 = _context4.sent;
tmpPath = _ref6.tmpPath;
cleanup = _ref6.cleanup;
// Create a read stream with JSON parsing
rs = fs.createReadStream(_this2._filename).pipe(JSONStream.parse.apply(JSONStream, JSON_PARSE_ARGS));
ws = JSONStream.stringifyObject();
// Track whether or not the item exists in the stream:
// - If it doesn't, it should be added at the end of the stream
// - If it does, it should be replaced mid-stream
newItems = Object.keys(itemsInd);
rs.on("data", function (currentItem) {
// Item found, replace
if (itemsInd.hasOwnProperty(currentItem.id)) {
var newItemInd = newItems.indexOf(currentItem.id);
if (newItemInd >= 0) {
newItems.splice(newItemInd, 1);
}
if (itemsInd[currentItem.id] === null) {
// Delete
return;
}
ws.write([currentItem.id, itemsInd[currentItem.id]]);
return;
}
// Another item we're not looking for, write it immediately
if (method === "append") {
ws.write([currentItem.id, currentItem]);
}
});
// Wait for the read stream to end
endOfStream(rs, function () {
newItems.forEach(function (newItemID) {
// Item wasn't found, so add it
ws.write([newItemID, itemsInd[newItemID]]);
});
ws.end();
});
// Now pump the new write stream into the temp file (as we can't simply overwrite
// the current live file)
_context4.next = 18;
return new Promise(function (resolve, reject) {
return pump(ws, fs.createWriteStream(tmpPath), function (err) {
if (err) {
return reject(err);
}
resolve();
});
});
case 18:
_context4.next = 20;
return new Promise(function (resolve, reject) {
return pump(fs.createReadStream(tmpPath), fs.createWriteStream(_this2._filename), function (err) {
if (err) {
return reject(err);
}
resolve();
});
});
case 20:
// Cleanup the temp file, without waiting
cleanup();
case 21:
case "end":
return _context4.stop();
}
}
}, _callee4, _this2);
})));
case 2:
case "end":
return _context5.stop();
}
}
}, _callee5, this);
}));
function setItems(_x5) {
return _ref4.apply(this, arguments);
}
return setItems;
}()
/**
* Stream all items
* @returns {Promise.<ReadableStream>} A promise that resolves with a readable stream
* @memberof FileStorage
*/
}, {
key: "streamItems",
value: function () {
var _ref7 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee7() {
var _this3 = this;
var originalExists;
return regeneratorRuntime.wrap(function _callee7$(_context7) {
while (1) {
switch (_context7.prev = _context7.next) {
case 0:
_context7.next = 2;
return fileExists(this._filename);
case 2:
originalExists = _context7.sent;
if (originalExists) {
_context7.next = 5;
break;
}
return _context7.abrupt("return", _get(FileStorage.prototype.__proto__ || Object.getPrototypeOf(FileStorage.prototype), "streamItems", this).call(this));
case 5:
_context7.next = 7;
return new Promise(function () {
var _ref8 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee6(consumerResolve) {
return regeneratorRuntime.wrap(function _callee6$(_context6) {
while (1) {
switch (_context6.prev = _context6.next) {
case 0:
_context6.next = 2;
return _this3._queue.channel("stream").enqueue(function () {
return new Promise(function (channelResolve) {
var stream = fs.createReadStream(_this3._filename).pipe(JSONStream.parse.apply(JSONStream, JSON_PARSE_ARGS));
// Resolve the consumer's promise with the stream instance, while
// we wait for the stream to end next..
consumerResolve(stream);
// We resolve the channel later as we need to wait for the stream
// to close naturally
endOfStream(stream, function () {
return channelResolve();
});
});
});
case 2:
case "end":
return _context6.stop();
}
}
}, _callee6, _this3);
}));
return function (_x7) {
return _ref8.apply(this, arguments);
};
}());
case 7:
return _context7.abrupt("return", _context7.sent);
case 8:
case "end":
return _context7.stop();
}
}
}, _callee7, this);
}));
function streamItems() {
return _ref7.apply(this, arguments);
}
return streamItems;
}()
}]);
return FileStorage;
}(Storage);
module.exports = FileStorage;