vulpes
Version:
Job management framework
711 lines (579 loc) • 30.5 kB
JavaScript
"use strict";
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
var migrate = function () {
var _ref = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee(service, manager) {
var _this = this;
var jobs, _loop, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, job;
return regeneratorRuntime.wrap(function _callee$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
jobs = void 0;
case 1:
_context2.next = 3;
return service.queryJobs({
"result.data": function resultData(data) {
return !!data && !!Object.keys(data).find(function (key) {
return ATTACHMENT_REXP.test(key) && !!data[key].data;
});
}
}, { limit: 20 });
case 3:
jobs = _context2.sent;
_loop = /*#__PURE__*/regeneratorRuntime.mark(function _loop(job) {
var attachments, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, attachment, id, data, mime, writeStream, _parseDataURI, textBuffer, buff, resultData;
return regeneratorRuntime.wrap(function _loop$(_context) {
while (1) {
switch (_context.prev = _context.next) {
case 0:
attachments = Object.keys(job.result.data).filter(function (key) {
return ATTACHMENT_REXP.test(key);
}).map(function (key) {
return Object.assign({}, job.result.data[key], {
key: key,
id: key.replace(/^%attachment:/, "")
});
});
_iteratorNormalCompletion2 = true;
_didIteratorError2 = false;
_iteratorError2 = undefined;
_context.prev = 4;
_iterator2 = attachments[Symbol.iterator]();
case 6:
if (_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done) {
_context.next = 21;
break;
}
attachment = _step2.value;
if (attachment.data) {
_context.next = 10;
break;
}
return _context.abrupt("continue", 18);
case 10:
id = attachment.id, data = attachment.data, mime = attachment.mime;
_context.next = 13;
return manager.getArtifactWriteStream(id);
case 13:
writeStream = _context.sent;
if (/^text\//.test(mime)) {
_parseDataURI = parseDataURI(data), textBuffer = _parseDataURI.data;
writeStream.write(textBuffer.toString("utf8"));
} else {
buff = dataURIToBuffer(data);
writeStream.write(buff);
}
writeStream.end();
_context.next = 18;
return waitForStream(writeStream);
case 18:
_iteratorNormalCompletion2 = true;
_context.next = 6;
break;
case 21:
_context.next = 27;
break;
case 23:
_context.prev = 23;
_context.t0 = _context["catch"](4);
_didIteratorError2 = true;
_iteratorError2 = _context.t0;
case 27:
_context.prev = 27;
_context.prev = 28;
if (!_iteratorNormalCompletion2 && _iterator2.return) {
_iterator2.return();
}
case 30:
_context.prev = 30;
if (!_didIteratorError2) {
_context.next = 33;
break;
}
throw _iteratorError2;
case 33:
return _context.finish(30);
case 34:
return _context.finish(27);
case 35:
resultData = Object.assign({}, job.result.data);
attachments.forEach(function (attachment) {
var key = attachment.key;
var updatedAttachment = Object.assign({}, attachment);
delete updatedAttachment.data;
delete updatedAttachment.key;
resultData[key] = updatedAttachment;
});
_context.next = 39;
return service.updateJob(job.id, { result: { data: resultData } }, { stripResults: true });
case 39:
case "end":
return _context.stop();
}
}
}, _loop, _this, [[4, 23, 27, 35], [28,, 30, 34]]);
});
_iteratorNormalCompletion = true;
_didIteratorError = false;
_iteratorError = undefined;
_context2.prev = 8;
_iterator = jobs[Symbol.iterator]();
case 10:
if (_iteratorNormalCompletion = (_step = _iterator.next()).done) {
_context2.next = 16;
break;
}
job = _step.value;
return _context2.delegateYield(_loop(job), "t0", 13);
case 13:
_iteratorNormalCompletion = true;
_context2.next = 10;
break;
case 16:
_context2.next = 22;
break;
case 18:
_context2.prev = 18;
_context2.t1 = _context2["catch"](8);
_didIteratorError = true;
_iteratorError = _context2.t1;
case 22:
_context2.prev = 22;
_context2.prev = 23;
if (!_iteratorNormalCompletion && _iterator.return) {
_iterator.return();
}
case 25:
_context2.prev = 25;
if (!_didIteratorError) {
_context2.next = 28;
break;
}
throw _iteratorError;
case 28:
return _context2.finish(25);
case 29:
return _context2.finish(22);
case 30:
if (jobs.length > 0) {
_context2.next = 1;
break;
}
case 31:
case "end":
return _context2.stop();
}
}
}, _callee, this, [[8, 18, 22, 30], [23,, 25, 29]]);
}));
return function migrate(_x, _x2) {
return _ref.apply(this, arguments);
};
}();
/**
* @event ArtifactManager#migrationComplete
*/
/**
* Artifact Manager
* @augments EventEmitter
* @memberof module:Vulpes
*/
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; }
function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
var os = require("os");
var path = require("path");
var fs = require("fs");
var EventEmitter = require("eventemitter3");
var uuid = require("uuid/v4");
var pify = require("pify");
var rimraf = pify(require("rimraf"));
var mkdirp = require("mkdirp");
var VError = require("verror");
var dataURIToBuffer = require("data-uri-to-buffer");
var parseDataURI = require("parse-data-uri");
var fileExists = require("file-exists");
var _require = require("./streams.js"),
waitForStream = _require.waitForStream;
var _require2 = require("./symbols.js"),
ATTACHMENT_PREFIX = _require2.ATTACHMENT_PREFIX,
ATTACHMENT_REXP = _require2.ATTACHMENT_REXP,
ERROR_CODE_ARTIFACT_NOTFOUND = _require2.ERROR_CODE_ARTIFACT_NOTFOUND;
function getArtifactPath(storagePath, artifactID) {
return path.join(storagePath, artifactID + ".vulpesartifact");
}
function getDefaultStoragePath() {
if (global.VULPES_ARTIFACTS_PATH) {
return global.VULPES_ARTIFACTS_PATH;
}
return path.join(os.homedir(), ".vulpes/artifacts");
}
var ArtifactManager = function (_EventEmitter) {
_inherits(ArtifactManager, _EventEmitter);
/**
* Constructor for the artifact manager
* @param {String=} storagePath The path to store artifacts in. Defaults to
* `~/.vulpes/artifacts` if not specified.
*/
function ArtifactManager() {
var storagePath = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : getDefaultStoragePath();
_classCallCheck(this, ArtifactManager);
var _this2 = _possibleConstructorReturn(this, (ArtifactManager.__proto__ || Object.getPrototypeOf(ArtifactManager)).call(this));
_this2._path = storagePath;
_this2._migrating = false;
_this2.service = null;
return _this2;
}
/**
* @typedef {Object} NewJobAttachmentOptions
* @property {String=} title The title of the attachment
* @property {Buffer|String|ReadableStream} data The attachment data
* @property {String} mime The mime type of the attachment
* @property {Number=} created The timestamp of creation
*/
/**
* Add a new job attachment
* @param {String} jobID The job ID to add the attachment to
* @param {NewJobAttachmentOptions} param1 Options for the new attachment
* @returns {Promise.<String>} A promise that resolves with the attachment ID
*/
_createClass(ArtifactManager, [{
key: "addJobAttachment",
value: function () {
var _ref2 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee2(jobID) {
var _ref3 = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {},
_ref3$id = _ref3.id,
id = _ref3$id === undefined ? uuid() : _ref3$id,
_ref3$title = _ref3.title,
title = _ref3$title === undefined ? "Untitled" : _ref3$title,
data = _ref3.data,
mime = _ref3.mime,
_ref3$created = _ref3.created,
created = _ref3$created === undefined ? Date.now() : _ref3$created;
var attachment, writeStream;
return regeneratorRuntime.wrap(function _callee2$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
attachment = {
id: id,
title: title,
mime: mime,
created: created
};
_context3.next = 3;
return this.getArtifactWriteStream(attachment.id);
case 3:
writeStream = _context3.sent;
if (!(Buffer.isBuffer(data) || typeof data === "string")) {
_context3.next = 9;
break;
}
writeStream.write(data);
writeStream.end();
_context3.next = 12;
break;
case 9:
// Assume stream
data.pipe(writeStream);
_context3.next = 12;
return waitForStream(data);
case 12:
_context3.next = 14;
return waitForStream(writeStream);
case 14:
_context3.next = 16;
return this.service.updateJob(jobID, {
result: {
data: _defineProperty({}, "" + ATTACHMENT_PREFIX + attachment.id, attachment)
}
});
case 16:
return _context3.abrupt("return", attachment.id);
case 17:
case "end":
return _context3.stop();
}
}
}, _callee2, this);
}));
function addJobAttachment(_x4) {
return _ref2.apply(this, arguments);
}
return addJobAttachment;
}()
/**
* Initialise the manager (called by Service)
* @param {Service} service The service instance we're attached to
* @returns {Promise}
* @fires ArtifactManager#migrationComplete
*/
}, {
key: "initialise",
value: function () {
var _ref4 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee4(service) {
var _this3 = this;
return regeneratorRuntime.wrap(function _callee4$(_context5) {
while (1) {
switch (_context5.prev = _context5.next) {
case 0:
this.service = service;
_context5.next = 3;
return mkdirp(this._path);
case 3:
this.service.once("initialised", _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee3() {
return regeneratorRuntime.wrap(function _callee3$(_context4) {
while (1) {
switch (_context4.prev = _context4.next) {
case 0:
_context4.prev = 0;
_this3._migrating = true;
_context4.next = 4;
return migrate(_this3.service, _this3);
case 4:
_context4.next = 8;
break;
case 6:
_context4.prev = 6;
_context4.t0 = _context4["catch"](0);
case 8:
_this3._migrating = false;
_this3.emit("migrationComplete");
case 10:
case "end":
return _context4.stop();
}
}
}, _callee3, _this3, [[0, 6]]);
})));
case 4:
case "end":
return _context5.stop();
}
}
}, _callee4, this);
}));
function initialise(_x6) {
return _ref4.apply(this, arguments);
}
return initialise;
}()
/**
* Get a readable stream of an artifact
* @param {String} artifactID The artifact's ID
* @returns {Promise.<ReadableStream>}
*/
}, {
key: "getArtifactReadStream",
value: function () {
var _ref6 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee5(artifactID) {
var filename, exists;
return regeneratorRuntime.wrap(function _callee5$(_context6) {
while (1) {
switch (_context6.prev = _context6.next) {
case 0:
filename = getArtifactPath(this._path, artifactID);
_context6.next = 3;
return fileExists(filename);
case 3:
exists = _context6.sent;
if (exists) {
_context6.next = 6;
break;
}
throw new VError({
info: { code: ERROR_CODE_ARTIFACT_NOTFOUND }
}, "No artifact found at path: " + filename);
case 6:
return _context6.abrupt("return", fs.createReadStream(filename));
case 7:
case "end":
return _context6.stop();
}
}
}, _callee5, this);
}));
function getArtifactReadStream(_x7) {
return _ref6.apply(this, arguments);
}
return getArtifactReadStream;
}()
/**
* Get a writeable stream for an artifact
* @param {String} artifactID The artifact's ID
* @returns {Promise.<WritableStream>}
*/
}, {
key: "getArtifactWriteStream",
value: function () {
var _ref7 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee6(artifactID) {
var filename;
return regeneratorRuntime.wrap(function _callee6$(_context7) {
while (1) {
switch (_context7.prev = _context7.next) {
case 0:
filename = getArtifactPath(this._path, artifactID);
return _context7.abrupt("return", fs.createWriteStream(filename));
case 2:
case "end":
return _context7.stop();
}
}
}, _callee6, this);
}));
function getArtifactWriteStream(_x8) {
return _ref7.apply(this, arguments);
}
return getArtifactWriteStream;
}()
/**
* @typedef {Object} JobAttachment
* @property {String} id - The attachment/artifact ID
* @property {String} title - The artifact title
* @property {String} mime - The mime type for the artifact
* @property {Number} created - The JS timestamp of creation/addition
*/
/**
* Get all job attachments
* @param {String} jobID The ID of the job to fetch artifacts for
* @returns {JobAttachment[]}
*/
}, {
key: "getJobAttachments",
value: function () {
var _ref8 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee7(jobID) {
var job;
return regeneratorRuntime.wrap(function _callee7$(_context8) {
while (1) {
switch (_context8.prev = _context8.next) {
case 0:
_context8.next = 2;
return this.service.getJob(jobID);
case 2:
job = _context8.sent;
return _context8.abrupt("return", Object.keys(job.result.data || {}).reduce(function (payload, resultKey) {
if (ATTACHMENT_REXP.test(resultKey)) {
payload.push(Object.assign(job.result.data[resultKey], {
id: job.result.data[resultKey].id || resultKey.replace(ATTACHMENT_PREFIX, "")
}));
}
return payload;
}, []));
case 4:
case "end":
return _context8.stop();
}
}
}, _callee7, this);
}));
function getJobAttachments(_x9) {
return _ref8.apply(this, arguments);
}
return getJobAttachments;
}()
/**
* Remove an artifact (does not affect jobs)
* @param {String} artifactID The ID of the artifact to remove
* @returns {Promise}
*/
}, {
key: "removeArtifact",
value: function () {
var _ref9 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee8(artifactID) {
var filename;
return regeneratorRuntime.wrap(function _callee8$(_context9) {
while (1) {
switch (_context9.prev = _context9.next) {
case 0:
filename = getArtifactPath(this._path, artifactID);
_context9.next = 3;
return rimraf(filename);
case 3:
case "end":
return _context9.stop();
}
}
}, _callee8, this);
}));
function removeArtifact(_x10) {
return _ref9.apply(this, arguments);
}
return removeArtifact;
}()
/**
* Remove an attachment from a job, also removing
* the associated artifact
* @param {String} jobID The ID of the job containing the artifact
* @param {String} artifactID The ID of the artifact to remove from
* the job
* @returns {Promise}
*/
}, {
key: "removeJobAttachment",
value: function () {
var _ref10 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee9(jobID, artifactID) {
var job, results;
return regeneratorRuntime.wrap(function _callee9$(_context10) {
while (1) {
switch (_context10.prev = _context10.next) {
case 0:
_context10.next = 2;
return this.service.getJob(jobID);
case 2:
job = _context10.sent;
results = job.result.data || {};
delete results["" + ATTACHMENT_PREFIX + artifactID];
_context10.next = 7;
return this.service.updateJob(jobID, { result: { data: results } }, { stripResults: true });
case 7:
_context10.next = 9;
return this.removeArtifact(artifactID);
case 9:
case "end":
return _context10.stop();
}
}
}, _callee9, this);
}));
function removeJobAttachment(_x11, _x12) {
return _ref10.apply(this, arguments);
}
return removeJobAttachment;
}()
/**
* Shutdown the artifact manager
* @returns {Promise}
*/
}, {
key: "shutdown",
value: function () {
var _ref11 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee10() {
var _this4 = this;
return regeneratorRuntime.wrap(function _callee10$(_context11) {
while (1) {
switch (_context11.prev = _context11.next) {
case 0:
return _context11.abrupt("return", new Promise(function (resolve) {
if (!_this4._migrating) {
return resolve();
}
_this4.once("migrationComplete", resolve);
}));
case 1:
case "end":
return _context11.stop();
}
}
}, _callee10, this);
}));
function shutdown() {
return _ref11.apply(this, arguments);
}
return shutdown;
}()
}]);
return ArtifactManager;
}(EventEmitter);
module.exports = ArtifactManager;