vulpes
Version:
Job management framework
454 lines (387 loc) • 21 kB
JavaScript
;
var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; };
var addJobBatch = function () {
var _ref = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee3(service, jobs) {
var _this = this;
var results, resolvedIDs, i, processBatch;
return regeneratorRuntime.wrap(function _callee3$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
results = JSON.parse(JSON.stringify(jobs));
// Create empty array to track resolutions
resolvedIDs = jobs.map(function () {
return "";
});
i = 0;
case 3:
if (!(i < jobs.length)) {
_context3.next = 9;
break;
}
if (!(typeof jobs[i].id === "undefined" || jobs[i].id === null)) {
_context3.next = 6;
break;
}
throw new VError({ info: { code: ERROR_CODE_JOB_BATCH_IDS } }, "Failed adding job batch: All jobs must have an ID (non-UUID)");
case 6:
i += 1;
_context3.next = 3;
break;
case 9:
processBatch = function () {
var _ref2 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee2() {
var workPerformed, work;
return regeneratorRuntime.wrap(function _callee2$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
workPerformed = false, work = Promise.resolve();
results.filter(function (pendingJob, index) {
return !resolvedIDs[index];
}).forEach(function (pendingJob) {
var id = pendingJob.id,
_pendingJob$parents = pendingJob.parents,
parentsRaw = _pendingJob$parents === undefined ? [] : _pendingJob$parents;
var parents = parentsRaw;
if (UUID_REXP.test(id)) {
throw new VError({ info: { code: ERROR_CODE_JOB_BATCH_ID_FORMAT } }, "Failed adding job batch: Cannot add jobs with pre-set UUID: " + id);
}
// First check if the parents are resolved
if (parents && parents.length > 0) {
var resolvedParents = parents.map(function (parentID) {
if (UUID_REXP.test(parentID) === false) {
// Try to find job in resolved IDs
var targetIndex = jobs.findIndex(function (job) {
return job.id === parentID;
});
if (targetIndex >= 0 === false && !resolvedIDs[targetIndex]) {
throw new VError({ info: { code: ERROR_CODE_JOB_BATCH_PARENT_RESOLUTION } }, "Failed adding job batch: Failed resolving parent ID: " + parentID);
}
return resolvedIDs[targetIndex];
}
return parentID;
});
var allResolved = resolvedParents.every(function (parentID) {
return UUID_REXP.test(parentID);
});
if (!allResolved) {
// Some IDs could not be resolved
return;
}
parents = resolvedParents;
}
// Add this job to the service
workPerformed = true;
work = work.then(_asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee() {
var jobID, index;
return regeneratorRuntime.wrap(function _callee$(_context) {
while (1) {
switch (_context.prev = _context.next) {
case 0:
_context.next = 2;
return service.addJob(Object.assign(pendingJob, {
id: null,
parents: parents
}));
case 2:
jobID = _context.sent;
index = jobs.findIndex(function (job) {
return job.id === id;
});
resolvedIDs[index] = jobID;
_context.next = 7;
return service.getJob(jobID);
case 7:
results[index] = _context.sent;
case 8:
case "end":
return _context.stop();
}
}
}, _callee, _this);
})));
});
if (workPerformed) {
_context2.next = 4;
break;
}
throw new VError({ info: { code: ERROR_CODE_JOB_BATCH_DEPENDENCIES } }, "Failed adding job batch: Stalled while resolving dependencies");
case 4:
_context2.next = 6;
return work;
case 6:
if (!(resolvedIDs.every(function (id) {
return !!id;
}) === false)) {
_context2.next = 9;
break;
}
_context2.next = 9;
return processBatch();
case 9:
case "end":
return _context2.stop();
}
}
}, _callee2, _this);
}));
return function processBatch() {
return _ref2.apply(this, arguments);
};
}();
_context3.next = 12;
return processBatch();
case 12:
return _context3.abrupt("return", results);
case 13:
case "end":
return _context3.stop();
}
}
}, _callee3, this);
}));
return function addJobBatch(_x, _x2) {
return _ref.apply(this, arguments);
};
}();
var ensureParentsComplete = function () {
var _ref4 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee4(service, job) {
var complete;
return regeneratorRuntime.wrap(function _callee4$(_context4) {
while (1) {
switch (_context4.prev = _context4.next) {
case 0:
_context4.next = 2;
return checkParentsComplete(service, job);
case 2:
complete = _context4.sent;
if (complete) {
_context4.next = 5;
break;
}
throw new VError({ info: { code: ERROR_CODE_PARENTS_INCOMPLETE } }, "Job " + job.id + " has parents that have not completed successfully");
case 5:
case "end":
return _context4.stop();
}
}
}, _callee4, this);
}));
return function ensureParentsComplete(_x3, _x4) {
return _ref4.apply(this, arguments);
};
}();
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
var VError = require("verror");
var nested = require("nested-property");
var _require = require("./symbols.js"),
ERROR_CODE_JOB_BATCH_DEPENDENCIES = _require.ERROR_CODE_JOB_BATCH_DEPENDENCIES,
ERROR_CODE_JOB_BATCH_IDS = _require.ERROR_CODE_JOB_BATCH_IDS,
ERROR_CODE_JOB_BATCH_ID_FORMAT = _require.ERROR_CODE_JOB_BATCH_ID_FORMAT,
ERROR_CODE_JOB_BATCH_PARENT_RESOLUTION = _require.ERROR_CODE_JOB_BATCH_PARENT_RESOLUTION,
ERROR_CODE_PARENTS_INCOMPLETE = _require.ERROR_CODE_PARENTS_INCOMPLETE,
JOB_RESULT_TYPE_SUCCESS = _require.JOB_RESULT_TYPE_SUCCESS,
JOB_RESULT_TYPES_RESTARTABLE_REXP = _require.JOB_RESULT_TYPES_RESTARTABLE_REXP,
JOB_STATUS_PENDING = _require.JOB_STATUS_PENDING,
JOB_STATUS_STOPPED = _require.JOB_STATUS_STOPPED,
UUID_REXP = _require.UUID_REXP;
function checkParentsComplete(service, job) {
if (job.parents.length === 0) {
return Promise.resolve(true);
}
return Promise.all(job.parents.map(function (parentID) {
return service.getJob(parentID).then(function (job) {
return {
status: job.status,
result: job.result.type
};
});
})).then(function (jobStates) {
return jobStates.every(function (jobState) {
return jobState.status === JOB_STATUS_STOPPED && jobState.result === JOB_RESULT_TYPE_SUCCESS;
});
});
}
function jobCanBeRestarted(job) {
return job.status === JOB_STATUS_STOPPED && (JOB_RESULT_TYPES_RESTARTABLE_REXP.test(job.result.type) || job.result.type === null);
}
/**
* @typedef {Object} PredicatesTestResult
* @property {Boolean} satisfies - True if all predicates satisfied, false otherwise
* @property {String=} predicate - Name of the failing predicate
*/
/**
* Test if a job satisfies all of its predicates
* @param {Service} service The service
* @param {Job} job The job to test
* @returns {PredicatesTestResult} Test results
*/
function jobSatisfiesPredicates(service, job) {
return Promise.resolve().then(function () {
var _job$predicate = job.predicate,
attemptsMax = _job$predicate.attemptsMax,
locked = _job$predicate.locked,
timeBetweenRetries = _job$predicate.timeBetweenRetries;
var attempts = job.attempts,
status = job.status;
var lastStopped = job.times.stopped;
var now = Date.now();
if (typeof attemptsMax === "number" && attempts >= attemptsMax) {
return { satisfies: false, predicate: "attemptsMax" };
}
if (attempts > 0 && now - lastStopped < timeBetweenRetries && status !== JOB_STATUS_PENDING // If in pending state, the job has been restarted and can begin
) {
return { satisfies: false, predicate: "timeBetweenRetries" };
}
// @todo CRON timings
return locked ? { satisfies: false, predicate: "locked" } : { satisfies: true };
});
}
function normaliseJobData(data) {
return data && (typeof data === "undefined" ? "undefined" : _typeof(data)) === "object" ? data : {};
}
function pickFirstJob(service, jobs) {
var _this2 = this;
var jobsCollection = [].concat(_toConsumableArray(jobs));
var tryNext = function () {
var _ref5 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee5() {
var job, _ref6, satisfies, parentsComplete;
return regeneratorRuntime.wrap(function _callee5$(_context5) {
while (1) {
switch (_context5.prev = _context5.next) {
case 0:
job = jobsCollection.shift();
if (job) {
_context5.next = 3;
break;
}
return _context5.abrupt("return", Promise.resolve(null));
case 3:
_context5.next = 5;
return jobSatisfiesPredicates(service, job);
case 5:
_ref6 = _context5.sent;
satisfies = _ref6.satisfies;
if (satisfies) {
_context5.next = 11;
break;
}
_context5.next = 10;
return tryNext();
case 10:
return _context5.abrupt("return", _context5.sent);
case 11:
_context5.next = 13;
return checkParentsComplete(service, job);
case 13:
parentsComplete = _context5.sent;
if (parentsComplete) {
_context5.next = 18;
break;
}
_context5.next = 17;
return tryNext();
case 17:
return _context5.abrupt("return", _context5.sent);
case 18:
return _context5.abrupt("return", Promise.resolve(job));
case 19:
case "end":
return _context5.stop();
}
}
}, _callee5, _this2);
}));
return function tryNext() {
return _ref5.apply(this, arguments);
};
}();
return tryNext();
}
function pickOnlySticky(data) {
return Object.keys(data).filter(function (key) {
return (/^\$/.test(key)
);
}).reduce(function (output, key) {
return Object.assign(output, _defineProperty({}, key, data[key]));
}, {});
}
function prepareJobForWorker(service, job) {
var id = job.id,
type = job.type,
data = job.data,
result = job.result,
parents = job.parents,
priority = job.priority,
status = job.status,
timeLimit = job.timeLimit;
// Create the job structure (data is added later)
var workerJob = {
id: id,
type: type,
data: {},
priority: priority,
status: status,
timeLimit: timeLimit
};
return service.queryJobs({
id: function id(jobID) {
return parents.indexOf(jobID) >= 0;
}
}).then(function (parentJobs) {
// Add data from parents first
parentJobs.forEach(function (parentJob) {
Object.assign(workerJob.data, stripLazyProperties(normaliseJobData(parentJob.data)), stripLazyProperties(normaliseJobData(parentJob.result.data)));
});
// Update data to merge actual job's data and its previous sticky
// results if it has been run earlier
Object.assign(workerJob.data, normaliseJobData(data), pickOnlySticky(normaliseJobData(result.data)));
workerJob.data = resolveLazyProperties(workerJob.data);
return workerJob;
});
}
function resolveLazyProperties(dataset) {
var output = Object.assign({}, dataset);
(function __resolveLevel(lvl) {
Object.keys(lvl).forEach(function (key) {
if (/^\?/.test(key) && /^\?/.test(lvl[key]) === false) {
var newKey = key.replace(/^\?/, "");
nested.set(lvl, newKey, nested.get(lvl, lvl[key]));
lvl[key] = undefined;
delete lvl[key];
}
});
Object.keys(lvl).forEach(function (key) {
if (lvl[key] && _typeof(lvl[key]) === "object") {
__resolveLevel(lvl[key]);
}
});
})(output);
return output;
}
function stripLazyProperties(dataset) {
var output = Object.assign({}, dataset);
(function __resolveLevel(lvl) {
Object.keys(lvl).forEach(function (key) {
if (/^\?/.test(key)) {
lvl[key] = undefined;
delete lvl[key];
} else if (lvl[key] && _typeof(lvl[key]) === "object") {
__resolveLevel(lvl[key]);
}
});
})(output);
return output;
}
module.exports = {
addJobBatch: addJobBatch,
ensureParentsComplete: ensureParentsComplete,
jobCanBeRestarted: jobCanBeRestarted,
jobSatisfiesPredicates: jobSatisfiesPredicates,
pickFirstJob: pickFirstJob,
prepareJobForWorker: prepareJobForWorker,
resolveLazyProperties: resolveLazyProperties
};