topology-runner
Version:
Run a topology consisting of a directed acyclic graph
427 lines (425 loc) • 16.4 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.resumeTopology = exports.getResumeSnapshot = exports.runTopology = exports.initSnapshotData = exports.getMissingSpecNodes = exports.getInputData = exports.getNodesReadyToRun = exports.filterDAG = exports.TopologyError = void 0;
const fp_1 = __importDefault(require("lodash/fp"));
const util_1 = require("./util");
const eventemitter3_1 = __importDefault(require("eventemitter3"));
const make_error_1 = __importDefault(require("make-error"));
exports.TopologyError = (0, make_error_1.default)('TopologyError');
/**
* Remove excludesNodes from DAG and from the dependency lists
* of nodes.
*/
const removeExcludeNodes = (excludeNodes) => fp_1.default.flow(fp_1.default.omit(excludeNodes), fp_1.default.mapValues(fp_1.default.update('deps', fp_1.default.pullAll(excludeNodes))));
/**
* Only include nodes from DAG in includeNodes and remove dependencies
* not in includeNodes.
*/
const pickIncludeNodes = (includeNodes) => fp_1.default.flow(fp_1.default.pick(includeNodes), fp_1.default.mapValues(fp_1.default.update('deps', fp_1.default.intersection(includeNodes))));
/**
* Handle excludeNodes and includeNodes options, transforming the
* DAG accordingly. Only one option will be handled, not both.
*/
const filterDAG = (dag, options = {}) => {
if (options.excludeNodes)
return removeExcludeNodes(options.excludeNodes)(dag);
if (options.includeNodes)
return pickIncludeNodes(options.includeNodes)(dag);
return dag;
};
exports.filterDAG = filterDAG;
/**
* Get the dependents of the node.
*/
const getDependents = (node, dag) => {
const nodes = [];
for (const key in dag) {
const { deps } = dag[key];
if (deps.includes(node)) {
nodes.push(key);
}
}
return nodes;
};
/**
* Get a list of nodes where all dependencies have completed and the
* node's status is pending.
*/
const getNodesReadyToRun = (dag, data) => {
// Get completed nodes
const completed = (0, util_1.findKeys)({ status: 'completed' }, data);
const nodes = [];
for (const node in dag) {
const { deps } = dag[node];
// The node does not exist or status is pending
const isPending = !data[node] || data[node].status === 'pending';
// Dependencies for the node are all completed and the node is currently
// pending.
if (fp_1.default.difference(deps, completed).length === 0 && isPending) {
nodes.push(node);
}
}
return nodes;
};
exports.getNodesReadyToRun = getNodesReadyToRun;
/**
* Get the input data from dependencies.
*/
const getInputData = (dag, node, data) => {
// Use existing input data from snapshot if it exists
const snapshotInput = fp_1.default.get([node, 'input'], data);
if (snapshotInput) {
return snapshotInput;
}
const deps = dag[node].deps;
const input = [];
for (const dep of deps) {
const type = dag[dep].type;
// Input is the output of the dependency
if (type === 'work') {
input.push(fp_1.default.get([dep, 'output'], data));
}
// Input is the input of the dependency
else {
const depInput = fp_1.default.get([dep, 'input'], data);
if (depInput) {
input.push(...depInput);
}
}
}
return input;
};
exports.getInputData = getInputData;
/**
* Update the snapshot when various node-level events take place
* and emit the modified snapshot.
*/
const nodeEventHandler = (snapshot, emitter) => (node) => {
const updateState = (state) => {
// Update snapshot
snapshot.data[node].state = state;
// Emit
emitter.emit('data', snapshot);
};
const running = (data) => {
// Update snapshot
snapshot.data[node].started = new Date();
snapshot.data[node].input = data;
snapshot.data[node].status = 'running';
// Emit
emitter.emit('data', snapshot);
};
const completed = (output) => {
// Update snapshot
if (output !== undefined) {
snapshot.data[node].output = output;
}
snapshot.data[node].status = 'completed';
snapshot.data[node].finished = new Date();
// Emit
emitter.emit('data', snapshot);
};
const branched = (selected, reason) => {
// Update snapshot
snapshot.data[node].status = 'completed';
snapshot.data[node].selected = selected;
if (reason) {
snapshot.data[node].reason = reason;
}
snapshot.data[node].finished = new Date();
// Emit
emitter.emit('data', snapshot);
};
const skipped = () => {
// Update snapshot
snapshot.data[node].status = 'skipped';
snapshot.data[node].finished = new Date();
// Emit
emitter.emit('data', snapshot);
};
const suspended = () => {
// Update snapshot
snapshot.data[node].status = 'suspended';
snapshot.data[node].finished = new Date();
// Emit
emitter.emit('data', snapshot);
};
const errored = (error) => {
// Update snapshot
snapshot.data[node].status = 'errored';
snapshot.data[node].finished = new Date();
// Capture stack and any abritrary properties if instance of Error
snapshot.data[node].error =
error instanceof Error ? { ...error, stack: error.stack } : error;
// Emit
emitter.emit('data', snapshot);
};
return {
updateState,
running,
completed,
branched,
skipped,
suspended,
errored,
};
};
/**
* Check if all nodes in the DAG are in the spec
*/
const getMissingSpecNodes = (spec, dag) => fp_1.default.difference(Object.keys(dag), Object.keys(spec));
exports.getMissingSpecNodes = getMissingSpecNodes;
const branch = (node, reason) => ({
node,
reason,
});
const NONE = Symbol('NONE');
const none = (reason) => ({
node: NONE,
reason,
});
// eslint-disable-next-line
const noOp = async () => { };
const _runTopology = (spec, dag, snapshot, context) => {
const missingSpecNodes = (0, exports.getMissingSpecNodes)(spec, dag);
if (missingSpecNodes.length) {
throw new exports.TopologyError(`Missing the following nodes in spec: ${missingSpecNodes.join(', ')}`);
}
const abortController = new AbortController();
// Track node promises
const promises = {};
// Event emitter
const emitter = new eventemitter3_1.default();
const eventHandler = nodeEventHandler(snapshot, emitter);
const start = async () => {
// Emit initial snapshot
emitter.emit('data', snapshot);
// Loop
while (true) {
// Get nodes with resolved dependencies that have not been run
const readyToRunNodes = abortController.signal.aborted
? []
: (0, exports.getNodesReadyToRun)(dag, snapshot.data);
// There is no more work to be done
if (fp_1.default.isEmpty(readyToRunNodes) && fp_1.default.isEmpty(promises)) {
// Get errored nodes
const errored = (0, util_1.findKeys)({ status: 'errored' }, snapshot.data);
const hasErrors = !fp_1.default.isEmpty(errored);
// Was part of the toplogy suspended?
const suspended = (0, util_1.findKeys)({ status: 'suspended' }, snapshot.data);
// Update snapshot
const status = hasErrors
? 'errored'
: suspended.length
? 'suspended'
: 'completed';
snapshot.status = status;
// Get nodes with pending status
const pending = (0, util_1.findKeys)({ status: 'pending' }, snapshot.data);
// Suspend pending nodes if topology is suspended
if (status === 'suspended') {
pending.forEach((node) => eventHandler(node).suspended());
}
// Skip pending nodes if topology is completed
if (status === 'completed') {
pending.forEach((node) => eventHandler(node).skipped());
}
snapshot.finished = new Date();
// Emit
emitter.emit(hasErrors ? 'error' : 'done', snapshot);
// Throw an exception, causing the promise to reject, if one or more
// nodes have errored
if (hasErrors) {
throw new exports.TopologyError(`Errored nodes: ${JSON.stringify(errored)}`);
}
// Exit loop
return;
}
// Run nodes that have not been run yet
for (const node of readyToRunNodes) {
// Snapshot updater
const events = eventHandler(node);
// Get the node from the spec
const { run, type } = spec[node];
// Get the input data from dependencies
const data = (0, exports.getInputData)(dag, node, snapshot.data);
// Run fn input
const baseInput = { data, node, context };
// Update snapshot
events.running(data);
// Get dependent nodes
const dependents = getDependents(node, dag);
// Handle branching node type
if (type === 'branching') {
const runFn = async () => run({ ...baseInput, branch, none });
promises[node] = runFn()
.then(({ node: selected, reason }) => {
events.branched(selected.toString(), reason);
// Explicitly skip all dependents
if (selected === NONE) {
// Skip all dependents
dependents.forEach((node) => eventHandler(node).skipped());
}
else if (typeof selected === 'string') {
// Check if selected branch is a dependent of the node
if (!dependents.includes(selected)) {
throw new exports.TopologyError(`Branch not found: ${selected}`);
}
// Skip dependents that were not selected
fp_1.default.pull(selected, dependents).forEach((node) => eventHandler(node).skipped());
}
})
.catch(events.errored)
.finally(() => {
delete promises[node];
});
}
// Handle suspension and work node types
else {
// Callback to update state
const updateState = events.updateState;
// Get node state. Will only be present if resuming.
const state = snapshot.data[node]?.state;
const input = {
...baseInput,
updateState,
state,
signal: abortController.signal,
};
const handleSuspension = () => {
// Set status to completed
events.completed();
// Suspend all dependent nodes
dependents.forEach((node) => eventHandler(node).suspended());
};
// The run fn may not exist for suspension node type
const runFn = run || noOp;
promises[node] = runFn(input)
.then(type === 'suspension' ? handleSuspension : events.completed)
.catch(events.errored)
.finally(() => {
delete promises[node];
});
}
}
// Wait for a promise to resolve
await Promise.race(fp_1.default.values(promises));
}
};
const getSnapshot = () => snapshot;
const stop = () => {
abortController.abort();
};
return { start, stop, emitter, getSnapshot };
};
const getNodesWithNoDeps = (dag) => (0, util_1.findKeys)(({ deps }) => fp_1.default.isEmpty(deps), dag);
/**
* Set input for nodes with no dependencies to data, if exists.
*/
const initSnapshotData = (dag, data) => {
// Get nodes with no dependencies
const noDepsNodes = getNodesWithNoDeps(dag);
const snapshotData = {};
for (const node in dag) {
snapshotData[node] = { ...dag[node], status: 'pending' };
// Data exists and node has no dependencies
if (data !== undefined && noDepsNodes.includes(node)) {
snapshotData[node].input = [data];
}
}
return snapshotData;
};
exports.initSnapshotData = initSnapshotData;
const extractDag = (obj) => {
const dag = {};
for (const node in obj) {
const nodeDef = obj[node];
// Default node type to work if not set
dag[node] = { deps: nodeDef.deps, type: nodeDef.type || 'work' };
}
return dag;
};
/**
* Run a topology consisting of a DAG and functions for each node in the
* DAG. A subset of the DAG can be executed by setting either includeNodes
* or excludeNodes. Initial data is passed via options.data. Optionally
* pass a context blob to all nodes.
*
* Returns an event emitter and a promise. The event emitter emits data
* every time the topology snapshot updates. Events include: data, error, and
* done.
*/
const runTopology = (spec, options) => {
const _dag = extractDag(spec);
// Get the filtered dag
const dag = (0, exports.filterDAG)(_dag, options);
// Initialize snapshot data
const data = (0, exports.initSnapshotData)(dag, options?.data);
// Initial snapshot
const snapshot = {
status: 'running',
started: new Date(),
data,
};
// Run the topology
return _runTopology(spec, dag, snapshot, options?.context);
};
exports.runTopology = runTopology;
/**
* Set uncompleted nodes to 'pending' and reset appropriate
* NodeDef fields. Ignore skipped nodes.
*/
const resetUncompletedNodes = (data) => fp_1.default.mapValues((nodeData) => {
const { status, ...obj } = nodeData;
return status === 'completed' || status === 'skipped'
? nodeData
: {
...fp_1.default.pick(['input', 'state', 'deps', 'type'], obj),
status: 'pending',
};
}, data);
/**
* Transform a previously run snapshot into one that is ready to
* be run again.
*/
const getResumeSnapshot = (snapshot) => {
const snap = {
...snapshot,
status: 'running',
started: new Date(),
data: resetUncompletedNodes(snapshot.data),
};
delete snap.finished;
return snap;
};
exports.getResumeSnapshot = getResumeSnapshot;
/**
* Resume a topology from a previous snapshot. Optionally pass a context
* blob to all nodes.
*
* Returns an event emitter and a promise. The event emitter emits data
* every time the topology snapshot updates. Events include: data, error, and
* done.
*/
const resumeTopology = (spec, snapshot, options) => {
if (!snapshot) {
throw new exports.TopologyError('Snapshot is undefined');
}
// Ensures resumption is idempotent
if (snapshot.status === 'completed') {
const emitter = new eventemitter3_1.default();
const getSnapshot = () => snapshot;
/* eslint-disable-next-line */
const stop = () => { };
return { start: () => Promise.resolve(), stop, emitter, getSnapshot };
}
// Initialize snapshot for running
const snap = (0, exports.getResumeSnapshot)(snapshot);
const dag = extractDag(snap.data);
// Run the topology
return _runTopology(spec, dag, snap, options?.context);
};
exports.resumeTopology = resumeTopology;