UNPKG

msgflo

Version:

Polyglot FBP runtime based on message queues

989 lines (931 loc) 29.8 kB
var Coordinator, EventEmitter, async, common, connId, connectionFromBinding, debug, findPort, findQueue, fromConnId, fromIipId, fs, https, iipId, library, participantsByRole, path, pingUrl, setup, url, waitForParticipant, indexOf = [].indexOf, boundMethodCheck = function(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new Error('Bound instance method accessed before binding'); } }; debug = require('debug')('msgflo:coordinator'); EventEmitter = require('events').EventEmitter; fs = require('fs'); path = require('path'); async = require('async'); https = require('https'); url = require('url'); setup = require('./setup'); library = require('./library'); common = require('./common'); findPort = function(def, type, portName) { var i, len, port, ports; ports = type === 'inport' ? def.inports : def.outports; for (i = 0, len = ports.length; i < len; i++) { port = ports[i]; if (port.id === portName) { return port; } } return null; }; connId = function(fromId, fromPort, toId, toPort) { return `${fromId} ${fromPort} -> ${toPort} ${toId}`; }; fromConnId = function(id) { var t; t = id.split(' '); return [t[0], t[1], t[4], t[3]]; }; iipId = function(part, port) { return `${part} ${port}`; }; fromIipId = function(id) { return id.split(' '); }; participantsByRole = function(participants, role) { var m, matchRole; matchRole = (id) => { var part; part = participants[id]; return part.role === role; }; m = Object.keys(participants).filter(matchRole); return m; }; // XXX: there is now a mixture of participant id and role used here findQueue = (participants, partId, dir, portName) => { var i, len, part, partIdByRole, port, ref; part = participants[partId]; partIdByRole = participantsByRole(participants, partId)[0]; if (part == null) { part = participants[partIdByRole]; } if (part == null) { throw new Error(`No participant info found for '${partId}'`); } ref = part[dir]; for (i = 0, len = ref.length; i < len; i++) { port = ref[i]; if (port.id === portName) { if (!port.queue) { throw new Error(`Queue for ${dir} '${portName}' missing in ${JSON.stringify(part)}`); } return port.queue; } } throw new Error(`No matching port found for ${dir} '${portName}' in ${JSON.stringify(part)}`); }; connectionFromBinding = function(participants, binding) { var byRole, connection, findNodePort, id, part; byRole = {}; for (id in participants) { part = participants[id]; byRole[part.role] = part; } findNodePort = function(queue, dir) { var i, len, port, r, ref, role; for (role in byRole) { part = byRole[role]; ref = part[dir]; for (i = 0, len = ref.length; i < len; i++) { port = ref[i]; if (port.queue === queue) { r = { node: role, port: port.id }; return r; } } } }; connection = { src: findNodePort(binding.src, 'outports'), tgt: findNodePort(binding.tgt, 'inports') }; return connection; }; waitForParticipant = function(coordinator, role, callback) { var existing, onParticipantAdded, onTimeout, timeout; existing = participantsByRole(coordinator.participants, role); if (existing.length) { return callback(null, coordinator.participants[existing[0]]); } onTimeout = () => { return callback(new Error(`Waiting for participant ${role} timed out`)); }; timeout = setTimeout(onTimeout, coordinator.options.waitTimeout * 1000); onParticipantAdded = (part) => { if (part.role === role) { debug('onParticipantAdded', part.role); // FIXME: take into account multiple participants with same role clearTimeout(timeout); coordinator.removeListener('participant-added', onParticipantAdded); return callback(null); } }; return coordinator.on('participant-added', onParticipantAdded); }; pingUrl = function(address, method, callback) { var req, u; u = url.parse(address); if (u.protocol === 'http' && !u.port) { u.port = 80; } u.method = method; u.timeout = 10 * 1000; req = https.request(u, function(res) { var status; status = res.statusCode; if (status !== 200) { return callback(new Error(`Ping ${method} ${address} failed with ${status}`)); } return callback(null); }); req.on('error', function(err) { return callback(err); }); return req.end(); }; Coordinator = class Coordinator extends EventEmitter { constructor(broker, options = {}) { var libraryOptions; super(); this._onConnectionData = this._onConnectionData.bind(this); this.broker = broker; this.options = options; this.participants = {}; // participantId -> Definition (from discovery) this.connections = {}; // connId -> { queue: opt String, handler: opt function } this.iips = {}; // iipId -> { metadata, data } this.nodes = {}; // role -> { metadata: {} } this.started = false; this.processes = {}; libraryOptions = { configfile: this.options.library, componentdir: this.options.componentdir, config: this.options.config }; this.library = new library.Library(libraryOptions); this.exported = { inports: {}, outports: {} }; if (this.options.waitTimeout == null) { this.options.waitTimeout = 40; } this.graphName = null; this.on('participant', this.checkParticipantConnections); this.alivePingInterval = null; } clearGraph(graphName, callback) { this.connections = {}; this.iips = {}; this.graphName = graphName; this.nodes = {}; this.participants = {}; // NOTE: also removes discovered things, not setup by us. But should soon be discovered again return setup.killProcesses(this.processes, 'SIGTERM', (err) => { this.processes = {}; return callback(err); }); } start(callback) { return this.library.load((err) => { if (err) { return callback(err); } return this.broker.connect((err) => { var alivePing; debug('connected', err); if (err) { return callback(err); } this.broker.subscribeParticipantChange((msg) => { var e; try { this.handleFbpMessage(msg.data); } catch (error) { e = error; console.error('Participant discovery failed:', e.message, '\n', e.stack, '\n', JSON.stringify(msg.data, 2, null)); } return this.broker.ackMessage(msg); }); this.started = true; debug('started', err, this.started); alivePing = () => { if (!this.options.pingInterval) { return; } return pingUrl(this.options.pingUrl, this.options.pingMethod, function(err) { if (err) { return debug('alive-ping-error', err); } return debug('alive-ping-success'); }); }; this.alivePingInterval = setInterval(alivePing, this.options.pingInterval * 1000); alivePing(); return callback(null); }); }); } stop(callback) { return this.clearGraph(this.graphName, (clearErr) => { return this.broker.disconnect((err) => { if (clearErr) { return callback(clearErr); } return callback(err); }); }); } handleFbpMessage(data) { if (data.protocol === 'discovery' && data.command === 'participant') { return this.participantDiscovered(data.payload); } else { throw new Error(`Unknown FBP message: ${typeof data} ${(data != null ? data.protocol : void 0)}:${(data != null ? data.command : void 0)}`); } } participantDiscovered(definition) { if (!definition.id) { throw new Error("Discovery message missing .id"); } if (!definition.component) { throw new Error("Discovery message missing .component"); } if (!definition.role) { throw new Error("Discovery message missing .role"); } if (!definition.inports) { throw new Error("Discovery message missing .inports"); } if (!definition.outports) { throw new Error("Discovery message missing .outports"); } if (this.participants[definition.id]) { return this.updateParticipant(definition); } else { return this.addParticipant(definition); } } updateParticipant(definition) { var k, newDefinition, original, v; debug('updateParticipant', definition.id); original = this.participants[definition.id]; if (!definition.extra) { definition.extra = {}; } definition.extra.lastSeen = new Date; newDefinition = common.clone(definition); for (k in original) { v = original[k]; if (!newDefinition[k]) { newDefinition[k] = v; } } this.participants[definition.id] = newDefinition; this.library._updateDefinition(newDefinition.component, newDefinition); this.emit('participant-updated', newDefinition); return this.emit('participant', 'updated', newDefinition); } addParticipant(definition) { debug('addParticipant', definition.id); if (!definition.extra) { definition.extra = {}; } definition.extra.firstSeen = new Date; definition.extra.lastSeen = new Date; this.participants[definition.id] = definition; if (!this.nodes[definition.role]) { // Ensure we have a node also for discovered participants this.nodes[definition.role] = { metadata: {} }; } this.nodes[definition.role].component = definition.component; this.library._updateDefinition(definition.component, definition); this.emit('participant-added', definition); this.emit('participant', 'added', definition); return this.emit('graph-changed'); } removeParticipant(id) { var definition; definition = this.participants[id]; delete this.participants[id]; this.emit('participant-removed', definition); this.library._updateDefinition(definition.component, null); this.emit('participant', 'removed', definition); return this.emit('graph-changed'); } addComponent(name, language, code, callback) { return this.library.addComponent(name, language, code, callback); } getComponentSource(component, callback) { return this.library.getSource(component, callback); } startParticipant(node, component, metadata, callback) { var cmd, commands, iips, options, ref, ref1; if (typeof metadata === 'function') { callback = metadata; metadata = {}; } if (!metadata) { metadata = {}; } if (((ref = this.options.ignore) != null ? ref.length : void 0) && indexOf.call(this.options.ignore, node) >= 0) { console.log(`WARNING: Not restarting ignored participant ${node}`); return callback(null); } if (!((ref1 = this.library.components[component]) != null ? ref1.command : void 0)) { console.log(`WARNING: Attempting to start participant with missing component: ${node}(${component})`); // XXX: should be an error, but Flowhub does this in project mode.. return callback(null); } if (!this.nodes[node]) { this.nodes[node] = { metadata: {} }; } this.nodes[node].metadata = metadata; this.nodes[node].component = component; iips = {}; cmd = this.library.componentCommand(component, node, iips); commands = {}; commands[node] = cmd; options = { broker: this.options.broker, forward: this.options.forward || '' }; return setup.startProcesses(commands, options, (err, processes) => { var k, v; if (err) { return callback(err); } for (k in processes) { v = processes[k]; this.processes[k] = v; } return waitForParticipant(this, node, function(err) { return callback(err, processes); }); }); } stopParticipant(node, component, callback) { var k, processes, ref, removeDiscoveredParticipants, v; if (!((node != null) && typeof node === 'string')) { return callback(new Error("stopParticipant(): Missing node argument")); } processes = {}; ref = this.processes; for (k in ref) { v = ref[k]; if (k === node) { processes[k] = v; } } delete this.nodes[node]; removeDiscoveredParticipants = (role) => { var def, id, keep, match, ref1; keep = {}; ref1 = this.participants; for (id in ref1) { def = ref1[id]; match = def.role === role; if (!match) { keep[id] = def; } } return this.participants = keep; }; // we know it should stop sending discovery, pre-emptively remove removeDiscoveredParticipants(node); this.emit('graph-changed'); return setup.killProcesses(processes, 'SIGTERM', (err) => { if (err) { return callback(err); } for (k in processes) { v = processes[k]; delete this.processes[k]; } // might have been discovered again during shutdown removeDiscoveredParticipants(node); return callback(null, processes); }); } updateNodeMetadata(node, metadata, callback) { var process; if (!metadata) { metadata = {}; } process = null; if (!this.nodes[node]) { return callback(new Error(`Node ${node} not found`)); } this.nodes[node].metadata = metadata; return callback(null); } sendTo(participantId, inport, message, callback) { var defaultCallback, id, part, port; debug('sendTo', participantId, inport, message); defaultCallback = function(err) { if (err) { throw err; } }; if (!callback) { callback = defaultCallback; } part = this.participants[participantId]; id = participantsByRole(this.participants, participantId)[0]; if (part == null) { part = this.participants[id]; } port = findPort(part, 'inport', inport); if (!port) { return callback(new Error(`Cannot find inport ${inport}`)); } return this.broker.sendTo('inqueue', port.queue, message, callback); } subscribeTo(participantId, outport, handler, callback) { var ackHandler, defaultCallback, id, part, port, readQueue; defaultCallback = function(err) { if (err) { throw err; } }; if (!callback) { callback = defaultCallback; } part = this.participants[participantId]; id = participantsByRole(this.participants, participantId)[0]; if (part == null) { part = this.participants[id]; } debug('subscribeTo', participantId, outport); port = findPort(part, 'outport', outport); ackHandler = (msg) => { if (!this.started) { return; } handler(msg); return this.broker.ackMessage(msg); }; if (!port) { return callback(new Error(`Could not find outport ${outport} for role ${participantId}`)); } // Cannot subscribe directly to an outqueue, must create and bind an inqueue readQueue = 'msgflo-export-' + Math.floor(Math.random() * 999999); return this.broker.createQueue('inqueue', readQueue, (err) => { if (err) { return callback(err); } return this.broker.addBinding({ type: 'pubsub', src: port.queue, tgt: readQueue }, (err) => { if (err) { return callback(err); } return this.broker.subscribeToQueue(readQueue, ackHandler, function(err) { return callback(err, readQueue); // caller should teardown readQueue }); }); }); } unsubscribeFrom() {} // FIXME: implement connect(fromId, fromPort, toId, toName, metadata, callback) { var edge, edgeId; if (typeof metadata === 'function') { callback = metadata; metadata = {}; } if (!metadata) { metadata = {}; } if (!callback) { callback = (function(err) {}); } // NOTE: adding partial connection info to make checkParticipantConnections logic work edgeId = connId(fromId, fromPort, toId, toName); edge = { fromId: fromId, fromPort: fromPort, toId: toId, toName: toName, srcQueue: null, tgtQueue: null, metadata: metadata }; debug('connect', edge); this.connections[edgeId] = edge; // might be that it was just added/started, not yet discovered return waitForParticipant(this, fromId, (err) => { if (err) { return callback(err); } return waitForParticipant(this, toId, (err) => { var binding, edgeWithQueues; if (err) { return callback(err); } // TODO: support roundtrip this.connections[edgeId].srcQueue = findQueue(this.participants, fromId, 'outports', fromPort); this.connections[edgeId].tgtQueue = findQueue(this.participants, toId, 'inports', toName); edgeWithQueues = this.connections[edgeId]; this.emit('graph-changed'); binding = { type: 'pubsub', src: edgeWithQueues.srcQueue, tgt: edgeWithQueues.tgtQueue }; if (!binding.src) { return callback(new Error(`Source queue for connection ${fromId} ${fromPort} not found`)); } if (!binding.tgt) { return callback(new Error(`Target queue for connection ${toName} ${toPort} not found`)); } return this.broker.addBinding(binding, (err) => { return callback(err); }); }); }); } disconnect(fromId, fromPort, toId, toPort, callback) { var edge, edgeId; edgeId = connId(fromId, fromPort, toId, toPort); edge = this.connections[edgeId]; if (!edge) { return callback(new Error(`Could not find connection ${edgeId}`)); } if (!edge.srcQueue && edge.tgtQueue) { return callback(new Error(`No queues for connection ${edgeId}`)); } return this.broker.removeBinding({ type: 'pubsub', src: edge.srcQueue, tgt: edge.tgtQueue }, (err) => { if (err) { return callback(err); } delete this.connections[edgeId]; this.emit('graph-changed'); return callback(null); }); } updateEdge(fromId, fromPort, toId, toPort, metadata, callback) { var edge, edgeId; if (!metadata) { metadata = {}; } edgeId = connId(fromId, fromPort, toId, toPort); edge = this.connections[edgeId]; if (!edge) { return callback(new Error(`Could not find connection ${edgeId}`)); } this.connections[edgeId].metadata = metadata; return callback(null); } checkParticipantConnections(action, participant) { var e, findConnectedPorts, i, isConnected, j, l, len, len1, len2, m, matches, port, ref, ref1, results1, role; findConnectedPorts = (dir, srcPort) => { var conn, i, id, len, part, port, ref, ref1; conn = []; ref = this.participants; // return conn if not srcPort.queue for (id in ref) { part = ref[id]; ref1 = part[dir]; for (i = 0, len = ref1.length; i < len; i++) { port = ref1[i]; if (!port.queue) { continue; } if (port.queue === srcPort.queue) { conn.push({ part: part, port: port }); } } } return conn; }; isConnected = (e) => { var fromId, fromPort, id, toId, toPort; [fromId, fromPort, toId, toPort] = e; id = connId(fromId, fromPort, toId, toPort); return this.connections[id] != null; }; if (action === 'added') { role = participant.role; ref = participant.inports; // inbound for (i = 0, len = ref.length; i < len; i++) { port = ref[i]; matches = findConnectedPorts('outports', port); for (j = 0, len1 = matches.length; j < len1; j++) { m = matches[j]; e = [m.part.role, m.port.id, role, port.id]; if (!isConnected(e)) { this.connect(e[0], e[1], e[2], e[3]); } } } ref1 = participant.outports; // outbound results1 = []; for (l = 0, len2 = ref1.length; l < len2; l++) { port = ref1[l]; matches = findConnectedPorts('inports', port); results1.push((function() { var len3, n, results2; results2 = []; for (n = 0, len3 = matches.length; n < len3; n++) { m = matches[n]; e = [role, port.id, m.part.role, m.port.id]; if (!isConnected(e)) { results2.push(this.connect(e[0], e[1], e[2], e[3])); } else { results2.push(void 0); } } return results2; }).call(this)); } return results1; } else if (action === 'removed') { return null; // TODO: implement } else { return null; // ignored } } addInitial(partId, portId, data, metadata, callback) { var id; if (typeof metadata === 'function') { callback = metadata; metadata = {}; } if (!metadata) { metadata = {}; } id = iipId(partId, portId); this.iips[id] = { data: data, metadata: metadata }; return waitForParticipant(this, partId, (err) => { if (err) { return callback(err); } if (this.started) { return this.sendTo(partId, portId, data, function(err) { return callback(err); }); } else { return callback(null); } }); } removeInitial(partId, portId) {} // FIXME: implement // Do we need to remove it from the queue?? exportPort(direction, external, node, internal, metadata, callback) { var graph, target; if (typeof metadata === 'function') { callback = metadata; metadata = {}; } if (!metadata) { metadata = {}; } target = direction.indexOf("in") === 0 ? this.exported.inports : this.exported.outports; target[external] = { role: node, port: internal, subscriber: null, queue: null }; graph = null; // FIXME: capture // Wait for target node to exist return waitForParticipant(this, node, (err) => { var handler; if (err) { return callback(err); } if (direction.indexOf('out') === 0) { handler = (msg) => { return this.emit('exported-port-data', external, msg.data, this.graphName); }; return this.subscribeTo(node, internal, handler, function(err, readQueue) { if (err) { return callback(err); } target[external].subscriber = handler; target[external].queue = readQueue; return callback(null); }); } else { return callback(null); } }); } unexportPort() {} // FIXME: implement sendToExportedPort(port, data, callback) { var internal; // FIXME lookup which node, port this corresponds to internal = this.exported.inports[port]; debug('sendToExportedPort', port, internal); if (!internal) { return callback(new Error(`Cannot find exported port ${port}`)); } return this.sendTo(internal.role, internal.port, data, callback); } startNetwork(networkId, callback) { // Don't have a concept of started/stopped so far, no-op return setTimeout(callback, 10); } stopNetwork(networkId, callback) { // Don't have a concept of started/stopped so far, no-op return setTimeout(callback, 10); } _onConnectionData(binding, data) { var connection; boundMethodCheck(this, Coordinator); connection = connectionFromBinding(this.participants, binding); connection.graph = this.graphName; return this.emit('connection-data', connection, data); } clearSubscriptions(callback) { return this.broker.listSubscriptions((err, subs) => { if (err) { return callback(err); } return async.map(subs, (sub, cb) => { return this.broker.unsubscribeData(sub, this._onConnectionData, cb); }, callback); }); } subscribeConnection(fromRole, fromPort, toRole, toPort, callback) { return waitForParticipant(this, fromRole, (err) => { if (err) { return callback(err); } return waitForParticipant(this, toRole, (err) => { var binding; if (err) { return callback(err); } binding = { src: findQueue(this.participants, fromRole, 'outports', fromPort), tgt: findQueue(this.participants, toRole, 'inports', toPort) }; return this.broker.subscribeData(binding, this._onConnectionData, callback); }); }); } unsubscribeConnection(fromRole, fromPort, toRole, toPort, callback) { return waitForParticipant(this, fromRole, (err) => { if (err) { return callback(err); } return waitForParticipant(this, toRole, (err) => { var binding; if (err) { return callback(err); } binding = { src: findQueue(this.participants, fromRole, 'outports', fromPort), tgt: findQueue(this.participants, toRole, 'inports', toPort) }; this.broker.unsubscribeData(binding, this._onConnectionData, callback); return callback(null); }); }); } serializeGraph(name) { var conn, connectionIds, edge, graph, i, id, iip, iipIds, j, l, len, len1, len2, node, nodeNames, parts; graph = { properties: { name: name, environment: { type: 'msgflo' } }, processes: {}, connections: [], inports: [], outports: [] }; nodeNames = Object.keys(this.nodes).sort(); for (i = 0, len = nodeNames.length; i < len; i++) { name = nodeNames[i]; node = this.nodes[name]; graph.processes[name] = { component: node.component, metadata: node.metadata || {} }; } connectionIds = Object.keys(this.connections).sort(); for (j = 0, len1 = connectionIds.length; j < len1; j++) { id = connectionIds[j]; conn = this.connections[id]; parts = fromConnId(id); edge = { src: { process: parts[0], port: parts[1] }, tgt: { process: parts[2], port: parts[3] }, metadata: this.connections[id].metadata }; graph.connections.push(edge); } iipIds = Object.keys(this.iips).sort(); for (l = 0, len2 = iipIds.length; l < len2; l++) { id = iipIds[l]; iip = this.iips[id]; parts = fromIipId(id); edge = { data: iip.data, tgt: { process: parts[0], port: parts[1] }, metadata: iip.metadata }; graph.connections.push(edge); } return graph; } loadGraphFile(path, opts, callback) { var availableComponents, k, options, rolesNoComponent, rolesWithComponent, v; debug('loadGraphFile', path); options = { graphfile: path, libraryfile: this.library.configfile }; for (k in opts) { v = opts[k]; options[k] = v; } // Avoid trying to instantiate // Probably these are external participants, which *should* be running // TODO: check whether the participants do indeed show up rolesWithComponent = []; rolesNoComponent = []; availableComponents = Object.keys(this.library.components); return common.readGraph(options.graphfile, (err, graph) => { var componentName, i, len, process, ref, ref1, role, rolesToSetup, setupConnections, setupParticipants; if (err) { return callback(err); } ref = graph.processes; for (role in ref) { process = ref[role]; if (ref1 = process.component, indexOf.call(availableComponents, ref1) >= 0) { rolesWithComponent.push(role); } else { rolesNoComponent.push(role); } } if (rolesNoComponent.length) { console.log('Skipping setup for participants without component available. Assuming already setup:'); } for (i = 0, len = rolesNoComponent.length; i < len; i++) { role = rolesNoComponent[i]; componentName = graph.processes[role].component; console.log(`\t${role}(${componentName})`); } rolesToSetup = rolesWithComponent.concat([]).filter(function(r) { return indexOf.call(options.ignore, r) < 0; }); options.only = rolesToSetup; setupParticipants = (setupCallback) => { var participantStartConcurrency; participantStartConcurrency = 10; return async.mapLimit(options.only, participantStartConcurrency, (role, cb) => { var metadata; componentName = graph.processes[role].component; metadata = graph.processes[role].metadata || {}; return this.startParticipant(role, componentName, metadata, cb); }, setupCallback); }; setupConnections = (setupCallback) => { return async.map(graph.connections, (c, cb) => { if (c.data) { return this.addInitial(c.tgt.process, c.tgt.port, c.data, c.metadata, cb); } else { return this.connect(c.src.process, c.src.port, c.tgt.process, c.tgt.port, c.metadata, cb); } }, setupCallback); }; return async.parallel({ connections: setupParticipants, participants: setupConnections }, function(err, results) { if (err) { return callback(err); } return callback(null); }); }); } }; exports.Coordinator = Coordinator;