msgflo/lib/setup.js

Polyglot FBP runtime based on message queues

var Library, addBindings, async, bindingsFromDefinitions, child_process, common, debug, discoverParticipantQueues, fbp, graphBindings, main, normalize, parse, participantCommands, path, pretty, program, queueName, setupBindings, setupParticipants, startProcess, startProcesses, staticGraphBindings, transport,
  indexOf = [].indexOf;

common = require('./common');

({Library} = require('./library'));

transport = require('msgflo-nodejs').transport;

fbp = require('fbp');

async = require('async');

debug = require('debug')('msgflo:setup');

child_process = require('child_process');

path = require('path');

program = require('commander');

addBindings = function(broker, bindings, callback) {
  var addBinding;
  addBinding = function(b, cb) {
    return broker.addBinding(b, cb);
  };
  return async.map(bindings, addBinding, callback);
};

queueName = function(c) {
  return common.queueName(c.process, c.port);
};

startProcess = function(cmd, options, callback) {
  var args, child, childoptions, env, execCmd, prog, returned;
  env = process.env;
  env['MSGFLO_BROKER'] = options.broker;
  if (options.shell) {
    prog = options.shell;
    args = ['-c', cmd];
  } else {
    prog = cmd.split(' ')[0];
    args = cmd.split(' ').splice(1);
  }
  execCmd = prog + ' ' + args.join(' ');
  childoptions = {
    env: env
  };
  debug('participant process start', execCmd);
  child = child_process.spawn(prog, args, childoptions);
  returned = false;
  child.on('error', function(err) {
    debug('participant error', prog, args.join(' '));
    if (returned) {
      return;
    }
    returned = true;
    return callback(err, child);
  });
  // We assume that when somethis is send on stdout, starting is complete
  child.stdout.on('data', function(data) {
    if (options.forward.indexOf('stdout') !== -1) {
      process.stdout.write(data.toString());
    }
    debug('participant stdout', data.toString());
    if (returned) {
      return;
    }
    returned = true;
    return callback(null, child);
  });
  child.stderr.on('data', function(data) {
    if (options.forward.indexOf('stderr') !== -1) {
      process.stderr.write(data.toString());
    }
    return debug('participant stderr', data.toString());
  });
  //return if returned
  //returned = true
  //return callback new Error data.toString(), child
  child.on('exit', function(code, signal) {
    debug('child exited', code, signal);
    if (returned) {
      return;
    }
    returned = true;
    return callback(new Error(`Child '${execCmd}' (pid=${child.pid}) exited with ${code} ${signal}`));
  });
  return child;
};

participantCommands = function(graph, library, only, ignore) {
  var cmd, commands, component, i, iips, isParticipant, len, name, participants;
  isParticipant = function(name) {
    return common.isParticipant(graph.processes[name]);
  };
  commands = {};
  participants = Object.keys(graph.processes);
  if (only != null) {
    participants = only;
  }
  participants = participants.filter(isParticipant);
  for (i = 0, len = participants.length; i < len; i++) {
    name = participants[i];
    if (ignore.indexOf(name) !== -1) {
      continue;
    }
    component = graph.processes[name].component;
    iips = common.iipsForRole(graph, name);
    cmd = library.componentCommand(component, name, iips);
    commands[name] = cmd;
  }
  return commands;
};

exports.startProcesses = startProcesses = function(commands, options, callback) {
  var names, start;
  start = (name, cb) => {
    var cmd;
    cmd = commands[name];
    return startProcess(cmd, options, function(err, child) {
      if (err) {
        return cb(err);
      }
      return cb(err, {
        name: name,
        command: cmd,
        child: child
      });
    });
  };
  debug('starting participants', Object.keys(commands));
  names = Object.keys(commands);
  return async.map(names, start, function(err, processes) {
    var i, len, p, processMap;
    if (err) {
      return callback(err);
    }
    processMap = {};
    for (i = 0, len = processes.length; i < len; i++) {
      p = processes[i];
      processMap[p.name] = p.child;
    }
    return callback(null, processMap);
  });
};

exports.killProcesses = function(processes, signal, callback) {
  var kill, names, pids;
  if (!processes) {
    return callback(null);
  }
  if (!signal) {
    signal = 'SIGTERM';
  }
  kill = function(name, cb) {
    var child;
    child = processes[name];
    if (!child) {
      return cb(null);
    }
    child.once('exit', function(code, signal) {
      return cb(null);
    });
    return child.kill(signal);
  };
  pids = Object.keys(processes).map(function(n) {
    return processes[n].pid;
  });
  debug('killing participants', pids);
  names = Object.keys(processes);
  return async.map(names, kill, callback);
};

// Extact the queue bindings, including types from an FBP graph definition
exports.graphBindings = graphBindings = function(graph) {
  var binding, bindings, conn, i, isParticipant, len, n, name, process, ref, ref1, ref2, ref3, roundRobinNames, roundRobins;
  bindings = [];
  roundRobins = {};
  ref = graph.processes;
  for (name in ref) {
    process = ref[name];
    if (process.component === 'msgflo/RoundRobin') {
      roundRobins[name] = {
        type: 'roundrobin'
      };
    }
  }
  isParticipant = function(node) {
    return common.isParticipant(graph.processes[node]);
  };
  roundRobinNames = Object.keys(roundRobins);
  ref1 = graph.connections;
  for (i = 0, len = ref1.length; i < len; i++) {
    conn = ref1[i];
    if (conn.data) {

    // IIP
    } else if (ref2 = conn.src.process, indexOf.call(roundRobinNames, ref2) >= 0) {
      binding = roundRobins[conn.src.process];
      if (conn.src.port === 'deadletter') {
        binding.deadletter = queueName(conn.tgt);
      } else {
        binding.tgt = queueName(conn.tgt);
      }
    } else if (ref3 = conn.tgt.process, indexOf.call(roundRobinNames, ref3) >= 0) {
      binding = roundRobins[conn.tgt.process];
      binding.src = queueName(conn.src);
    } else if (isParticipant(conn.tgt.process) && isParticipant(conn.src.process)) {
      // ordinary connection
      bindings.push({
        type: 'pubsub',
        src: queueName(conn.src),
        tgt: queueName(conn.tgt)
      });
    } else {
      debug('no binding for', queueName(conn.src, queueName(conn.tgt)));
    }
  }
  for (n in roundRobins) {
    binding = roundRobins[n];
    bindings.push(binding);
  }
  return bindings;
};

staticGraphBindings = function(broker, graph, callback) {
  var bindings;
  bindings = graphBindings(graph);
  return callback(null, bindings);
};

// callbacks with bindings
discoverParticipantQueues = function(broker, graph, options, callback) {
  var foundParticipantsByRole;
  foundParticipantsByRole = {};
  if (typeof broker === 'string') {
    broker = transport.getBroker(broker);
  }
  return broker.connect(function(err) {
    var addParticipant, onTimeout, timeout;
    if (err) {
      return callback(err);
    }
    addParticipant = function(definition) {
      var found, foundRoles, i, idx, len, missingRoles, name, proc, ref, wanted, wantedRoles;
      foundParticipantsByRole[definition.role] = definition;
      // determine if we are still lacking any definitions
      wantedRoles = [];
      ref = graph.processes;
      for (name in ref) {
        proc = ref[name];
        wantedRoles.push(name);
      }
      foundRoles = Object.keys(foundParticipantsByRole);
      missingRoles = [];
      for (i = 0, len = wantedRoles.length; i < len; i++) {
        wanted = wantedRoles[i];
        idx = foundRoles.indexOf(wanted);
        found = idx !== -1;
        if (!found) {
          //console.log 'found?', wanted, found
          missingRoles.push(wanted);
        }
      }
      debug('addparticipant', definition.role, missingRoles);
      //console.log 'wanted, found, missing', wantedRoles, foundRoles, missingRoles
      if (missingRoles.length === 0) {
        if (!callback) {
          return;
        }
        callback(null, foundParticipantsByRole);
        callback = null;
      }
    };
    onTimeout = () => {
      if (!callback) {
        return;
      }
      callback(new Error('setup: Participant discovery timed out'));
      callback = null;
    };
    timeout = setTimeout(onTimeout, options.timeout);
    return broker.subscribeParticipantChange((msg) => {
      var data;
      data = msg.data;
      if (data.protocol === 'discovery' && data.command === 'participant') {
        addParticipant(data.payload);
        return broker.ackMessage(msg);
      } else {
        broker.nackMessage(msg);
        throw new Error('Unknown FBP message');
      }
    });
  });
};

bindingsFromDefinitions = function(graph, definitions) {
  var bindings, conn, findQueue, i, isParticipant, len, ref, srcDef, srcQueue, tgtDef, tgtQueue;
  bindings = [];
  isParticipant = function(node) {
    return common.isParticipant(graph.processes[node]);
  };
  findQueue = function(ports, portname) {
    var found, i, len, port;
    found = null;
    for (i = 0, len = ports.length; i < len; i++) {
      port = ports[i];
      //console.log 'port', port.id, portname, port.id == portname, port.queue
      if (port.id === portname) {
        found = port.queue;
      }
    }
    if (!found) {
      throw new Error(`Could not find ${portname} in ${JSON.stringify(ports)}`);
    }
    return found;
  };
  ref = graph.connections;
  for (i = 0, len = ref.length; i < len; i++) {
    conn = ref[i];
    if (!conn.src) { // IIP
      continue;
    }
    if (isParticipant(conn.tgt.process) && isParticipant(conn.src.process)) {
      // ordinary connection
      srcDef = definitions[conn.src.process];
      tgtDef = definitions[conn.tgt.process];
      srcQueue = findQueue(srcDef.outports, conn.src.port);
      tgtQueue = findQueue(tgtDef.inports, conn.tgt.port);
      console;
      bindings.push({
        type: 'pubsub',
        src: srcQueue,
        tgt: tgtQueue
      });
    } else {
      debug('no binding for', queueName(conn.src, queueName(conn.tgt)));
    }
  }
  return bindings;
};

exports.normalizeOptions = normalize = function(options) {
  common.normalizeOptions(options);
  if (!options.libraryfile) {
    options.libraryfile = path.join(process.cwd(), 'package.json');
  }
  if (typeof options.only === 'string' && options.only) {
    options.only = options.only.split(',');
  }
  if (typeof options.ignore === 'string' && options.ignore) {
    options.ignore = options.ignore.split(',');
  }
  if (typeof options.forward === 'string' && options.forward) {
    options.forward = options.forward.split(',');
  }
  if (!options.only) {
    options.only = null;
  }
  if (!options.ignore) {
    options.ignore = [];
  }
  if (!options.forward) {
    options.forward = [];
  }
  if (!options.extrabindings) {
    options.extrabindings = [];
  }
  if (options.discover == null) {
    options.discover = false;
  }
  if (options.forever == null) {
    options.forever = false;
  }
  if (options.timeout == null) {
    options.timeout = 10000;
  }
  return options;
};

exports.bindings = setupBindings = function(options, callback) {
  var getBindings;
  options = normalize(options);
  getBindings = staticGraphBindings; // use queue name convention, read directly from graph file
  if (options.discover) {
    // wait for FBP discovery messsages, use queues from there
    getBindings = function(broker, graph, cb) {
      return discoverParticipantQueues(options.broker, graph, options, function(err, defs) {
        var bindings;
        if (err) {
          return cb(err);
        }
        //console.log 'got defs', definitions
        bindings = bindingsFromDefinitions(graph, defs);
        return cb(null, bindings);
      });
    };
  }
  if (!callback) {
    throw new Error('ffss');
  }
  return common.readGraph(options.graphfile, function(err, graph) {
    var broker;
    if (err) {
      return callback(err);
    }
    broker = transport.getBroker(options.broker);
    return broker.connect(function(err) {
      if (err) {
        return callback(err);
      }
      return getBindings(broker, graph, function(err, bindings) {
        if (err) {
          return callback(err);
        }
        bindings = bindings.concat(options.extrabindings);
        return addBindings(broker, bindings, function(err) {
          return callback(err, bindings, graph);
        });
      });
    });
  });
};

exports.participants = setupParticipants = function(options, callback) {
  options = normalize(options);
  return common.readGraph(options.graphfile, function(err, graph) {
    var lib;
    if (err) {
      return callback(err);
    }
    lib = new Library({
      configfile: options.libraryfile,
      componentdir: options.componentdir
    });
    return lib.load(function(err) {
      var commands;
      if (err) {
        return callback(err);
      }
      commands = participantCommands(graph, lib, options.only, options.ignore);
      return startProcesses(commands, options, callback);
    });
  });
};

exports.parse = parse = function(args) {
  var graph;
  graph = null;
  program.arguments('<graph.fbp/.json>').option('--broker <URL>', 'URL of broker to connect to', String, null).option('--participants [BOOL]', 'Also set up participants, not just bindings', Boolean, false).option('--only <one,two,three>', 'Only set up participants for these roles', String, '').option('--ignore <one,two,three>', 'Do not set up participants for these roles', String, null).option('--library <FILE.json>', 'Library definition to use', String, 'package.json').option('--componentdir <DIR>', 'Directory where components are located', String, '').option('--forward [stderr,stdout]', 'Forward child process stdout and/or stderr', String, '').option('--discover [BOOL]', 'Whether to wait for FBP discovery messages for queue info', Boolean, false).option('--timeout <SECONDS>', 'How long to wait for discovery messages', Number, 30).option('--forever [BOOL]', 'Keep running until killed by signal', Boolean, false).option('--shell [shell]', 'Run participant commands in a shell', String, '').action(function(gr, env) {
    return graph = gr;
  }).parse(args);
  program.libraryfile = program.library;
  program.graphfile = graph;
  program.timeout = program.timeout * 1000;
  return program;
};

exports.prettyFormatBindings = pretty = function(bindings) {
  var b, i, len, lines, type;
  lines = [];
  for (i = 0, len = bindings.length; i < len; i++) {
    b = bindings[i];
    type = b.type.toUpperCase();
    if (b.type === 'roundrobin') {
      if (b.tgt && b.deadletter) {
        lines.push(`DEADLETTER:\t ${b.tgt} -> ${b.deadletter}`);
      }
      if (b.tgt && b.src) {
        lines.push(`ROUNDROBIN:\t ${b.src} -> ${b.tgt}`);
      }
    } else if (b.type === 'pubsub') {
      lines.push(`PUBSUB: \t ${b.src} -> ${b.tgt}`);
    } else {
      lines.push(`UNKNOWN binding type: ${b.type}`);
    }
  }
  return lines.join('\n');
};

exports.main = main = function() {
  var maybeSetupParticipants, options;
  options = parse(process.argv);
  if (!options.graphfile) {
    console.error('ERROR: No graph file specified');
    program.help();
    process.exit();
  }
  maybeSetupParticipants = function(options, callback) {
    return callback(null, {});
  };
  if (options.participants) {
    maybeSetupParticipants = setupParticipants;
  }
  return maybeSetupParticipants(options, function(err, p) {
    if (err) {
      throw err;
    }
    console.log('Set up participants', Object.keys(p));
    return setupBindings(options, function(err, bindings) {
      if (err) {
        throw err;
      }
      console.log('Set up bindings:\n', pretty(bindings));
      if (options.forever) {
        console.log('--forever enabled, keeping alive');
        return setInterval(function() {
          return null; // just keep alive
        }, 1000);
      } else {
        return process.exit(0);
      }
    });
  });
};