msgflo/lib/library.js

Polyglot FBP runtime based on message queues

var EventEmitter, Library, baseComponentCommand, cleanComponentDefinition, common, componentCommandForFile, componentsFromConfig, componentsFromDirectory, debug, defaultHandlers, ext, extensionToLanguage, fs, lang, languageExtensions, normalizeConfig, path, replaceMarker, replaceVariables, yaml,
  indexOf = [].indexOf;

fs = require('fs');

path = require('path');

yaml = require('js-yaml');

debug = require('debug')('msgflo:library');

EventEmitter = require('events').EventEmitter;

common = require('./common');

defaultHandlers = {
  ".yml": "msgflo-register --role #ROLE:#FILENAME",
  ".js": "msgflo-nodejs --name #ROLE #FILENAME",
  ".coffee": "msgflo-nodejs --name #ROLE #FILENAME",
  ".py": "msgflo-python #FILENAME #ROLE",
  ".json": "noflo-runtime-msgflo --name #ROLE --graph #COMPONENT --iips #IIPS",
  ".fbp": "noflo-runtime-msgflo --name #ROLE --graph #COMPONENT --iips #IIPS"
};

languageExtensions = {
  'python': 'py',
  'coffeescript': 'coffee',
  'javascript': 'js',
  'c++': 'cpp',
  'rust': 'rs',
  'yaml': 'yml'
};

extensionToLanguage = {};

for (lang in languageExtensions) {
  ext = languageExtensions[lang];
  extensionToLanguage[`.${ext}`] = lang;
}

replaceMarker = function(str, marker, value) {
  marker = '#' + marker.toUpperCase();
  return str.replace(new RegExp(marker, 'g'), value);
};

exports.replaceVariables = replaceVariables = function(str, variables) {
  var marker, value;
  for (marker in variables) {
    value = variables[marker];
    str = replaceMarker(str, marker, value);
  }
  return str;
};

baseComponentCommand = function(config, component, cmd, filename) {
  var componentName, variables;
  variables = common.clone(config.variables);
  componentName = component.split('/')[1];
  if (!componentName) {
    componentName = component;
  }
  if (filename) {
    variables['FILENAME'] = filename;
  }
  variables['COMPONENTNAME'] = componentName;
  variables['COMPONENT'] = component;
  return replaceVariables(cmd, variables);
};

componentCommandForFile = function(config, filename) {
  var cmd, component;
  ext = path.extname(filename);
  component = path.basename(filename, ext);
  cmd = config.handlers[ext];
  return baseComponentCommand(config, component, cmd, filename);
};

componentsFromConfig = function(config) {
  var cmd, component, components, ref;
  components = {};
  ref = config.components;
  for (component in ref) {
    cmd = ref[component];
    components[component] = {
      language: null, // XXX: Could try to guess from cmd/template??
      command: baseComponentCommand(config, component, cmd)
    };
  }
  return components;
};

componentsFromDirectory = function(directory, config, callback) {
  var components, extensions;
  components = {};
  extensions = Object.keys(config.handlers);
  return fs.exists(directory, function(exists) {
    if (!exists) {
      return callback(null, {});
    }
    return fs.readdir(directory, function(err, filenames) {
      var component, filename, filepath, i, len, supported, unsupported;
      if (err) {
        return callback(err);
      }
      supported = filenames.filter(function(f) {
        var ref;
        return ref = path.extname(f), indexOf.call(extensions, ref) >= 0;
      });
      unsupported = filenames.filter(function(f) {
        var ref;
        return !(ref = path.extname(f), indexOf.call(extensions, ref) >= 0);
      });
      if (unsupported.length) {
        debug('unsupported component files', unsupported);
      }
      for (i = 0, len = supported.length; i < len; i++) {
        filename = supported[i];
        ext = path.extname(filename);
        lang = extensionToLanguage[ext];
        component = path.basename(filename, ext);
        if (config.namespace) {
          component = `${config.namespace}/${component}`;
        }
        debug('loading component from file', filename, component);
        filepath = path.join(directory, filename);
        components[component] = {
          language: lang,
          command: componentCommandForFile(config, filepath)
        };
      }
      return callback(null, components);
    });
  });
};

normalizeConfig = function(config) {
  var k, namespace, ref, repository, v;
  if (!config) {
    config = {};
  }
  namespace = config.name || null;
  repository = config.repository;
  if ((ref = config.repository) != null ? ref.url : void 0) {
    // package.json convention
    repository = config.repository.url;
  }
  if (config.msgflo) { // Migth be under a .msgflo key, for instance in package.json
    config = config.msgflo;
  }
  if (typeof config.repository !== 'string') {
    config.repository = repository;
  }
  if (config.namespace == null) {
    config.namespace = namespace;
  }
  if (!config.components) {
    config.components = {};
  }
  if (!config.variables) {
    config.variables = {};
  }
  if (!config.handlers) {
    config.handlers = {};
  }
  for (k in defaultHandlers) {
    v = defaultHandlers[k];
    if (!config.handlers[k]) {
      config.handlers[k] = defaultHandlers[k];
    }
  }
  return config;
};

// Remove instance-specific data like role and extra from library data
cleanComponentDefinition = function(discovered) {
  var component, i, j, len, len1, port, ref, ref1;
  if (!(discovered != null ? discovered.definition : void 0)) {
    return discovered;
  }
  // Start by cloning the definition
  component = common.clone(discovered);
  if (!(component != null ? component.definition : void 0)) {
    return component;
  }
  delete component.definition.extra;
  delete component.definition.id;
  delete component.definition.role;
  ref = component.definition.inports;
  for (i = 0, len = ref.length; i < len; i++) {
    port = ref[i];
    delete port.queue;
  }
  ref1 = component.definition.outports;
  for (j = 0, len1 = ref1.length; j < len1; j++) {
    port = ref1[j];
    delete port.queue;
  }
  return component;
};

Library = class Library extends EventEmitter {
  constructor(options) {
    super();
    if (options.configfile) {
      options.config = JSON.parse(fs.readFileSync(options.configfile, 'utf-8'));
    }
    if (!options.componentdir) {
      console.log('WARNING:', 'Default components directory for MsgFlo will change to "components" in next release');
      options.componentdir = 'participants';
    }
    options.config = normalizeConfig(options.config);
    this.options = options;
    this.components = {}; // "name" -> { command: "", language: ''|null }.  lazy-loaded using load()
  }

  getComponent(name) {
    var withNamespace, withoutNamespace;
    if (this.components[name]) {
      // Direct match
      return this.components[name];
    }
    withoutNamespace = path.basename(name);
    if (this.components[withoutNamespace]) {
      return this.components[withoutNamespace];
    }
    if (name.indexOf('/' === -1 && this.options.config.namespace)) {
      withNamespace = this.options.config.namespace + '/' + name;
      if (this.components[withNamespace]) {
        return this.components[withNamespace];
      }
    }
    return null;
  }

  _updateComponents(components) {
    var comp, discovered, existing, k, name, names, v;
    names = [];
    for (name in components) {
      comp = components[name];
      if (!comp) {
        // removed
        this.components[name] = null;
        if (names.indexOf(name) === -1) {
          names.push(name);
        }
        continue;
      }
      discovered = cleanComponentDefinition(comp);
      existing = this.getComponent(name);
      if (!existing) {
        // added
        this.components[name] = discovered;
        if (names.indexOf(name) === -1) {
          names.push(name);
        }
        continue;
      }
      if (JSON.stringify(existing.definition) !== JSON.stringify(discovered.definition)) {
// updated
        for (k in discovered) {
          v = discovered[k];
          this.components[name][k] = v;
        }
        if (names.indexOf(name) === -1) {
          names.push(name);
        }
        continue;
      }
    }
    // Send components-changed only if something changed
    if (names.length) {
      return this.emit('components-changed', names, this.components);
    }
  }

  load(callback) {
    return componentsFromDirectory(this.options.componentdir, this.options.config, (err, components) => {
      if (err) {
        return callback(err);
      }
      this._updateComponents(components);
      this._updateComponents(componentsFromConfig(this.options.config));
      return callback(null);
    });
  }

  // call when MsgFlo discovery message has come in
  _updateDefinition(name, def) {
    var changes;
    if (!def) { // Ignore participants being removed
      return;
    }
    changes = {};
    changes[name] = {
      definition: def
    };
    return this._updateComponents(changes);
  }

  getSource(name, callback) {
    var basename, component, filename, library, ref, ref1, source;
    debug('requesting component source', name);
    component = this.getComponent(name);
    if (!component) {
      return callback(new Error(`Component not found for ${name}`));
    }
    basename = name;
    library = null;
    if (name.indexOf('/') !== -1) {
      // FBP protocol component:getsource unfortunately bakes in library in this case
      [library, basename] = name.split('/');
    } else if (((ref = this.options.config) != null ? ref.namespace : void 0) != null) {
      library = (ref1 = this.options.config) != null ? ref1.namespace : void 0;
    }
    if (!component.language) {
      // Component that doesn't come from handlers, send discovery info since source isn't available
      debug('component without source', name, component.command);
      source = {
        name: basename,
        library: library,
        code: yaml.safeDump(component.definition || {}),
        language: 'discovery'
      };
      return callback(null, source);
    }
    lang = component.language;
    ext = languageExtensions[lang];
    filename = path.join(this.options.componentdir, `${basename}.${ext}`);
    return fs.readFile(filename, 'utf-8', function(err, code) {
      debug('component source file', filename, lang, err);
      if (err) {
        return callback(new Error(`Could not find component source for ${name}: ${err.message}`));
      }
      source = {
        name: basename,
        library: library,
        code: code,
        language: component.language
      };
      return callback(null, source);
    });
  }

  addComponent(name, language, code, callback) {
    var filename, ref;
    debug('adding component', name, language);
    ext = languageExtensions[language];
    ext = ext || language; // default to input lang for open-ended extensibility
    filename = path.join(this.options.componentdir, `${path.basename(name)}.${ext}`);
    if (name.indexOf('/') === -1 && ((ref = this.options.config) != null ? ref.namespace : void 0)) {
      name = `${this.options.config.namespace}/${name}`;
    }
    return fs.writeFile(filename, code, (err) => {
      var changes;
      if (err) {
        return callback(err);
      }
      changes = {};
      changes[name] = {
        language: language,
        command: componentCommandForFile(this.options.config, filename)
      };
      this._updateComponents(changes);
      return callback(null);
    });
  }

  componentCommand(component, role, iips = {}) {
    var cmd, ref, vars;
    cmd = (ref = this.getComponent(component)) != null ? ref.command : void 0;
    if (!cmd) {
      throw new Error(`No component ${component} defined for role ${role}`);
    }
    vars = {
      'ROLE': role,
      'IIPS': `'${JSON.stringify(iips)}'`
    };
    cmd = replaceVariables(cmd, vars);
    return cmd;
  }

};

exports.Library = Library;