noderank/lib/npmstats.js

run pagerank algorithm on npm metadata to discover the most popular module

var request = require('request');
var Q = require('q');
var fs = require('fs');
var async = require('async');
var Registry = require('npm-registry');
var debug = require('debug')('noderank:npmstats');
/**
 * retrieve NPM statistics
 * @class
 * @param {object} options optional hash with class options like registry url
 */
function NpmStats(options) {
  if (options && options.url) {
    this.url = options.url;
  } else {
    this.url = 'https://skimdb.npmjs.com/registry/_all_docs';
  }
  this.allModules = {};
}

NpmStats.prototype.getAllModules = function() {
  var getAllModulesPromise = Q.defer();

  debug('requesting modules from %s', this.url);

  request.get(this.url, function getAllModulesRequestCallback(err, msg, body) {
    var response;
    var modules = [];
    if (err) {
      debug('error getting all projects: %j', err);
      return getAllModulesPromise.reject(new Error(err));
    }
    debug('parsing response');
    response = JSON.parse(body);
    debug('body length: %d', response.rows.length);
    response.rows.forEach(function forEachRow(row) {
      modules.push(row.key);
    });

    getAllModulesPromise.resolve(modules);
  });
  return getAllModulesPromise.promise;
};

NpmStats.prototype.get = function(module) {
  var self = this;
  var _module = module;
  var getPromise = Q.defer();

  this.npm = new Registry({retries: 5});
  debug('getting %s', _module);
  this.npm.packages.get(_module, function getResponse(err, response) {
    debug('err %j data %j', err, response);
    if (err) {
      debug('error getting %s: %j', _module, err);
      return getPromise.reject(new Error(err));
    }
    self.allModules[_module] = response[0];
    debug('setting response to %j', self.allModules[_module]);
    getPromise.resolve(response[0]);
  });
  return getPromise.promise;
};

NpmStats.prototype.getDependencies = function(module) {
  var getDependenciesPromise = Q.defer();
  debug('trying to get latest dependencies for  %s', module);
  var self = this;

  async.retry(5, function retryGetDependencies(retry) {
    debug('getting dependencies for  %s', module);
    request.get('http://registry.npmjs.org/' + module + '/latest', function getDependenciesCallback(err, msg, body) {
      var dependencies = [];
      var response;
      if (err) {
        debug('error getting dependencies for  %s: %j', module, err);
        debug('retrying %s', module);
        return retry(err);
      }
      try {
        debug('parsing response %s', body);
        response = JSON.parse(body);
        self.allModules[module] = response;
        debug('dependencies for  %s: %j', module, response.dependencies);
        if (response.dependencies) {
          dependencies = dependencies.concat(Object.keys(response.dependencies));
        }
        if (response.peerDependencies) {
          dependencies = dependencies.concat(Object.keys(response.peerDependencies));
        }
        if (response.devDependencies) {
          dependencies = dependencies.concat(Object.keys(response.devDependencies));
        }
        if (response.optionalDependencies) {
          dependencies = dependencies.concat(Object.keys(response.optionalDependencies));
        }
        for (var i = 0; i < dependencies.length; i++) {
          dependencies[i] = dependencies[i];
        }
        return retry(null, dependencies);
      } catch (e) {
        debug('retrying', body);
        return retry(e);
      }
    });
  }, function onFinishedRetrying(err, results) {
    if (err) {
      debug('error getting dependencies %j %j', err.message, results);
      return getDependenciesPromise.reject(new Error(err));
    }
    debug('resolving dependencies %j', results);
    return getDependenciesPromise.resolve(results);
  });

  return getDependenciesPromise.promise;
};

NpmStats.prototype.buildDependencyGraph = function() {
  var buildDependencyGraphPromise = Q.defer();
  var self = this;

  debug('getting all modules');
  debug('self.getDependencies %j', self.getDependencies);
  self.getAllModules()
    .then(function onGetAllModulesResolved(modules) {
      var dependencyGraph = {};
      debug('modules resolved %j', modules);
      debug('self.getDependencies %j', self.getDependencies);
      async.eachLimit(modules, 10, function getDependenciesforModule(module, nextModule) {
        var _module = module;
        debug('getting dependencies for  %s', _module);
        self.getDependencies(_module)
          .then(function onGetDependenciesResolved(dependencies) {
            debug('setting dependencies for  %s to %j', _module, dependencies);
            debug('allModules[%s] %j', _module, self.allModules[_module]);
            dependencyGraph[_module] = dependencies;
            nextModule();
          }, function onGetDependenciesRejected(err) {
            debug('error getting dependencies for  %s: %j', module, err);
            nextModule(err);
          });
      }, function onAllDependenciesGotten(err) {
        debug('all dependencies gotten %j', dependencyGraph);
        if (err) {
          debug('error getting dependencies: %j', err);
          buildDependencyGraphPromise.reject(new Error(err));
        } else {
          debug('writing to file');
          fs.writeFileSync('dependency_graph.json', JSON.stringify(dependencyGraph));
          buildDependencyGraphPromise.resolve(dependencyGraph);
        }
      });
    }, function onGetAllModulesRejected(err) {
      debug('error getting modules: %j', err);
      buildDependencyGraphPromise.reject(new Error(err));
    });
  return buildDependencyGraphPromise.promise;
};

NpmStats.prototype.normalizeDependencyGraph = function(dgraph, token) {
  var module;
  var dep;

  if (dgraph === undefined) {
    throw new Error('missing required deependency graph argument');
  }
  if (token === undefined) {
    token = '#';
  }
  // add the token to each key and to each dependency
  for (module in dgraph) {
    if (dgraph.hasOwnProperty(module)) {
      for (dep in dgraph[module]) {
        if (dgraph[module].hasOwnProperty(dep)) {
          dgraph[module][dep] = token + dgraph[module][dep];
        }
      }
    }
    dgraph[token + module] = dgraph[module];
    delete dgraph[module];
  }
  return dgraph;
};

module.exports = NpmStats;