npm-package-resolver
Version:
An API for Resolving NPM package dependencies
277 lines (185 loc) • 7.47 kB
JavaScript
/**
* Created by Liam Bilich on 07/03/2016.
*
* PAckage resolver class, re recurse pacakges here and connect oredis
*/
var util = require('util');
var _config = require('./config.js');
var config = process.env['NODE_ENV']=="production"?_config.production:_config.development;
var redis = require('./redis-transport.js');
var request = require('request');
var _ = require('underscore');
var async = require('async');
var logger = require('./logger.js')('package-resolver');
var moment = require('moment');
var log = logger.log;
var err = logger.err;
var warn = logger.warn;
// number of cuncurrent requests to get package dependncies
var CONCURRENCY = 30;
// used for moment tracking
var this_moment,that_moment;
redis.connect(function(){
log('redis connected!!!');
})
/**
* return the package dependencies, we use callback(err,data) paradigm
* @param package_name
* @param callback
*/
function getPacakageDeps(package_name,callback) {
var scope = this;
this.package_name = package_name;
if (!redis.connected) {
throw new Error('no redis');
}
/**
* create the task to resolve dependencies from registry.npmjs
*/
// no redis, we do manual labor.
if (!redis.connected) {
scope._resolveTask(package_name,callback);
return;
}
that_moment = (new Date()).getTime()
// we have redis, lookup dependencies
redis.client.lrange(package_name,0,-1,function(err,data){
if (data.length>0) { // we have a redis cache for the package dependencies
log('Redis result for package_name:',package_name);
log('REDIS LRANGE Data:'+data+' length:',data.length);
_resolveTime()
if (callback) {
callback(null, {
'dependencies':data.sort(),
'total':data.length,
'resolve_time':_resolveTime()
});
}
} else { //we don't have dependencies for this
log('No Redis Answer for package:'+package_name);
scope._resolveTask(package_name,callback);
}
});
}
/**
* create a concurrent task queue to resolve dependencies for package via registry.npmjs.com webservice
* @param package_name
* @param callback
*/
function dependecyResolveTask(package_name,callback){
var scope = this;
// Assign an asyn queue model
scope.q = async.queue(function (task, _clbk) {
// task get dependencies of pacakge
scope._innerPackage(task.package_name,_clbk);
}, CONCURRENCY);
// last invocation when all tasks in Queue has finished
scope.q.drain = function() {
log('all items have been processed, total deps:'+scope.total_deps);
var deps_array = _.keys(scope.deps);
redis.lpush(package_name,deps_array );
// simply show how much time took (in log) and return the value
callback(null, {
'dependencies':deps_array.sort(),
'total':deps_array.length,
'resolve_time':_resolveTime()
} );
}
var reqEndpoint = 'https://registry.npmjs.org/'+package_name+'/latest?json=true';
log('dependecyResolve Call:>'+reqEndpoint);
// no data in redis, we make a call and update redis
request(reqEndpoint, function (error, response, body) {
if (error) {
err('error',error);
callback(error);
}
if (!error && response.statusCode == 200) {
// wrap in try catch make sure bad results do no affect client
var json = JSON.parse(body);
if (json && json['dependencies']) {
// iterating dependencies for this pacakge
_.each(json.dependencies,function(value,key){
// create a unique key for the dependency binding with with version
var insertKey = key+'@'+value;
log('first level dep insertKEy:'+insertKey+" package name"+key+' version:'+value);
if (!scope.deps[insertKey]) { // new dependency, key does not exist in our total dependenices list
// tsaving the dependency with both name and version
scope.deps[insertKey] = [key,value];
// go fetch it's dependencies
// adding a new task to the QUEUE
scope.q.push({package_name:key,with_version:insertKey});
} else {
log('dependency already cached for resolve')
}
scope.total_deps++;
});
/*callback(null,json.dependencies)*/
} else {
callback('no dependencies!!!')
}
//log(json) // this is the json response
} else {
callback('no such package!!! e:',error);
}
})
}
/**
* gets all deps for a specfic package, and start iterating it.
* @param package_name
*/
function makeInnerRequest(package_name,callback) {
var scope=this;
//log('requesting package:::',package_name)
request('https://registry.npmjs.org/'+package_name+'/latest?json=true', function (error, response, body) {
if (!error && response.statusCode == 200) {
var json = JSON.parse(body);
// for debugging
//log('result for makeInnerRequest:',json);
if (json && json['dependencies']) {
_.each(json.dependencies,function(value,key){
// collect the dependencies
var insertKey = key+'@'+value;
if (!scope.deps[insertKey]) { // new dependency
log('package:['+package_name+'] iterating dependency:'+key+' version:'+value);
scope.deps[insertKey] = [key,value,0];
// add package to queue of npm registry requests
scope.q.push({package_name:key});
}
// increase counter of total deps (not unique)
scope.total_deps++;
});
//callback(null,json.dependencies)
} else {
err('no dependencies inside result, invalid response?',JSON);
}
//log(json) // this is the json response
} else {
log('no such package!!!',error);
}
// get out of here
callback();
})
}
function PackageResolver(){
log('PackageResolver constructor()');
this.callback = null;
// dependencies are stored asa key value
this.deps = {};
this.package_name = "";
this.q=null;
this.total_deps = 0;
}
function _resolveTime(){
this_moment = (new Date()).getTime();
var delta = this_moment-that_moment;
log('Package Resolve took:'+moment.duration(delta));
return delta;
}
PackageResolver.prototype.getDeps = getPacakageDeps;
PackageResolver.prototype._innerPackage = makeInnerRequest;
PackageResolver.prototype._resolveTask = dependecyResolveTask;
module.exports = {
"create":function() {
return new PackageResolver();
}
};