node-nutch/plugins/basicURLNormalizer.js

A set of Gulp commands that provide similar functionality to Apache Nutch.

var URI = require('URIjs');

var BasicURLNormalizer = function (){};

BasicURLNormalizer.prototype.normalize = function(urlString){
  return URI(urlString)
    .fragment('')
    .normalize()
    .toString();
};

module.exports = new BasicURLNormalizer();