jul11co-sitespider/spider.js

Jul11Co Web Spider

var async = require('async');
var util = require('util');
var fs = require('fs');
var path = require('path');
var urlutil = require('url');

var cheerio = require('cheerio');

var utils = require('jul11co-wdt').Utils;
var downloader = require('jul11co-wdt').Downloader;
var scraper = require('jul11co-wdt').Scraper;

var Saver = require('jul11co-wdt').Saver;

// helper
// {
//   name: String,
//   match: function(url, options) {...}
//   preprocess: function($, page, options) {...}
//   postprocess: function($, page, options) {...}
// }
var page_helpers = [];
if (utils.directoryExists(__dirname + '/helpers')) {
  fs.readdirSync(__dirname + '/helpers').forEach(function(file) {
    if (file.indexOf('.js') > 0) {
      var helper = require("./helpers/" + file);
      page_helpers.push(helper);
    }
  });
}

var Spider = function(options) {
  Saver.call(this, options);
  if (options.verbose) console.log(options);
  this._scrapers = [];
  this.process_queue = [];
  this.queued = {};
}

util.inherits(Spider, Saver);

Spider.prototype.getConfig = function(key) {
  return this._config[key];
}

Spider.prototype.setConfig = function(key, value) {
  this._config[key] = value;
}

// scraper
// {
//   name: String,
//   match: function(url, options) {...}
//   scrape: function($, page, options) {...}
// }
Spider.prototype.addScraper = function(scraper) {
  // console.log('Spider:', 'Add scraper:', scraper.name);
  this._scrapers.push(scraper);
}

Spider.prototype.importScrapersFromDir = function(directory) {
  var self = this;
  if (utils.directoryExists(directory)) {
    fs.readdirSync(directory).forEach(function(file) {
      if (file.indexOf('.js') > 0) {
        self._scrapers.push(require(path.join(directory, file)));
      }
    });
  }
}

var getLocalFilePath = function(file_url, options) {
  var file_url_obj = urlutil.parse(file_url);
  var file_path = file_url_obj.host + file_url_obj.pathname;
  return path.join((options.output_dir || '.'), file_path);
}

Spider.prototype.scrapePage = function($, page, options) {
  var self = this;
  
  var scrapers = [];
  if (self._scrapers.length) {
    for (var i = 0; i < self._scrapers.length; i++) {
      if (self._scrapers[i].match(page.url, options)) {
        scrapers.push(self._scrapers[i]);
      }
    }
  }
  if (scrapers.length) {
    scrapers.forEach(function(scraper) {
      if (typeof scraper.scrape == 'function') {
        scraper.scrape($, page, options);
      }
    });
  }

  var files = [];
  var filemap = {};

  var index_file = '';
  var page_url_obj = urlutil.parse(page.url);
  if (page_url_obj.query) {
    index_file = path.resolve(page.output_dir, 'index-' + page_url_obj.query + '.html');
  } else {
    index_file = path.resolve(page.output_dir, 'index.html');
  }
  // if (options.verbose) console.log('Index file:', index_file);

  if (options.download_scripts) {
    var script_blacklist = options.script_blacklist || [];
    var script_filters = options.script_filters || [];
    var scripts = [];

    $('script').each(function() {
      var script_src = $(this).attr('src');
      if (script_src && script_src != '' && script_src !== '//') {
        // script_src = self.fixLink(script_src, page);
        if (isBlacklisted(script_src, script_blacklist)) return;
        if (isFilteredOut(script_src, script_filters)) return;
        scripts.push(script_src);
        var local_file = getLocalFilePath(script_src, options);
        files.push({
          type: $(this).attr('type') || 'text/javascript',
          url: script_src,
          local_file: local_file
        });
        filemap[script_src] = local_file;
        $(this).attr('src', path.relative(page.output_dir, local_file));
      }
    });
    if (options.verbose) console.log(scripts);
    page.scripts = scripts;
  }

  if (options.download_stylesheets) {
    var stylesheet_blacklist = options.stylesheet_blacklist || [];
    var stylesheet_filters = options.stylesheet_filters || [];
    var stylesheets = [];

    $('link[rel="stylesheet"]').each(function() {
      var style_href = $(this).attr('href');
      if (style_href && style_href != '') {
        // style_href = self.fixLink(style_href, page);
        // stylesheets.push(style_href);
        if (isBlacklisted(style_href, stylesheet_blacklist)) return;
        if (isFilteredOut(style_href, stylesheet_filters)) return;
        var local_file = getLocalFilePath(style_href, options);
        files.push({
          type: $(this).attr('type') || 'text/css',
          url: style_href,
          local_file: local_file
        });
        filemap[style_href] = local_file;
        $(this).attr('href', path.relative(page.output_dir, local_file));
      }
    });
    if (options.verbose) console.log(stylesheets);
    page.stylesheets = stylesheets;
  }

  if (options.download_images) {
    // var images = self.getImages($, page, 'body');
    var image_blacklist = options.image_blacklist || [];
    var image_filters = options.image_filters || [];
    var images = [];

    $('img').each(function() {
      var image_src = $(this).attr('src');
      if (image_src && image_src != '') {
        if (image_src.indexOf('data:image') == 0) return;
        // image_src = self.fixLink(image_src, page, {});
        if (isBlacklisted(image_src, image_blacklist)) return;
        if (isFilteredOut(image_src, image_filters)) return;
        images.push(image_src);
        var local_file = getLocalFilePath(image_src, options);
        files.push({
          type: 'image',
          url: image_src,
          local_file: local_file
        });
        filemap[image_src] = local_file;
        $(this).attr('src', path.relative(page.output_dir, local_file));
      }
    });
    if (options.verbose) console.log(images);
    page.images = images;
  }

  // if (options.verbose) console.log(files);

  page.index_file = index_file;
  page.files = files;
  page.filemap = filemap;
}

Spider.prototype.savePage = function($, page, options, callback) {
  var self = this;

  if (options.no_html_cache) {
    return callback();
  }

  if (page.index_file && !page.html_cached) {
    self.saveHtmlSync(page.index_file, $.html());
  }
  
  if (page.files && page.files.length) {
    downloader.downloadFiles(page.files, function(err) {
      if (err) return callback(err);
      callback();
    });
  } else {
    callback();
  }
}

Spider.prototype.getIndexFilePath = function(page_url, options) {
  var self = this;
  options = options || {};

  var index_file = '';
  var page_url_obj = urlutil.parse(page_url);
  var page_output_dir_path = path.join(page_url_obj.host, page_url_obj.pathname);

  if (path.basename(page_url_obj.pathname) == 'index.html') { // special case
    page_output_dir_path = path.dirname(page_output_dir_path);
  }

  var page_output_dir = path.join((options.output_dir || '.'), page_output_dir_path);
  if (options.output_html_dir) {
    page_output_dir = path.join(page_output_dir, options.output_html_dir);
  }

  if (page_url_obj.query) {
    index_file = path.resolve(page_output_dir, 'index-' + page_url_obj.query + '.html');
  } else {
    index_file = path.resolve(page_output_dir, 'index.html');
  }

  return index_file;
}

Spider.prototype.getPageHtml = function(link, options, callback) {
  var self = this;

  var local_file = self.getIndexFilePath(link.url, options);
  if (!options.no_html_cache && !link.cache_bypass && utils.fileExists(local_file)) {
    if (options.debug) console.log('Cached: ' + local_file);

    if (link.url.substring(link.url.length-16) == '.html/index.html') {
      link.url = link.url.substring(0, link.url.length-11);
      if (options.verbose) console.log('URL: ' + link.url);
    }

    var page = {
      url: link.url
    };

    var page_html = self.loadHtmlSync(local_file);
    if (page_html) {
      page.html_cached = true;
      page.html = page_html;

      return callback(null, page);
    }
    // download new HTML
    // fall through
  }

  if (options.verbose) console.log('Visit: ' + link.url);

  downloader.downloadPage(link.url, options, function(err, result) {
    if (err) {
      return callback(err);
    }

    if (!result.html) {
      return callback(new Error('Invalid HTML ($==null)'));
    }

    var page = {
      url: result.url,
      html: result.html
    };

    return callback(null, page);
  });
}

Spider.prototype.processPage = function(link, options, callback) {
  var self = this;

  if (options.verbose) console.log('Process:', link.url);

  self.getPageHtml(link, options, function(err, page) {
    if (err) {
      // self.setVisited(link.url); // set visited
      self.emit('page_error', err, link);
      return callback(err);
    }
    if (!page.html) {
      self.setVisited(link.url); // set visited
      self.emit('page_error', new Error('Invalid HTML ($==null)'), link.url);
      return callback(new Error('Invalid HTML ($==null)'));
    }
    
    var $ = cheerio.load(page.html);

    page.title = $('title').first().text();
    if (page.title) {
      page.title = page.title.replace(/(\r\n|\n|\r)/gm, '');
    }

    // console.log('\x1b[2m%s\x1b[0m', 'Page URL: ' + page.url);
    
    var link_obj = urlutil.parse(page.url);
    var output_dir_path = path.join(link_obj.host, link_obj.pathname);
    if (path.basename(link_obj.pathname) == 'index.html') { // special case
      output_dir_path = path.dirname(output_dir_path);
    }
    var output_dir = (options.output_dir || '.');
    if (options.output_html_dir) {
      output_dir = path.join(output_dir, options.output_html_dir);
    }
    output_dir = path.join(output_dir, output_dir_path);
    if (options.verbose) console.log('Output directory: ' + output_dir);
    
    page.output_dir = output_dir;

    if ($('head base').length) {
      page.base_url = $('head base').attr('href');
      // console.log('Base URL:', page.base_url);
    }

    // self.fixLinks($, page, 'body');
    self.fixLinks($, page, 'html', options);
    self.fixImages($, page, 'body', options);

    $('script').each(function() {
      var script_src = $(this).attr('src');
      if (script_src && script_src != '' && script_src !== '//') {
        script_src = self.fixLink(script_src, page);
        $(this).attr('src', script_src);
      }
    });
    $('link[rel="stylesheet"]').each(function() {
      var style_href = $(this).attr('href');
      if (style_href && style_href != '') {
        style_href = self.fixLink(style_href, page);
        $(this).attr('href', style_href);
      }
    });
    $('img').each(function() {
      var image_src = $(this).attr('src');
      if (image_src && image_src != '') {
        if (image_src.indexOf('data:image') == 0) return;
        image_src = self.fixLink(image_src, page);
        $(this).attr('src', image_src);
      }
    });

    var helpers = [];
    if (page_helpers.length) {
      for (var i = 0; i < page_helpers.length; i++) {
        if (page_helpers[i].match(page.url, options)) {
          helpers.push(page_helpers[i]);
        }
      }
    }

    // Preprocess
    if (helpers.length) {
      helpers.forEach(function(helper) {
        if (helper.preprocess) helper.preprocess($, page, options);
      });
    }

    self.scrapePage($, page, options);

    // Postprocess
    if (helpers.length) {
      helpers.forEach(function(helper) {
        if (helper.postprocess) helper.postprocess($, page, options);
      });
    }

    self.setVisited(link.url); // set visited

    self.savePage($, page, options, function(err) {
      if (err) {
        self.emit('page_error', err, link.url);
        return callback(err);
      }

      var depth = options.depth || 0;
      var max_depth = options.max_depth || 1;

      if (max_depth !== -1 && depth >= max_depth) {
        if (options.debug) console.log('Depth:', depth, max_depth);
      } else {
        // extract links
        var link_container = options.link_container || 'body';
        var link_blacklist = options.link_blacklist || [];
        var link_filters = options.link_filters || [];

        var link_options = {
          blacklist: link_blacklist, 
          filters: link_filters,
          exclude_visited_links: true
        };
        if (!options.link_to_other_hosts) {
          link_options.filter_host = link_obj.host; // same host
        }
        var links = self.getLinks($, page, link_container, link_options);
        
        console.log('[d=' + options.depth + '] Links count: ' + links.length);

        links.forEach(function(link) {
          var state = self.getStateData(link);
          if (!self.isQueued(link) && (!state || !state.visited)) {
            self.updateStateData(link, { visited: 0 });
            self.pushToQueue({
              url: link,
              depth: depth+1
            });
          }
        });
      }

      $ = null;

      self.emit('page_done', page);

      callback(null, page);
    });
  });
}

Spider.prototype.isQueued = function(link) {
  return this.queued[link];
}

Spider.prototype.pushToQueue = function(item) {
  if (item && item.url) {
    this.queued[item.url] = 1;
  }
  this.process_queue.push(item);
}

Spider.prototype.processQueue = function(options) {
  var self = this;

  var link_nofollow = options.link_nofollow || [];

  var nextLink = function(previous_page) {
    
    if (self.process_queue.length == 0) {
      console.log('Page queue: empty.');
      // self.exit();
      if (!options.no_links_fix) {
        self.fix_links(options.output_dir, options);
      }
      return;
    }
    
    console.log('Page queue:', self.process_queue.length);

    var next_page = self.process_queue.shift();
    if (!next_page.force && self.isVisited(next_page.url)) {
      self.emit('page_done', next_page);
    } else if (link_nofollow && link_nofollow.length > 0
      && isBlacklisted(next_page.url, link_nofollow)) {
      if (options.debug) console.log('Nofollow: ' + next_page.url);
      // nextLink(next_page);
      self.emit('page_done', next_page);
    } else {
      var delay = 1000;
      if (previous_page && previous_page.html_cached) {
        delay = 0;
      }
      setTimeout(function() {
        options.depth = next_page.depth;
        self.processPage(next_page, options, function(err) {
          if (err) {
            console.log(err);
            // self.exit(err);
          }
        });
      }, delay);
    }
  }

  self.on('page_done', function(page) {
    process.nextTick(function() {
      nextLink(page);
    });
  });

  self.on('page_error', function(err) {
    console.log('Page error:', err);
    // self.exit(err);
    process.nextTick(function() {
      nextLink();
    });
  });

  nextLink();
}

function isBlacklisted(url, blacklist) {
  if (!blacklist || blacklist.length == 0) return false;
  var blacklisted = false;
  for (var i = 0; i < blacklist.length; i++) {
    if (url.indexOf(blacklist[i]) >= 0) {
      blacklisted = true;
      break;
    }
  }
  return blacklisted;
}

function isFilteredOut(url, filters) {
  if (!filters || filters.length == 0) return false;
  var filter_out = true;
  for (var i = 0; i < filters.length; i++) {
    if (url.indexOf(filters[i]) >= 0) {
      filter_out = false;
      break;
    }
  }
  return filter_out;
}

Spider.prototype.start = function(start_link, output_dir, options) {
  var self = this;

  console.log('Spider:', 'start');

  options.output_dir = output_dir;

  options = Object.assign(self._config, options);

  if (options.download) {
    self.setStateData('start', start_link);
  }

  if (Array.isArray(start_link)) {
    start_link.forEach(function(link) {
      self.pushToQueue({
        url: link,
        force: true,
        depth: 0,
        cache_bypass: true
      });
    });
  } else { // string
    self.pushToQueue({
      url: start_link,
      force: true,
      depth: 0,
      cache_bypass: true
    });
  }

  self.processQueue(options);
}

Spider.prototype.resume = function(output_dir, options) {
  var self = this;

  console.log('Spider:', 'resume');

  options.output_dir = output_dir;

  options = Object.assign(self._config, options);

  var link_blacklist = options.link_blacklist || [];
  var link_filters = options.link_filters || [];
  var link_nofollow = options.link_nofollow || [];

  var state = self.getState();
  for (var url in state) {
    if (!/^((http|https):\/\/)/.test(url)) continue;
    if (isBlacklisted(url, link_blacklist)) continue;
    if (isFilteredOut(url, link_filters)) continue;
    if (isBlacklisted(url, link_nofollow)) continue;
    if (state[url] && !state[url].visited && !self.isQueued(url)) {
      self.pushToQueue({
        url: url,
        depth: 0
      });
    }
  }

  self.processQueue(options);
}

Spider.prototype.update = function(output_dir, options) {
  var self = this;

  console.log('Spider:', 'update');

  options.output_dir = output_dir;

  options = Object.assign(self._config, options);

  var link_container = options.link_container || 'body';
  var link_blacklist = options.link_blacklist || [];
  var link_filters = options.link_filters || [];
  var link_nofollow = options.link_nofollow || [];

  var local_pages = [];
  var state = self.getState();
  for (var url in state) {
    if (!/^((http|https):\/\/)/.test(url)) continue;
    if (isBlacklisted(url, link_blacklist)) {
      console.log('Spider:', 'URL blacklisted:', url);
      self.deleteStateData(url);
      continue;
    }
    if (isFilteredOut(url, link_filters)) continue;
    if (state[url] && state[url].visited) {
      var local_file = self.getIndexFilePath(url, options);
      local_pages.push({
        url: url,
        local_file: local_file
      });
    }
  }

  var getPageLinks = function(page_url, local_file, options) {

    var page_url_obj = urlutil.parse(page_url);
    var page_output_dir = path.dirname(local_file);
    var page_html = self.loadHtmlSync(local_file);
    if (page_html != '') {

      var $ = cheerio.load(page_html);

      // extract links
      var page = {
        url: page_url
      };
      var link_options = {
        blacklist: link_blacklist, 
        filters: link_filters,
        exclude_visited_links: true,
        validator: function(link_href) {
          if (link_href.indexOf('http') == 0) return true;
          return false;
        }
      };
      if (!options.link_to_other_hosts) {
        link_options.filter_host = page_url_obj.host; // same host
      }
      var links = self.getLinks($, page, link_container, link_options);
      if (options.verbose) {
        console.log('Link count: ' + links.length);
        console.log(links);
      }
      links.forEach(function(link) {
        link = link.replace('/index.html', '');
        var state = self.getStateData(link);
        if (!self.isQueued(link) && (!state || !state.visited)) {
          self.updateStateData(link, { visited: 0 });
          self.pushToQueue({
            url: link,
            depth: 0
          });
        }
      });

      $ = null;
    }
  }

  console.log('Spider:', 'Getting links...');
  var total = local_pages.length;
  for (var i = 0; i < total; i++) {
    var local_page = local_pages[i];
    if (options.verbose) {
      console.log('[' + i + '/' + total + '] File:', local_page.local_file);
    }
    getPageLinks(local_page.url, local_page.local_file, options);
  }
  console.log('Spider:', 'Getting links... Done.');

  self.processQueue(options);
}

Spider.prototype.fix_links = function(output_dir, options) {
  var self = this;

  options.output_dir = output_dir;

  options = Object.assign(self._config, options);

  var link_blacklist = options.link_blacklist || [];
  var link_filters = options.link_filters || [];

  var link_cache = {};
  var fix_queue = [];

  var state = self.getState();
  for (var url in state) {
    if (!/^((http|https):\/\/)/.test(url)) continue;
    if (isBlacklisted(url, link_blacklist)) continue;
    if (isFilteredOut(url, link_filters)) continue;
    if (state[url] && state[url].visited) {
      var local_file = self.getIndexFilePath(url, options);
      fix_queue.push({
        url: url,
        local_file: local_file
      });
      link_cache[url] = {
        local_file: local_file
      };
    }
  }

  var fixPageLinks = function(page_url, local_file, options) {
    var page_output_dir = path.dirname(local_file);
    var page_html = self.loadHtmlSync(local_file);

    if (page_html != '') {
      var $ = cheerio.load(page_html);
      var updated = 0;
      $('a').each(function() {
        var link_href = $(this).attr('href');
        if (link_href  && link_href != '' && link_href.indexOf('http') == 0) {
          var local_index_file = '';
          if (link_cache[link_href]) {
            local_index_file = link_cache[link_href].local_file;
          } else {
            local_index_file = getIndexFilePath(link_href);
            link_cache[link_href] = {
              url: link_href,
              local_file: local_index_file
            }
          }
          if (utils.fileExists(local_index_file)) {
            var relative_path = path.relative(page_output_dir, local_index_file);
            // console.log(link_href + ' -> ' + relative_path);
            $(this).attr('href', relative_path);
            updated++;
          }
        }
      });
      if (updated > 0) console.log('Updated:', updated);
      self.saveHtmlSync(local_file, $.html());
    }
  }

  console.log('Spider:', 'Files to be fixed:', fix_queue.length);
  var total = fix_queue.length;
  for (var i = 0; i < total; i++) {
    var local_page = fix_queue[i];
    console.log('[' + i + '/' + total + '] Fixing:', local_page.url);
    // console.log('Local file:', local_page.local_file);
    fixPageLinks(local_page.url, local_page.local_file, options);
  }

  self.exit();
}

module.exports = Spider;