jul11co-sitespider/cli.js

Jul11Co Web Spider

#!/usr/bin/env node

var async = require('async');
var path = require('path');

var Spider = require('./spider');

function printUsage() {
  console.log('Usage: sitespider [OPTIONS...] [page_url] <output_dir>');
  console.log('');
  console.log('   --download         : Download');
  console.log('   --resume           : Resume');
  console.log('   --update           : Update (and check for incompleted links)');
  console.log('   --add-link         : Add link');
  console.log('   --fix-links        : Fix links');
  console.log('');
  console.log('   --verbose          : Verbose');
  console.log('   --force            : Forcibly action');
  console.log('');
  console.log('   --images           : Download images (default: false)');
  console.log('   --scripts          : Download scripts (default: false)');
  console.log('   --stylesheets      : Download stylesheets (default: false)');
  console.log('');
  console.log('   --max-depth=X      : Specify max depth');
  console.log('');
}

if (process.argv.length < 3) {
  printUsage();
  process.exit();
  return;
}

process.on('SIGINT', function() {
  console.log("\nCaught interrupt signal");
  process.exit();
});

var argv = [];
var options = {};
for (var i = 2; i < process.argv.length; i++) {
  if (process.argv[i] == '--force') {
    options.force = true;
  } else if (process.argv[i] == '--verbose') {
    options.verbose = true;
  } else if (process.argv[i] == '--proxy') {
    options.proxy = true;
  } else if (process.argv[i] == '--images') {
    options.download_images = true;
  } else if (process.argv[i] == '--scripts') {
    options.download_scripts = true;
  } else if (process.argv[i] == '--stylesheets') {
    options.download_stylesheets = true;
  } else if (process.argv[i] == '--fix-links') {
    options.fix_links = true;
  } else if (process.argv[i] == '--add-link') {
    options.add_link = true;
  } else if (process.argv[i] == '--download') {
    options.download = true;
  } else if (process.argv[i] == '--resume') {
    options.resume = true;
    options.max_depth = -1;
  } else if (process.argv[i] == '--update') {
    options.update = true;
  } else if (process.argv[i] == '--no-links-fix') {
    options.no_links_fix = true;
  } else if (process.argv[i].indexOf('--max-depth=') == 0) {
    var max_depth_str = process.argv[i].replace('--max-depth=','');
    var max_depth = parseInt(max_depth_str);
    if (!isNaN(max_depth)) options.max_depth = max_depth;
  } else {
    argv.push(process.argv[i]);
  }
}

if (typeof options.max_depth == 'undefined') {
  options.max_depth = -1;
}

if (argv.length >= 1) {

  var page_url = '';
  var output_dir = '';
  if (/^((http|https):\/\/)/.test(argv[0])) {
    page_url = argv[0];
    if (argv.length >= 2) {
      output_dir = argv[1];
    }
  } else {
    output_dir = argv[0];
  }
  
  if (page_url && page_url != '') {
    options.page_url = page_url;
    console.log('Page URL: ' + page_url);
  }
  
  options.output_dir = output_dir || './';
  console.log('Output directory: ' + options.output_dir);

  options.state_file = path.join(options.output_dir, 'spider-state.json');
  options.config_file = path.join(options.output_dir, 'spider.json');

  var spider = new Spider(options);

  spider.on('error', function(err) {
    console.log('Spider error:');
    console.log(err);
  });
  spider.on('exit', function(err) {
    if (err) console.log(err);
    else console.log('Done.');
  });

  if (options.fix_links) {
    spider.fix_links(output_dir, options);
  } else if (options.resume) {
    spider.resume(output_dir, options);
  } else if (options.update) {
    spider.update(output_dir, options);
  } else {
    spider.start(page_url, output_dir, options);
  }

} else {
  printUsage();
  process.exit();
}