hackernews2nntp/lib/hackernews2nntp-get.js

Post Hacker News Stories & Comments to an NNTP Server

// Generated by CoffeeScript 1.8.0
(function() {
  var Crawler2, Q, conf, fs, ids_get, ids_last, ids_range, livedata, maxitem_can_be_saved, maxitem_save, meta, path, program, request, u, util;

  fs = require('fs');

  path = require('path');

  util = require('util');

  meta = require('../package.json');

  Crawler2 = require('./crawler2');

  livedata = require('./livedata');

  u = require('./utils');

  program = require('commander');

  Q = require('q');

  request = require('request');

  conf = {
    url_pattern: 'https://hacker-news.firebaseio.com/v0/item/%d.json',
    verbose: 0,
    conn_per_sec: Crawler2.CONN_PER_SEC
  };

  ids_get = function(mode, spec) {
    if (mode === 'exact') {
      return Q.fcall(function() {
        var id;
        id = parseInt(spec[0]) || 0;
        if (id < 1) {
          throw new Error("`" + spec + "` must be > 0");
        }
        return [id];
      });
    }
    if (mode === 'last') {
      return ids_last(spec[0]);
    }
    if (mode === 'top100') {
      return livedata.top100();
    }
    if (mode === 'range') {
      return ids_range(spec);
    }
    if (mode === 'several') {
      return Q.fcall(function() {
        return spec;
      });
    } else {
      return Q.fcall(function() {
        throw new Error("invalid mode `" + mode + "`");
      });
    }
  };

  ids_range = function(spec) {
    var from, generate, to;
    generate = function(low, high) {
      var idx, max, _i, _results;
      max = 100000;
      if (low < 1) {
        throw new Error("" + low + " < 1");
      } else if (high < low) {
        throw new Error("" + high + " < " + low);
      } else if (high - low > max) {
        throw new Error("" + high + "-" + low + " > " + max);
      } else {
        _results = [];
        for (idx = _i = low; low <= high ? _i <= high : _i >= high; idx = low <= high ? ++_i : --_i) {
          _results.push(idx);
        }
        return _results;
      }
    };
    from = parseInt(spec[0]) || 0;
    to = parseInt(spec[1]) || 0;
    if (from >= 1 && to === 0) {
      return livedata.maxitem().then(function(maxitem) {
        return generate(from, maxitem);
      });
    } else {
      return Q.fcall(function() {
        return generate(from, to);
      });
    }
  };

  ids_last = function(spec) {
    var num;
    num = parseInt(spec) || 0;
    if (num < 1) {
      return Q.fcall(function() {
        throw new Error("`" + spec + "` must be >= 1");
      });
    }
    return livedata.maxitem().then(function(maxitem) {
      var idx, result, _i, _results;
      result = maxitem - num + 1;
      if (result < 1) {
        throw new Error("" + maxitem + "-" + num + " < 1");
      } else {
        _results = [];
        for (idx = _i = result; result <= maxitem ? _i <= maxitem : _i >= maxitem; idx = result <= maxitem ? ++_i : --_i) {
          _results.push(idx);
        }
        return _results;
      }
    });
  };

  maxitem_save = function(file, id) {
    var e;
    if (!file) {
      return;
    }
    try {
      return fs.writeFileSync(file, id);
    } catch (_error) {
      e = _error;
      return u.warnx("maxitem saiving failed: " + e.message);
    }
  };

  maxitem_can_be_saved = function(mode, stat) {
    var succ, threshold;
    if (!(mode != null ? mode.match(/^(last|range)$/) : void 0)) {
      return false;
    }
    threshold = 80;
    succ = (stat.downloaded.files / (stat.total() - stat.stale)) * 100;
    if (succ < threshold) {
      u.warnx("" + (Math.floor(succ)) + "% successful downloaded items is < that " + threshold + "% threshold, maxitem is not saved");
    }
    return succ >= threshold;
  };

  exports.main = function() {
    var mode;
    program.version(meta.version).usage("[options] mode [spec] \n  Available modes: top100, last <number>, exact <id>, range <from> <to>").option('-s, --show-stat', 'Print some statistics after all requests').option('-v, --verbose', 'Print HTTP status on stderr (implies -s)', function() {
      return conf.verbose++;
    }).option('--maxitem-save <file>', 'Write the highest id number to (for last & range modes only)').option('-u, --url-pattern <string>', "Debug. Default: " + conf.url_pattern, conf.url_pattern).option('--nokids', "Debug").option('--ids-only', "Debug").option('--conn-per-sec <digit>', "HTTP request limit (0 == no limit). Default: " + conf.conn_per_sec, conf.conn_per_sec).parse(process.argv);
    if (program.args.length < 1) {
      program.outputHelp();
      process.exit(1);
    }
    mode = program.args[0];
    return ids_get(mode, program.args.slice(1)).then(function(ids) {
      var crawler2;
      if (program.idsOnly) {
        console.error(ids);
        process.exit(0);
      }
      crawler2 = new Crawler2(program.connPerSec);
      crawler2.url_pattern = program.urlPattern;
      if (!conf.verbose) {
        crawler2.logger = function() {};
      }
      if (program.nokids) {
        crawler2.look4kids = false;
      }
      process.on('exit', function() {
        if (conf.verbose || program.showStat) {
          console.error("\nSTAT: " + util.inspect(crawler2.stat.succinct()));
        }
        if (conf.verbose >= 2) {
          console.error(util.inspect(crawler2.stat._history, {
            showHidden: false,
            depth: null
          }));
        }
        if (program.maxitemSave && maxitem_can_be_saved(mode, crawler2.stat)) {
          return maxitem_save(program.maxitemSave, ids[ids.length - 1]);
        }
      });
      crawler2.event.on('data', function(iter_id, data) {
        return process.stdout.write("" + data + "\n");
      });
      crawler2.event.on('herr', function(err) {
        if (conf.verbose) {
          return console.error("ERROR: ii=" + err.iter_id + "/id=" + err.id + ": " + err.message);
        }
      });
      return crawler2.event.emit('items', crawler2.stat.iter_id_next(), ids);
    })["catch"](function(err) {
      return u.errx(err.message);
    }).done();
  };

}).call(this);