hackernews2nntp
Version:
Post Hacker News Stories & Comments to an NNTP Server
482 lines (420 loc) • 13.6 kB
JavaScript
// Generated by CoffeeScript 1.8.0
(function() {
var Crawler2, CrawlerError, Dup, InvalidItem, InvalidJSON, NoConnect, NoItem, PolloptInIteration, Q, RateLimiter, Stat, events, makeError, meta, request, superError, u, util,
__hasProp = {}.hasOwnProperty,
__extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
events = require('events');
util = require('util');
Q = require('q');
RateLimiter = require('limiter').RateLimiter;
request = require('request');
u = require('./utils');
meta = require('../package.json');
superError = function(klass, errObj, msg) {
errObj.name = klass.name;
errObj.message = msg;
return Error.captureStackTrace(errObj, klass);
};
CrawlerError = (function(_super) {
__extends(CrawlerError, _super);
function CrawlerError(msg) {
superError(CrawlerError, this, msg);
}
return CrawlerError;
})(Error);
Dup = (function(_super) {
__extends(Dup, _super);
function Dup(msg) {
superError(Dup, this, msg);
}
return Dup;
})(CrawlerError);
InvalidJSON = (function(_super) {
__extends(InvalidJSON, _super);
function InvalidJSON(msg) {
superError(InvalidJSON, this, msg);
}
return InvalidJSON;
})(CrawlerError);
InvalidItem = (function(_super) {
__extends(InvalidItem, _super);
function InvalidItem(msg) {
superError(InvalidItem, this, msg);
}
return InvalidItem;
})(CrawlerError);
NoItem = (function(_super) {
__extends(NoItem, _super);
function NoItem(msg) {
superError(NoItem, this, msg);
}
return NoItem;
})(CrawlerError);
PolloptInIteration = (function(_super) {
__extends(PolloptInIteration, _super);
function PolloptInIteration(msg) {
superError(PolloptInIteration, this, msg);
}
return PolloptInIteration;
})(CrawlerError);
NoConnect = (function(_super) {
__extends(NoConnect, _super);
function NoConnect(msg) {
superError(NoConnect, this, msg);
}
return NoConnect;
})(CrawlerError);
makeError = function(klass, iter_id, id, msg) {
var err;
err = new klass(msg);
err.iter_id = iter_id;
err.id = id;
return err;
};
Stat = (function() {
function Stat(event) {
this.event = event;
this.iter_id = 0;
this._history = {};
this.bytes = 0;
this._iterations = [];
}
Stat.prototype.history = function(iter_id, id, err, iter_id_next) {
var info, last, o, _base;
if (iter_id_next == null) {
iter_id_next = null;
}
o = (_base = this._history)[iter_id] || (_base[iter_id] = {});
o[id] || (o[id] = []);
last = o[id][o[id].length - 1];
if ((last != null ? last.iter_id_next : void 0) === 'http get') {
o[id].pop();
}
info = {};
if (err) {
info.err = err;
}
if (iter_id_next) {
info.iter_id_next = iter_id_next.toString();
}
o[id].push(info);
if (info.err) {
return this.event.emit('herr', err);
}
};
Stat.prototype.iter_id_next = function() {
return ++this.iter_id;
};
Stat.prototype.has_been = function(id) {
var key, val, _ref;
id = id.toString();
_ref = this._history;
for (key in _ref) {
val = _ref[key];
if (id in val) {
return true;
}
}
return false;
};
Stat.prototype.succinct = function() {
var errclass2desc, id, ids, ikey, item, items, ival, r, val, _i, _j, _len, _len1, _ref;
ids = {};
_ref = this._history;
for (ikey in _ref) {
ival = _ref[ikey];
for (id in ival) {
val = ival[id];
ids[id] || (ids[id] = []);
ids[id].push(val);
}
}
r = {
bytes: this.bytes,
items: 0,
neterr: 0,
invalid: 0,
dup: 0,
uniq: (Object.keys(ids)).length
};
errclass2desc = {
'Dup': 'dup',
'InvalidJSON': 'invalid',
'InvalidItem': 'invalid',
'NoItem': 'neterr',
'PolloptInIteration': 'invalid',
'NoConnect': 'neterr'
};
for (id in ids) {
val = ids[id];
for (_i = 0, _len = val.length; _i < _len; _i++) {
items = val[_i];
for (_j = 0, _len1 = items.length; _j < _len1; _j++) {
item = items[_j];
if (!item.err) {
r.items++;
} else {
r[errclass2desc[item.err.constructor.name]]++;
}
}
}
}
return r;
};
Stat.prototype.iteration_begin = function(iter_id) {
var idx;
idx = this._iterations.indexOf(iter_id);
if (idx === -1) {
this._iterations.push(iter_id);
return this.event.emit('iteration:begin', iter_id);
}
};
Stat.prototype.iteration_end = function(iter_id) {
var idx;
idx = this._iterations.indexOf(iter_id);
if (idx !== -1) {
this._iterations.splice(idx, 1);
this.event.emit('iteration:end', iter_id);
if (this._iterations.length === 0) {
return this.event.emit('finish');
}
}
};
return Stat;
})();
Crawler2 = (function() {
Crawler2.ItemValidate = function(json) {
var _ref;
if (!json) {
return false;
}
if (!json.id) {
return false;
}
if (!((_ref = json.type) != null ? _ref.toString().match(/^(story|comment|poll|pollopt)$/) : void 0)) {
return false;
}
if (json.kids && !util.isArray(json.kids)) {
return false;
}
if (json.type === 'poll' && !json.parts) {
return false;
}
if (json.type === 'poll' && !util.isArray(json.parts)) {
return false;
}
if (json.type === 'poll' && json.parts.length === 0) {
return false;
}
return true;
};
Crawler2.idValidate = function(id, deferred) {
if (!u.isNum(id)) {
deferred.reject(new Error("invalid id `" + id + "`"));
return false;
}
return true;
};
Crawler2.iterIdValidate = function(iter_id, deferred) {
if (!(u.isStr(iter_id) || u.isNum(iter_id))) {
deferred.reject(new Error("invalid iter_id `" + iter_id + "`"));
return false;
}
return true;
};
Crawler2.CONN_PER_SEC = 20;
function Crawler2(conn_per_sec) {
this.conn_per_sec = conn_per_sec != null ? conn_per_sec : Crawler2.CONN_PER_SEC;
this.url_pattern = 'https://hacker-news.firebaseio.com/v0/item/%s.json';
this.look4kids = true;
this.logger = console.error;
this.event = new events.EventEmitter();
this.stat = new Stat(this.event);
this.limiter = new RateLimiter(this.conn_per_sec, 'sec');
this.event.on('items', (function(_this) {
return function(iter_id, idarr) {
return _this.iteration(iter_id, idarr);
};
})(this));
this.event.on('poll', (function(_this) {
return function(iter_id, item) {
return _this.poll(iter_id, item);
};
})(this));
this.http_headers = {
'User-Agent': [meta.name, meta.version].join('/')
};
}
Crawler2.prototype.iteration = function(iter_id, idarr) {
var id, idx, _i, _len, _results;
if (!util.isArray(idarr)) {
throw new Error("array of IDs is expected");
}
_results = [];
for (idx = _i = 0, _len = idarr.length; _i < _len; idx = ++_i) {
id = idarr[idx];
if (idx === 0) {
this.stat.iteration_begin(iter_id);
}
_results.push((function(_this) {
return function(idx) {
return _this.fetch_item(iter_id, id).then(function(r) {
var iter_id_next;
if (r.type === 'pollopt') {
throw makeError(PolloptInIteration, iter_id, r.id, "must be processed by Crawler2#poll()");
}
if (r.type === 'poll') {
iter_id_next = _this.stat.iter_id_next() + '.poll';
_this.stat.history(iter_id, r.id, null, iter_id_next);
return _this.event.emit('poll', iter_id_next, r);
} else {
_this.event.emit('data', iter_id, JSON.stringify(r));
if (!_this.iteration_4kids(iter_id, r)) {
return _this.stat.history(iter_id, r.id, null, null);
}
}
})["catch"](function(err) {
if (err instanceof CrawlerError) {
return _this.stat.history(iter_id, err.id, err, null);
} else {
throw err;
}
}).fin(function() {
if (idx === idarr.length - 1) {
return _this.stat.iteration_end(iter_id);
}
}).done();
};
})(this)(idx));
}
return _results;
};
Crawler2.prototype.iteration_4kids = function(iter_id, item) {
var iter_id_next;
if (item.kids && this.look4kids) {
iter_id_next = this.stat.iter_id_next();
this.stat.history(iter_id, item.id, null, iter_id_next);
return this.event.emit('items', iter_id_next, item.kids);
}
};
Crawler2.prototype.poll = function(iter_id, item) {
var id, idx, parts, promises, _fn, _i, _len, _ref;
this.iteration_4kids(iter_id, item);
parts = [JSON.stringify(item)];
promises = [];
_ref = item.parts;
_fn = (function(_this) {
return function(idx) {
var p;
p = _this.fetch_item(iter_id, id);
promises.push(p);
return p.then(function(r) {
_this.stat.history(iter_id, r.id, null, null);
return parts.push(JSON.stringify(r));
})["catch"](function(err) {
if (err instanceof CrawlerError) {
return _this.stat.history(iter_id, err.id, err, null);
} else {
throw err;
}
}).fin(function() {
if (idx === item.parts.length - 1) {
return _this.stat.iteration_end(iter_id);
}
}).done();
};
})(this);
for (idx = _i = 0, _len = _ref.length; _i < _len; idx = ++_i) {
id = _ref[idx];
if (idx === 0) {
this.stat.iteration_begin(iter_id);
}
_fn(idx);
}
return Q.all(promises).then((function(_this) {
return function() {
return _this.event.emit('data', iter_id, parts.join("\n"));
};
})(this));
};
Crawler2.prototype.http_get = function(iter_id, id) {
var cur_req, deferred, log, opt;
log = (function(_this) {
return function(msg) {
return _this.logger("ii=" + iter_id + "/id=" + id + " " + msg);
};
})(this);
deferred = Q.defer();
opt = {
url: util.format(this.url_pattern, id),
headers: this.http_headers
};
cur_req = request.get(opt, (function(_this) {
return function(err, res, body) {
if (err) {
deferred.reject(makeError(NoConnect, iter_id, id, err.message));
return deferred.promise;
}
if (res.statusCode === 200) {
_this.stat.bytes += Buffer.byteLength(body);
log("HTTP 200");
return deferred.resolve(body);
} else {
return deferred.reject(makeError(NoItem, iter_id, id, "HTTP " + res.statusCode));
}
};
})(this));
cur_req.on('request', (function(_this) {
return function() {
if (_this.stat.has_been(id)) {
if (!iter_id.toString().match(/\.poll$/)) {
deferred.reject(makeError(Dup, iter_id, id, "dup!"));
return cur_req.abort();
}
} else {
log("HTTP GET");
return _this.stat.history(iter_id, id, null, 'http get');
}
};
})(this));
return deferred.promise;
};
Crawler2.prototype.fetch_item_bare = function(iter_id, id, deferred) {
if (!Crawler2.idValidate(id, deferred)) {
return;
}
if (!Crawler2.iterIdValidate(iter_id, deferred)) {
return;
}
return this.http_get(iter_id, id).then(function(item) {
var e;
try {
item = JSON.parse(item);
} catch (_error) {
e = _error;
deferred.reject(makeError(InvalidJSON, iter_id, id, "invalid JSON"));
return;
}
if (Crawler2.ItemValidate(item)) {
return deferred.resolve(item);
} else {
return deferred.reject(makeError(InvalidItem, iter_id, id, "invalid data in the item"));
}
})["catch"](function(err) {
return deferred.reject(err);
}).done();
};
Crawler2.prototype.fetch_item = function(iter_id, id) {
var deferred;
deferred = Q.defer();
this.limiter.removeTokens(1, (function(_this) {
return function(lim_err, remainingRequests) {
return _this.fetch_item_bare(iter_id, id, deferred);
};
})(this));
return deferred.promise;
};
return Crawler2;
})();
module.exports = Crawler2;
}).call(this);