microfrontier
Version:
A web crawler frontier implementation in TypeScript backed by Redis. MicroFrontier is a scalable and distributed frontier implemented through Redis Queues.
254 lines (253 loc) • 10.2 kB
JavaScript
import regeneratorRuntime from "regenerator-runtime";
import * as fs from 'fs';
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
try {
var info = gen[key](arg);
var value = info.value;
} catch (error) {
reject(error);
return;
}
if (info.done) {
resolve(value);
} else {
Promise.resolve(value).then(_next, _throw);
}
}
function _asyncToGenerator(fn) {
return function() {
var self = this, args = arguments;
return new Promise(function(resolve, reject) {
var gen = fn.apply(self, args);
function _next(value) {
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
}
function _throw(err) {
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
}
_next(undefined);
});
};
}
function _classCallCheck(instance, Constructor) {
if (!(instance instanceof Constructor)) {
throw new TypeError("Cannot call a class as a function");
}
}
function _defineProperties(target, props) {
for(var i = 0; i < props.length; i++){
var descriptor = props[i];
descriptor.enumerable = descriptor.enumerable || false;
descriptor.configurable = true;
if ("value" in descriptor) descriptor.writable = true;
Object.defineProperty(target, descriptor.key, descriptor);
}
}
function _createClass(Constructor, protoProps, staticProps) {
if (protoProps) _defineProperties(Constructor.prototype, protoProps);
if (staticProps) _defineProperties(Constructor, staticProps);
return Constructor;
}
function _defineProperty(obj, key, value) {
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
}
return obj;
}
function _objectSpread(target) {
for(var i = 1; i < arguments.length; i++){
var source = arguments[i] != null ? arguments[i] : {
};
var ownKeys = Object.keys(source);
if (typeof Object.getOwnPropertySymbols === "function") {
ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
return Object.getOwnPropertyDescriptor(source, sym).enumerable;
}));
}
ownKeys.forEach(function(key) {
_defineProperty(target, key, source[key]);
});
}
return target;
}
export var UrlFrontier = /*#__PURE__*/ function() {
"use strict";
function UrlFrontier(redisClient, config) {
_classCallCheck(this, UrlFrontier);
this.defaultConfig = {
frontierName: 'frontier',
priorities: {
'high': {
probability: 0.6
},
'normal': {
probability: 0.3
},
'low': {
probability: 0.1
}
},
frontendQueueWorkers: 1,
backendQueueWorkers: 1,
defaultCrawlDelay: 1000
};
this.config = _objectSpread({
}, this.defaultConfig, config ? config : {
});
this.redisClient = redisClient;
var zfetchpostponeScript = fs.readFileSync(__dirname + '/LUA/zfetchpostpone.lua', 'utf8');
zfetchpostponeScript.toString();
this.redisClient.defineCommand("zfetchpostpone", {
numberOfKeys: 1,
lua: zfetchpostponeScript.toString()
});
}
_createClass(UrlFrontier, [
{
key: "add",
value: /**
* Add an URL to the frontier Front-end queue
* @param {string} url - URL to be added
* @param {string} priority - Priority queue name
*/ function add(url, priority, meta) {
return _asyncToGenerator(regeneratorRuntime.mark(function _callee() {
var queueName;
return regeneratorRuntime.wrap(function _callee$(_ctx) {
while(1)switch(_ctx.prev = _ctx.next){
case 0:
if (priority in this.config.priorities) {
_ctx.next = 2;
break;
}
throw new Error('Wrong priority specified');
case 2:
queueName = UrlFrontier.getFrontendQueueName(this.config, priority);
_ctx.next = 5;
return this.redisClient.lpush(queueName, JSON.stringify({
url: url,
meta: meta
}));
case 5:
case "end":
return _ctx.stop();
}
}, _callee, this);
}).bind(this))();
}
},
{
key: "setHostnameCrawlDelay",
value: /**
* Set a hostname crawl delay
* @param {string} hostname - eg. www.google.com
* @param {number} delay - In milliseconds
*/ function setHostnameCrawlDelay(hostname, delay) {
return _asyncToGenerator(regeneratorRuntime.mark(function _callee() {
return regeneratorRuntime.wrap(function _callee$(_ctx) {
while(1)switch(_ctx.prev = _ctx.next){
case 0:
_ctx.next = 2;
return this.redisClient.hset(UrlFrontier.getHostnameCrawlDelayName(this.config), hostname, delay);
case 2:
return _ctx.abrupt("return", _ctx.sent);
case 3:
case "end":
return _ctx.stop();
}
}, _callee, this);
}).bind(this))();
}
},
{
key: "get",
value: // TODO: Failsafe, now if something in the GET fails we risk to lose the host in the heap or having wrong host counter
function get() {
return _asyncToGenerator(regeneratorRuntime.mark(function _callee() {
var now, nextCrawlTime, hostToBeFetched, host, backPopped, parsedUrl;
return regeneratorRuntime.wrap(function _callee$(_ctx) {
while(1)switch(_ctx.prev = _ctx.next){
case 0:
now = new Date().getTime();
nextCrawlTime = now + this.config.defaultCrawlDelay;
_ctx.next = 4;
return this.redisClient.zfetchpostpone(UrlFrontier.getHeapName(this.config), now, nextCrawlTime, UrlFrontier.getHostnameCrawlDelayName(this.config));
case 4:
hostToBeFetched = _ctx.sent;
if (!(!hostToBeFetched || hostToBeFetched.length == 0)) {
_ctx.next = 7;
break;
}
return _ctx.abrupt("return", null);
case 7:
var ref;
ref = hostToBeFetched, host = ref[0], ref;
_ctx.next = 10;
return this.redisClient.rpop(UrlFrontier.getBackendQueueName(this.config, host));
case 10:
backPopped = _ctx.sent;
if (backPopped) {
_ctx.next = 13;
break;
}
return _ctx.abrupt("return", null);
case 13:
_ctx.next = 15;
return this.redisClient.zincrby(UrlFrontier.getHostnameUrlCountName(this.config), -1, host) // Decrement by one the hostname urls
;
case 15:
parsedUrl = JSON.parse(backPopped);
return _ctx.abrupt("return", parsedUrl);
case 17:
case "end":
return _ctx.stop();
}
}, _callee, this);
}).bind(this))();
}
}
], [
{
key: "getFrontendQueueName",
value: /**
* Get redis key name for queue+priority
* @param {string} priority
*/ function getFrontendQueueName(config, priority) {
// Check if priority queue exists
if (!(priority in config.priorities)) throw new Error('Wrong priority specified');
return config.frontierName + ':priority:' + priority;
}
},
{
key: "getBackendQueueName",
value: function getBackendQueueName(config, hostname) {
return config.frontierName + ':' + hostname;
}
},
{
key: "getHeapName",
value: function getHeapName(config) {
return config.frontierName + ':heap';
}
},
{
key: "getHostnameUrlCountName",
value: function getHostnameUrlCountName(config) {
return config.frontierName + ':hostnameUrls';
}
},
{
key: "getHostnameCrawlDelayName",
value: function getHostnameCrawlDelayName(config) {
return config.frontierName + ':crawlDelays';
}
}
]);
return UrlFrontier;
}();