microfrontier
Version:
A web crawler frontier implementation in TypeScript backed by Redis. MicroFrontier is a scalable and distributed frontier implemented through Redis Queues.
94 lines (93 loc) • 12 kB
JavaScript
;
var nconf = require("nconf");
var _micro = require("micro");
var MR = require("microrouter");
var _urlfrontier = require("./URLFrontier");
var _workerThreads = require("worker_threads");
nconf.env({
parseValues: true,
separator: '_'
}).argv();
const { get , post } = MR;
const frontier = new _urlfrontier.URLFrontier({
config: nconf.get('config') ? nconf.get('config') : null,
redisOptions: nconf.get('redis') ? nconf.get('redis') : {
}
});
var frontendWorkerPool;
/*
* Initialize frontend workers
*/ var frontendWorkers = [];
const FEW = process.env.FRONTEND_WORKERS ? process.env.FRONTEND_WORKERS : 1;
for(var i1 = 0; i1 < FEW; i1++){
const worker = new _workerThreads.Worker(__dirname + '/FrontendWorkerPool.js', {
workerData: {
config: frontier.config,
redisOptions: nconf.get('redis') ? nconf.get('redis') : {
}
}
});
frontendWorkers.push(worker);
}
const postFrontier = async (req, res)=>{
const body = await (0, _micro).json(req);
try {
await frontier.add(body.url, body.priority, body.meta);
(0, _micro).send(res, 200, {
status: 'OK'
});
} catch (e) {
(0, _micro).send(res, 500, {
status: 'ERROR',
error: e
});
}
};
const getFrontier = async (req, res)=>{
try {
const data = await frontier.get();
(0, _micro).send(res, 200, {
status: 'OK',
data
});
} catch (e) {
console.error(e);
(0, _micro).send(res, 500, {
status: 'ERROR',
error: e
});
}
};
const postFrontendWorkers = async (req, res)=>{
const body = await (0, _micro).json(req);
if (frontendWorkers.length > 0 || body.workers === 0) {
for (var wk of frontendWorkers){
await wk.terminate();
}
frontendWorkers = [];
}
for(var i = 0; i < body.workers; i++){
const worker = new _workerThreads.Worker(__dirname + '/FrontendWorkerPool.js', {
workerData: {
config: frontier.config,
redisOptions: nconf.get('redis') ? nconf.get('redis') : {
}
}
});
frontendWorkers.push(worker);
}
(0, _micro).send(res, 200, {
status: 'OK',
activeWorkers: frontendWorkers.length
});
};
const notfound = (req, res)=>(0, _micro).send(res, 404, 'Not found')
;
const server = (0, _micro).default((0, MR).router(post('/frontier', postFrontier), get('/frontier', getFrontier), post('/frontend-workers', postFrontendWorkers), get('/*', notfound)));
const port = nconf.get('port') ? nconf.get('port') : 8090;
const host = nconf.get('host') ? nconf.get('host') : '127.0.0.1';
server.listen(port, host);
server.addListener('listening', ()=>{
console.log(`Server Started on ${host}:${port}`);
});
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIi4uL3NyYy9NaWNyb1NlcnZlci50cyJdLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgKiBhcyBuY29uZiBmcm9tICduY29uZic7XG5uY29uZi5lbnYoe3BhcnNlVmFsdWVzOiB0cnVlLCBzZXBhcmF0b3I6J18nfSkuYXJndigpO1xuaW1wb3J0IHsgc2VuZCwganNvbiB9IGZyb20gJ21pY3JvJ1xuaW1wb3J0IHsgcm91dGVyIH0gZnJvbSAnbWljcm9yb3V0ZXInXG5pbXBvcnQgKiBhcyBNUiBmcm9tICdtaWNyb3JvdXRlcic7XG5pbXBvcnQgeyBVUkxGcm9udGllciB9IGZyb20gJy4vVVJMRnJvbnRpZXInXG5pbXBvcnQgbWljcm8gZnJvbSAnbWljcm8nO1xuaW1wb3J0IHsgV29ya2VyIH0gZnJvbSAnd29ya2VyX3RocmVhZHMnXG5cbmNvbnN0IHsgZ2V0LCBwb3N0IH0gPSBNUiBhcyBhbnk7XG5cbmNvbnN0IGZyb250aWVyID0gbmV3IFVSTEZyb250aWVyKHtcbiAgICBjb25maWc6IG5jb25mLmdldCgnY29uZmlnJykgPyBuY29uZi5nZXQoJ2NvbmZpZycpIDogbnVsbCxcbiAgICByZWRpc09wdGlvbnM6IG5jb25mLmdldCgncmVkaXMnKSA_IG5jb25mLmdldCgncmVkaXMnKSA6IHt9XG59KTtcbnZhciBmcm9udGVuZFdvcmtlclBvb2xcblxuLypcbiAqIEluaXRpYWxpemUgZnJvbnRlbmQgd29ya2Vyc1xuICovXG52YXIgZnJvbnRlbmRXb3JrZXJzID0gW11cbmNvbnN0IEZFVyA9IHByb2Nlc3MuZW52LkZST05URU5EX1dPUktFUlMgPyBwcm9jZXNzLmVudi5GUk9OVEVORF9XT1JLRVJTIDogMVxuZm9yKHZhciBpID0gMDsgaSA8IEZFVyA7IGkrKyl7XG4gICAgY29uc3Qgd29ya2VyID0gbmV3IFdvcmtlcihfX2Rpcm5hbWUgKyAnL0Zyb250ZW5kV29ya2VyUG9vbC5qcycsIHtcbiAgICAgICAgd29ya2VyRGF0YToge1xuICAgICAgICAgICAgY29uZmlnOiBmcm9udGllci5jb25maWcsXG4gICAgICAgICAgICByZWRpc09wdGlvbnM6IG5jb25mLmdldCgncmVkaXMnKSA_IG5jb25mLmdldCgncmVkaXMnKSA6IHt9XG4gICAgICAgIH1cbiAgICB9KTtcbiAgICBmcm9udGVuZFdvcmtlcnMucHVzaCh3b3JrZXIpO1xufVxuXG5cbmNvbnN0IHBvc3RGcm9udGllciA9IGFzeW5jIChyZXEsIHJlcykgPT4ge1xuICAgIGNvbnN0IGJvZHkgPSBhd2FpdCBqc29uKHJlcSlcbiAgICB0cnkge1xuICAgICAgICBhd2FpdCBmcm9udGllci5hZGQoYm9keS51cmwsIGJvZHkucHJpb3JpdHksIGJvZHkubWV0YSk7XG4gICAgICAgIHNlbmQocmVzLCAyMDAsIHsgc3RhdHVzOiAnT0snIH0pXG4gICAgfSBjYXRjaCAoZSkge1xuXG4gICAgICAgIHNlbmQocmVzLCA1MDAsIHsgc3RhdHVzOiAnRVJST1InLCBlcnJvcjogZSB9KVxuICAgIH1cbn1cblxuY29uc3QgZ2V0RnJvbnRpZXIgPSBhc3luYyAocmVxLCByZXMpID0-IHtcbiAgICB0cnkge1xuICAgICAgICBjb25zdCBkYXRhID0gYXdhaXQgZnJvbnRpZXIuZ2V0KClcbiAgICAgICAgc2VuZChyZXMsIDIwMCwgeyBzdGF0dXM6ICdPSycsIGRhdGEgfSlcbiAgICB9IGNhdGNoIChlKSB7XG4gICAgICAgIGNvbnNvbGUuZXJyb3IoZSlcbiAgICAgICAgc2VuZChyZXMsIDUwMCwgeyBzdGF0dXM6ICdFUlJPUicsIGVycm9yOiBlIH0pXG4gICAgfVxufVxuXG5jb25zdCBwb3N0RnJvbnRlbmRXb3JrZXJzID0gYXN5bmMgKHJlcSwgcmVzKSA9PiB7XG4gICAgY29uc3QgYm9keSA9IGF3YWl0IGpzb24ocmVxKVxuXG4gICAgaWYoZnJvbnRlbmRXb3JrZXJzLmxlbmd0aCA-IDAgfHwgYm9keS53b3JrZXJzID09PSAwKXtcbiAgICAgICAgZm9yKHZhciB3ayBvZiBmcm9udGVuZFdvcmtlcnMpe1xuICAgICAgICAgICAgYXdhaXQgd2sudGVybWluYXRlKCk7XG4gICAgICAgIH1cbiAgICAgICAgZnJvbnRlbmRXb3JrZXJzID0gW11cbiAgICB9XG5cbiAgICBmb3IodmFyIGkgPSAwOyBpIDwgYm9keS53b3JrZXJzOyBpKyspe1xuICAgICAgICBjb25zdCB3b3JrZXIgPSBuZXcgV29ya2VyKF9fZGlybmFtZSArICcvRnJvbnRlbmRXb3JrZXJQb29sLmpzJywge1xuICAgICAgICAgICAgd29ya2VyRGF0YToge1xuICAgICAgICAgICAgICAgIGNvbmZpZzogZnJvbnRpZXIuY29uZmlnLFxuICAgICAgICAgICAgICAgIHJlZGlzT3B0aW9uczogbmNvbmYuZ2V0KCdyZWRpcycpID8gbmNvbmYuZ2V0KCdyZWRpcycpIDoge31cbiAgICAgICAgICAgIH1cbiAgICAgICAgfSk7XG4gICAgICAgIGZyb250ZW5kV29ya2Vycy5wdXNoKHdvcmtlcik7XG4gICAgfVxuICAgIFxuICAgIHNlbmQocmVzLCAyMDAsIHsgc3RhdHVzOiAnT0snLCBhY3RpdmVXb3JrZXJzOiBmcm9udGVuZFdvcmtlcnMubGVuZ3RoIH0pXG59XG5cbmNvbnN0IG5vdGZvdW5kID0gKHJlcSwgcmVzKSA9PiBzZW5kKHJlcywgNDA0LCAnTm90IGZvdW5kJylcblxuXG5jb25zdCBzZXJ2ZXIgPSBtaWNybyhyb3V0ZXIoXG4gICAgcG9zdCgnL2Zyb250aWVyJywgcG9zdEZyb250aWVyKSxcbiAgICBnZXQoJy9mcm9udGllcicsIGdldEZyb250aWVyKSxcbiAgICBwb3N0KCcvZnJvbnRlbmQtd29ya2VycycsIHBvc3RGcm9udGVuZFdvcmtlcnMpLFxuICAgIGdldCgnLyonLCBub3Rmb3VuZClcbikpXG5cbmNvbnN0IHBvcnQgPSBuY29uZi5nZXQoJ3BvcnQnKSA_IG5jb25mLmdldCgncG9ydCcpIDogODA5MFxuY29uc3QgaG9zdCA9IG5jb25mLmdldCgnaG9zdCcpID8gbmNvbmYuZ2V0KCdob3N0JykgOiAnMTI3LjAuMC4xJ1xuc2VydmVyLmxpc3Rlbihwb3J0LCBob3N0KVxuc2VydmVyLmFkZExpc3RlbmVyKCdsaXN0ZW5pbmcnLCAoKSA9PiB7XG4gICAgY29uc29sZS5sb2coYFNlcnZlciBTdGFydGVkIG9uICR7aG9zdH06JHtwb3J0fWApXG59KSJdLCJuYW1lcyI6WyJuY29uZiIsImVudiIsInBhcnNlVmFsdWVzIiwic2VwYXJhdG9yIiwiYXJndiIsImdldCIsInBvc3QiLCJmcm9udGllciIsImNvbmZpZyIsInJlZGlzT3B0aW9ucyIsImZyb250ZW5kV29ya2VyUG9vbCIsImZyb250ZW5kV29ya2VycyIsIkZFVyIsInByb2Nlc3MiLCJGUk9OVEVORF9XT1JLRVJTIiwiaSIsIndvcmtlciIsIl9fZGlybmFtZSIsIndvcmtlckRhdGEiLCJwdXNoIiwicG9zdEZyb250aWVyIiwicmVxIiwicmVzIiwiYm9keSIsImFkZCIsInVybCIsInByaW9yaXR5IiwibWV0YSIsInN0YXR1cyIsImUiLCJlcnJvciIsImdldEZyb250aWVyIiwiZGF0YSIsImNvbnNvbGUiLCJwb3N0RnJvbnRlbmRXb3JrZXJzIiwibGVuZ3RoIiwid29ya2VycyIsIndrIiwidGVybWluYXRlIiwiYWN0aXZlV29ya2VycyIsIm5vdGZvdW5kIiwic2VydmVyIiwicG9ydCIsImhvc3QiLCJsaXN0ZW4iLCJhZGRMaXN0ZW5lciIsImxvZyJdLCJtYXBwaW5ncyI6IjtBQUFZQSxHQUFLLENBQUxBLEtBQUs7QUFFVSxHQUFPLENBQVAsTUFBTztBQUNYLEdBQWEsQ0FBYixFQUFhO0FBRVIsR0FBZSxDQUFmLFlBQWU7QUFFcEIsR0FBZ0IsQ0FBaEIsY0FBZ0I7QUFQM0JBLEtBQUssQ0FDWEMsR0FBRyxDQUFDLENBQUNDO0lBQUFBLFdBQVcsRUFBRSxJQUFJO0lBQUVDLFNBQVMsRUFBQyxDQUFHO0FBQUEsQ0FBQyxFQUFFQyxJQUFJO0FBUWxELEtBQUssQ0FBQyxDQUFDLENBQUNDLEdBQUcsR0FBRUMsSUFBSSxFQUFDLENBQUMsR0FOSSxFQUFhO0FBUXBDLEtBQUssQ0FBQ0MsUUFBUSxHQUFHLEdBQUcsQ0FOUSxZQUFlLGFBTVYsQ0FBQztJQUM5QkMsTUFBTSxFQVpFUixLQUFLLENBWUNLLEdBQUcsQ0FBQyxDQUFRLFdBWmxCTCxLQUFLLENBWXVCSyxHQUFHLENBQUMsQ0FBUSxXQUFJLElBQUk7SUFDeERJLFlBQVksRUFiSlQsS0FBSyxDQWFPSyxHQUFHLENBQUMsQ0FBTyxVQWJ2QkwsS0FBSyxDQWE0QkssR0FBRyxDQUFDLENBQU8sVUFBSSxDQUFDO0lBQUEsQ0FBQztBQUM5RCxDQUFDO0FBQ0QsR0FBRyxDQUFDSyxrQkFBa0I7QUFFdEIsRUFFRyxBQUZIOztDQUVHLEFBRkgsRUFFRyxDQUNILEdBQUcsQ0FBQ0MsZUFBZSxHQUFHLENBQUMsQ0FBQztBQUN4QixLQUFLLENBQUNDLEdBQUcsR0FBR0MsT0FBTyxDQUFDWixHQUFHLENBQUNhLGdCQUFnQixHQUFHRCxPQUFPLENBQUNaLEdBQUcsQ0FBQ2EsZ0JBQWdCLEdBQUcsQ0FBQztBQUMzRSxHQUFHLENBQUMsR0FBRyxDQUFDQyxFQUFDLEdBQUcsQ0FBQyxFQUFFQSxFQUFDLEdBQUdILEdBQUcsRUFBR0csRUFBQyxHQUFHLENBQUM7SUFDMUIsS0FBSyxDQUFDQyxNQUFNLEdBQUcsR0FBRyxDQWhCQyxjQUFnQixRQWdCVEMsU0FBUyxHQUFHLENBQXdCLHlCQUFFLENBQUM7UUFDN0RDLFVBQVUsRUFBRSxDQUFDO1lBQ1RWLE1BQU0sRUFBRUQsUUFBUSxDQUFDQyxNQUFNO1lBQ3ZCQyxZQUFZLEVBMUJaVCxLQUFLLENBMEJlSyxHQUFHLENBQUMsQ0FBTyxVQTFCL0JMLEtBQUssQ0EwQm9DSyxHQUFHLENBQUMsQ0FBTyxVQUFJLENBQUM7WUFBQSxDQUFDO1FBQzlELENBQUM7SUFDTCxDQUFDO0lBQ0RNLGVBQWUsQ0FBQ1EsSUFBSSxDQUFDSCxNQUFNO0FBQy9CLENBQUM7QUFHRCxLQUFLLENBQUNJLFlBQVksVUFBVUMsR0FBRyxFQUFFQyxHQUFHLEdBQUssQ0FBQztJQUN0QyxLQUFLLENBQUNDLElBQUksR0FBRyxLQUFLLEtBaENLLE1BQU8sT0FnQ05GLEdBQUc7SUFDM0IsR0FBRyxDQUFDLENBQUM7UUFDRCxLQUFLLENBQUNkLFFBQVEsQ0FBQ2lCLEdBQUcsQ0FBQ0QsSUFBSSxDQUFDRSxHQUFHLEVBQUVGLElBQUksQ0FBQ0csUUFBUSxFQUFFSCxJQUFJLENBQUNJLElBQUk7WUFsQ2xDLE1BQU8sT0FtQ3JCTCxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQUM7WUFBQ00sTUFBTSxFQUFFLENBQUk7UUFBQyxDQUFDO0lBQ25DLENBQUMsQ0FBQyxLQUFLLEVBQUVDLENBQUMsRUFBRSxDQUFDO1lBcENVLE1BQU8sT0FzQ3JCUCxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQUM7WUFBQ00sTUFBTSxFQUFFLENBQU87WUFBRUUsS0FBSyxFQUFFRCxDQUFDO1FBQUMsQ0FBQztJQUNoRCxDQUFDO0FBQ0wsQ0FBQztBQUVELEtBQUssQ0FBQ0UsV0FBVyxVQUFVVixHQUFHLEVBQUVDLEdBQUcsR0FBSyxDQUFDO0lBQ3JDLEdBQUcsQ0FBQyxDQUFDO1FBQ0QsS0FBSyxDQUFDVSxJQUFJLEdBQUcsS0FBSyxDQUFDekIsUUFBUSxDQUFDRixHQUFHO1lBNUNaLE1BQU8sT0E2Q3JCaUIsR0FBRyxFQUFFLEdBQUcsRUFBRSxDQUFDO1lBQUNNLE1BQU0sRUFBRSxDQUFJO1lBQUVJLElBQUk7UUFBQyxDQUFDO0lBQ3pDLENBQUMsQ0FBQyxLQUFLLEVBQUVILENBQUMsRUFBRSxDQUFDO1FBQ1RJLE9BQU8sQ0FBQ0gsS0FBSyxDQUFDRCxDQUFDO1lBL0NJLE1BQU8sT0FnRHJCUCxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQUM7WUFBQ00sTUFBTSxFQUFFLENBQU87WUFBRUUsS0FBSyxFQUFFRCxDQUFDO1FBQUMsQ0FBQztJQUNoRCxDQUFDO0FBQ0wsQ0FBQztBQUVELEtBQUssQ0FBQ0ssbUJBQW1CLFVBQVViLEdBQUcsRUFBRUMsR0FBRyxHQUFLLENBQUM7SUFDN0MsS0FBSyxDQUFDQyxJQUFJLEdBQUcsS0FBSyxLQXJESyxNQUFPLE9BcURORixHQUFHO0lBRTNCLEVBQUUsRUFBQ1YsZUFBZSxDQUFDd0IsTUFBTSxHQUFHLENBQUMsSUFBSVosSUFBSSxDQUFDYSxPQUFPLEtBQUssQ0FBQyxFQUFDLENBQUM7UUFDakQsR0FBRyxFQUFDLEdBQUcsQ0FBQ0MsRUFBRSxJQUFJMUIsZUFBZSxDQUFDLENBQUM7WUFDM0IsS0FBSyxDQUFDMEIsRUFBRSxDQUFDQyxTQUFTO1FBQ3RCLENBQUM7UUFDRDNCLGVBQWUsR0FBRyxDQUFDLENBQUM7SUFDeEIsQ0FBQztJQUVELEdBQUcsQ0FBQyxHQUFHLENBQUNJLENBQUMsR0FBRyxDQUFDLEVBQUVBLENBQUMsR0FBR1EsSUFBSSxDQUFDYSxPQUFPLEVBQUVyQixDQUFDLEdBQUcsQ0FBQztRQUNsQyxLQUFLLENBQUNDLE1BQU0sR0FBRyxHQUFHLENBMURILGNBQWdCLFFBMERMQyxTQUFTLEdBQUcsQ0FBd0IseUJBQUUsQ0FBQztZQUM3REMsVUFBVSxFQUFFLENBQUM7Z0JBQ1RWLE1BQU0sRUFBRUQsUUFBUSxDQUFDQyxNQUFNO2dCQUN2QkMsWUFBWSxFQXBFaEJULEtBQUssQ0FvRW1CSyxHQUFHLENBQUMsQ0FBTyxVQXBFbkNMLEtBQUssQ0FvRXdDSyxHQUFHLENBQUMsQ0FBTyxVQUFJLENBQUM7Z0JBQUEsQ0FBQztZQUM5RCxDQUFDO1FBQ0wsQ0FBQztRQUNETSxlQUFlLENBQUNRLElBQUksQ0FBQ0gsTUFBTTtJQUMvQixDQUFDO1FBdEVzQixNQUFPLE9Bd0V6Qk0sR0FBRyxFQUFFLEdBQUcsRUFBRSxDQUFDO1FBQUNNLE1BQU0sRUFBRSxDQUFJO1FBQUVXLGFBQWEsRUFBRTVCLGVBQWUsQ0FBQ3dCLE1BQU07SUFBQyxDQUFDO0FBQzFFLENBQUM7QUFFRCxLQUFLLENBQUNLLFFBQVEsSUFBSW5CLEdBQUcsRUFBRUMsR0FBRyxPQTNFQyxNQUFPLE9BMkVFQSxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQVc7O0FBR3pELEtBQUssQ0FBQ21CLE1BQU0sT0E5RWUsTUFBTyxjQUNYLEVBQWEsU0E4RWhDbkMsSUFBSSxDQUFDLENBQVcsWUFBRWMsWUFBWSxHQUM5QmYsR0FBRyxDQUFDLENBQVcsWUFBRTBCLFdBQVcsR0FDNUJ6QixJQUFJLENBQUMsQ0FBbUIsb0JBQUU0QixtQkFBbUIsR0FDN0M3QixHQUFHLENBQUMsQ0FBSSxLQUFFbUMsUUFBUTtBQUd0QixLQUFLLENBQUNFLElBQUksR0F2RkUxQyxLQUFLLENBdUZFSyxHQUFHLENBQUMsQ0FBTSxTQXZGakJMLEtBQUssQ0F1RnNCSyxHQUFHLENBQUMsQ0FBTSxTQUFJLElBQUk7QUFDekQsS0FBSyxDQUFDc0MsSUFBSSxHQXhGRTNDLEtBQUssQ0F3RkVLLEdBQUcsQ0FBQyxDQUFNLFNBeEZqQkwsS0FBSyxDQXdGc0JLLEdBQUcsQ0FBQyxDQUFNLFNBQUksQ0FBVztBQUNoRW9DLE1BQU0sQ0FBQ0csTUFBTSxDQUFDRixJQUFJLEVBQUVDLElBQUk7QUFDeEJGLE1BQU0sQ0FBQ0ksV0FBVyxDQUFDLENBQVcsZ0JBQVEsQ0FBQztJQUNuQ1osT0FBTyxDQUFDYSxHQUFHLEVBQUUsa0JBQWtCLEVBQUVILElBQUksQ0FBQyxDQUFDLEVBQUVELElBQUk7QUFDakQsQ0FBQyJ9