request-pool
Version:
An intelligent proxy and request manager.
152 lines (151 loc) • 6.43 kB
JavaScript
var request = require('request');
var cheerio = require('cheerio');
var RequestPool = (function () {
function RequestPool() {
this.ServerTimeout = 3000;
this.ServerInterval = 1000;
this.ServerList = new Map();
this.ServerState = new Map();
this.ServerAvailable = [];
this.ServerUsed = [];
this.ServerToCheck = [];
this.RequestQueue = [];
this.HighPriorityQueue = [];
}
RequestPool.prototype.Init = function (callback) {
var base = this;
function Get(url, callback) {
base.RequestQueue.unshift({ url: url, callback: callback });
}
function AddProxy(address) {
if (base.ServerList[address] == undefined) {
base.ServerList[address] = 1;
var state = {
alive: false,
total: 0,
fail: 0,
last: 0
};
base.ServerState[address] = state;
base.ServerToCheck.unshift(address);
}
}
function StartService(service, interval) {
setInterval(service, interval);
}
//Service:CheckServer
function Service_CheckServer() {
if (base.ServerToCheck.length > 0) {
base.ServerToCheck.forEach(function (addr) {
request("http://www.baidu.com", { proxy: addr, timeout: base.ServerTimeout }, function (err, res, body) {
if (!err && res.statusCode == 200) {
base.ServerUsed.unshift(addr);
}
});
});
base.ServerToCheck = [];
}
}
//---Service
//Service:ExecuteRequest
function Service_ExecuteRequest() {
if (base.ServerAvailable.length > 0) {
//Have requests and available servers
var req;
if (base.HighPriorityQueue.length > 0) {
req = base.HighPriorityQueue.pop(); //get element from right of the queue
}
else if (base.RequestQueue.length > 0)
req = base.RequestQueue.pop();
else
return; //no requests to execute
var addr = base.ServerAvailable.pop();
var url = req.url;
var cb = req.callback;
request(url, { proxy: addr, timeout: base.ServerTimeout }, function (err, res, body) {
if (err == 'ETIMEDOUT' || body == undefined) {
if (err != 'ETIMEDOUT')
base.ServerUsed.unshift(addr);
else {
//server is dead
//give up this server
}
base.HighPriorityQueue.unshift(req); //push in left
}
else {
//success
base.ServerState['last'] = Date.now();
base.ServerUsed.unshift(addr);
cb(err, res, body);
}
});
}
}
//---Service
//Service:ServerInterval
function Service_ServerInterval() {
for (var i = 0; i < base.ServerUsed.length; i++) {
var addr = base.ServerUsed[base.ServerUsed.length - 1];
var last = base.ServerState[addr]['last'];
if (Date.now() - last > base.ServerInterval) {
base.ServerUsed.pop();
base.ServerAvailable.unshift(addr);
}
else
return;
}
}
//---Service
setInterval(Service_CheckServer, 1000);
setInterval(Service_ExecuteRequest, 50);
setInterval(Service_ServerInterval, 200);
//GetProxy
//Auto Get Proxy
function AutoGetProxy() {
//xicidaili
for (var z = 1; z <= 5; z++) {
request("http://www.xicidaili.com/nn/" + z.toString(), function (error, res, content) {
if (!error && res.statusCode == 200) {
var $ = cheerio.load(content);
var tds = $('tr');
tds.each(function (index, element) {
if (index >= 1) {
var ip = "", port = "";
$(this).children().each(function (i, e) {
if (i == 1)
ip = $(this).html();
if (i == 2)
port = $(this).html();
});
var addr = "http://" + ip + ":" + port;
AddProxy(addr); //////////////////
}
});
}
});
}
//66ip
var dl1 = "http://www.66ip.cn/nmtq.php?getnum=8000&isp=0&anonymoustype=3&start=&ports=&export=&ipaddress=&area=1&proxytype=0&api=66ip";
var dl2 = "http://www.66ip.cn/mo.php?sxb=&tqsl=8000&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=";
function getip66(dl) {
request(dl, function (error, res, content) {
if (!error && res.statusCode == 200) {
var con2 = "" + content;
var regex = new RegExp("[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}:[0-9]{2,5}", "mg");
var ips = con2.match(regex);
ips.forEach(function (a) {
var addr = "http://" + a;
AddProxy(addr);
});
}
});
}
getip66(dl1);
getip66(dl2);
}
var handler = { Get: Get, AddProxy: AddProxy, AutoGetProxy: AutoGetProxy };
callback(handler);
};
return RequestPool;
}());
module.exports = RequestPool;