request-pool-beta/i.js

An intelligent proxy and request manager.

"use strict";
var request = require('request');
var cheerio = require('cheerio');
var RequestPool = (function () {
    function RequestPool() {
        this.ServerTimeout = 3000;
        this.ServerInterval = 1000;
        this.ServerList = new Map();
        this.ServerState = new Map();
        this.ServerAvailable = [];
        this.ServerUsed = [];
        this.ServerToCheck = [];
        this.RequestQueue = [];
        this.HighPriorityQueue = [];
    }
    RequestPool.prototype.Init = function (callback) {
        var base = this;
        function Get(url, callback) {
            base.RequestQueue.unshift({ url: url, callback: callback });
        }
        function AddProxy(address) {
            if (base.ServerList[address] == undefined) {
                base.ServerList[address] = 1;
                var state = {
                    alive: false,
                    total: 0,
                    fail: 0,
                    last: 0
                };
                base.ServerState[address] = state;
                base.ServerToCheck.unshift(address);
            }
        }
        function StartService(service, interval) {
            setInterval(service, interval);
        }
        //Service:CheckServer
        function Service_CheckServer() {
            if (base.ServerToCheck.length > 0) {
                base.ServerToCheck.forEach(function (addr) {
                    request("http://www.baidu.com", { proxy: addr, timeout: base.ServerTimeout }, function (err, res, body) {
                        if (!err && res.statusCode == 200) {
                            base.ServerUsed.unshift(addr);
                        }
                    });
                });
                base.ServerToCheck = [];
            }
        }
        //---Service
        //Service:ExecuteRequest
        function Service_ExecuteRequest() {
            if (base.ServerAvailable.length > 0) {
                //Have requests and available servers
                var req;
                if (base.HighPriorityQueue.length > 0) {
                    req = base.HighPriorityQueue.pop(); //get element from right of the queue
                }
                else if (base.RequestQueue.length > 0)
                    req = base.RequestQueue.pop();
                else
                    return; //no requests to execute
                var addr = base.ServerAvailable.pop();
                var url = req.url;
                var cb = req.callback;
                request(url, { proxy: addr, timeout: base.ServerTimeout }, function (err, res, body) {
                    if (err == 'ETIMEDOUT' || body == undefined) {
                        if (err != 'ETIMEDOUT')
                            base.ServerUsed.unshift(addr);
                        else {
                            //server is dead
                            //give up this server
                        }
                        base.HighPriorityQueue.unshift(req); //push in left
                    }
                    else {
                        //success
                        base.ServerState['last'] = Date.now();
                        base.ServerUsed.unshift(addr);
                        cb(err, res, body);
                    }
                });
            }
        }
        //---Service
        //Service:ServerInterval
        function Service_ServerInterval() {
            for (var i = 0; i < base.ServerUsed.length; i++) {
                var addr = base.ServerUsed[base.ServerUsed.length - 1];
                var last = base.ServerState[addr]['last'];
                if (Date.now() - last > base.ServerInterval) {
                    base.ServerUsed.pop();
                    base.ServerAvailable.unshift(addr);
                }
                else
                    return;
            }
        }
        //---Service
        setInterval(Service_CheckServer, 1000);
        setInterval(Service_ExecuteRequest, 50);
        setInterval(Service_ServerInterval, 200);
        //GetProxy
        var handler = { Get: Get, AddProxy: AddProxy };
        callback(handler);
    };
    return RequestPool;
}());
exports= RequestPool;