request-pool/i.ts

An intelligent proxy and request manager.

var request=require('request');
var cheerio=require('cheerio');

interface IServerState{
    alive:boolean;
    total:number;
    fail:number;
    last:number;
}

interface IRequestCallback{
    (err:any,res:any,body:string):void;
}

interface IRequest{
    url:string;
    callback:IRequestCallback;
}

interface IInitCallback{
    (Handler:IHandler)
}

interface IHandler{
    Get:IGetHandler;
    AddProxy:IAddProxyHandler;
    AutoGetProxy:IAutoGetProxyHandler;
}

interface IGetHandler{
    (url:string,callback:IRequestCallback):void;
}

interface IAutoGetProxyHandler{
    ():void;
}

interface IAddProxyHandler{
    (address:string):void;
}

class RequestPool{

    private ServerTimeout:number=3000;
    private ServerInterval:number=1000;

    private ServerList:Map<string,number>=new Map<string,number>();
    private ServerState:Map<string,IServerState>=new Map<string,IServerState>();
    private ServerAvailable:Array<string>=[];
    private ServerUsed:Array<string>=[];
    private ServerToCheck:Array<string>=[];

    private RequestQueue:Array<IRequest>=[];
    private HighPriorityQueue:Array<IRequest>=[];
    private Handler:any;

    constructor(){
        
    }

    public Init(callback:IInitCallback):void{

        var base=this;

        function Get(url:string,callback:IRequestCallback):void{
            base.RequestQueue.unshift({url:url,callback:callback});
        }
    
        function AddProxy(address:string){
            if(base.ServerList[address]==undefined){
                base.ServerList[address]=1;
                var state:IServerState={
                    alive:false,
                    total:0,
                    fail:0,
                    last:0
                }
                base.ServerState[address]=state;
                base.ServerToCheck.unshift(address);
            }
        }
        
        function StartService(service:Function,interval:number){
            setInterval(service, interval);
        }
    
        //Service:CheckServer
    
        function Service_CheckServer():void{
    
            if(base.ServerToCheck.length>0){
                base.ServerToCheck.forEach(function(addr){
                    request("http://www.baidu.com",{proxy:addr,timeout:base.ServerTimeout},function(err,res,body){
                        if(!err&&res.statusCode==200){
                            base.ServerUsed.unshift(addr);
                        }
                    })
                })
                base.ServerToCheck=[];
            }
        }
        //---Service
    
    
    
        //Service:ExecuteRequest
    
        function Service_ExecuteRequest():void{
    
    
            if(base.ServerAvailable.length>0){
                //Have requests and available servers
                var req:IRequest;
    
                if(base.HighPriorityQueue.length>0){
                    req=base.HighPriorityQueue.pop();//get element from right of the queue
                }
                else if(base.RequestQueue.length>0)
                    req=base.RequestQueue.pop();
                else
                    return;//no requests to execute
    
                var addr=base.ServerAvailable.pop();
    
                var url=req.url;
                var cb=req.callback;
    
                request(url,{proxy:addr,timeout:base.ServerTimeout},function(err,res,body){
    
                    if(err=='ETIMEDOUT'||body==undefined){//have some error
                        if(err!='ETIMEDOUT')//server unstable but not dead
                            base.ServerUsed.unshift(addr);
                        else{
                            //server is dead
                            //give up this server
                        }
                        base.HighPriorityQueue.unshift(req);//push in left
                    }
                    else{
                        //success
                        base.ServerState['last']=Date.now();
                        base.ServerUsed.unshift(addr);
                        cb(err,res,body);
                    }
                })
            }
        }
        //---Service
    
    
        //Service:ServerInterval
    
        function Service_ServerInterval():void{
    
            for(var i=0;i<base.ServerUsed.length;i++){
                
                var addr=base.ServerUsed[base.ServerUsed.length-1];
                var last=base.ServerState[addr]['last'];
    
                if(Date.now()-last>base.ServerInterval){
                    base.ServerUsed.pop();
                    base.ServerAvailable.unshift(addr);
                }
                else
                    return;
            }
        }
        //---Service

        setInterval(Service_CheckServer,1000);
        setInterval(Service_ExecuteRequest,50);
        setInterval(Service_ServerInterval,200);

        //GetProxy

        //Auto Get Proxy

        function AutoGetProxy():void{
                //xicidaili
            for(var z=1;z<=5;z++){
    
                request("http://www.xicidaili.com/nn/"+z.toString(),function(error,res,content){
                    if (!error && res.statusCode == 200){
                        var $=cheerio.load(content);
                        var tds=$('tr');
                        
                        tds.each(function(index,element){
                            if(index>=1){
                                var ip="",port="";
                                $(this).children().each(function(i,e){
                                    if(i==1)
                                        ip=$(this).html();
                                    if(i==2)
                                        port=$(this).html();
                                })
                                var addr="http://"+ip+":"+port;
                                AddProxy(addr);//////////////////
                            }
                        })
                    }
                })
            }
    
            //66ip
            var dl1="http://www.66ip.cn/nmtq.php?getnum=8000&isp=0&anonymoustype=3&start=&ports=&export=&ipaddress=&area=1&proxytype=0&api=66ip"        
            var dl2="http://www.66ip.cn/mo.php?sxb=&tqsl=8000&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea="
            
            function getip66(dl){
                request(dl,function(error,res,content){
                    if (!error && res.statusCode == 200){
                        var con2=""+content;
                        var regex = new RegExp("[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}:[0-9]{2,5}","mg");
                        var ips=con2.match(regex);
                        ips.forEach(function(a){
                            var addr="http://"+a;
                            AddProxy(addr);
                        })
    
                    }
                })
            }
    
            getip66(dl1);
            getip66(dl2);
        }
        
        var handler:IHandler={Get:Get,AddProxy:AddProxy,AutoGetProxy:AutoGetProxy}
        callback(handler);
    }
    
}

export=RequestPool;