request-pool-beta/ii.ts

An intelligent proxy and request manager.

var request=require('request');
var cheerio=require('cheerio');

export interface IServerState{
    alive:boolean;
    total:number;
    fail:number;
    last:number;
}

export interface IRequestCallback{
    (err:any,res:any,body:string):void;
}

export interface IRequest{
    url:string;
    callback:IRequestCallback;
}

export interface IInitCallback{
    (GetHandler:IHandler)
}

export interface IHandler{
    Get:IGetHandler;
    AddProxy:IAddProxyHandler;
}

export interface IGetHandler{
    (url:string,callback:IRequestCallback):void;
}

export interface IAddProxyHandler{
    (address:string):void;
}

export class RequestPool{

    private ServerTimeout:number=3000;
    private ServerInterval:number=1000;

    private ServerList:Map<string,number>=new Map<string,number>();
    private ServerState:Map<string,IServerState>=new Map<string,IServerState>();
    private ServerAvailable:Array<string>=[];
    private ServerUsed:Array<string>=[];
    private ServerToCheck:Array<string>=[];

    private RequestQueue:Array<IRequest>=[];
    private HighPriorityQueue:Array<IRequest>=[];
    private Handler:any;

    constructor(){
        
    }

    public Init(callback:IInitCallback):void{

        var base=this;

        function Get(url:string,callback:IRequestCallback):void{
            base.RequestQueue.unshift({url:url,callback:callback});
        }
    
        function AddProxy(address:string){
            if(base.ServerList[address]==undefined){
                base.ServerList[address]=1;
                var state:IServerState={
                    alive:false,
                    total:0,
                    fail:0,
                    last:0
                }
                base.ServerState[address]=state;
                base.ServerToCheck.unshift(address);
            }
        }
        
        function StartService(service:Function,interval:number){
            setInterval(service, interval);
        }
    
        //Service:CheckServer
    
        function Service_CheckServer():void{
    
            if(base.ServerToCheck.length>0){
                base.ServerToCheck.forEach(function(addr){
                    request("http://www.baidu.com",{proxy:addr,timeout:base.ServerTimeout},function(err,res,body){
                        if(!err&&res.statusCode==200){
                            base.ServerUsed.unshift(addr);
                        }
                    })
                })
                base.ServerToCheck=[];
            }
        }
        //---Service
    
    
    
        //Service:ExecuteRequest
    
        function Service_ExecuteRequest():void{
    
    
            if(base.ServerAvailable.length>0){
                //Have requests and available servers
                var req:IRequest;
    
                if(base.HighPriorityQueue.length>0){
                    req=base.HighPriorityQueue.pop();//get element from right of the queue
                }
                else if(base.RequestQueue.length>0)
                    req=base.RequestQueue.pop();
                else
                    return;//no requests to execute
    
                var addr=base.ServerAvailable.pop();
    
                var url=req.url;
                var cb=req.callback;
    
                request(url,{proxy:addr,timeout:base.ServerTimeout},function(err,res,body){
    
                    if(err=='ETIMEDOUT'||body==undefined){//have some error
                        if(err!='ETIMEDOUT')//server unstable but not dead
                            base.ServerUsed.unshift(addr);
                        else{
                            //server is dead
                            //give up this server
                        }
                        base.HighPriorityQueue.unshift(req);//push in left
                    }
                    else{
                        //success
                        base.ServerState['last']=Date.now();
                        base.ServerUsed.unshift(addr);
                        cb(err,res,body);
                    }
                })
            }
        }
        //---Service
    
    
        //Service:ServerInterval
    
        function Service_ServerInterval():void{
    
            for(var i=0;i<base.ServerUsed.length;i++){
                
                var addr=base.ServerUsed[base.ServerUsed.length-1];
                var last=base.ServerState[addr]['last'];
    
                if(Date.now()-last>base.ServerInterval){
                    base.ServerUsed.pop();
                    base.ServerAvailable.unshift(addr);
                }
                else
                    return;
            }
        }
        //---Service

        setInterval(Service_CheckServer,1000);
        setInterval(Service_ExecuteRequest,50);
        setInterval(Service_ServerInterval,200);

        //GetProxy
        
        
        var handler:IHandler={Get:Get,AddProxy:AddProxy}
        callback(handler);
    }
    
}