bulk-proxy-tester/start.js

Test proxies in bulk

'use strict';

const request = require('request-promise-native');
const fs = require('fs');

const filterOptions = JSON.parse(fs.readFileSync(__dirname + '/config/options.json', 'utf8'));
const {batchAmount, testUrl} = filterOptions;

const proxies = fs.readFileSync(__dirname + '/config/proxies.txt', 'utf8').split('\n');

let tested = 0;
async function test(proxy){
    tested++;
    const {address, port, username, password} = getProxyDetails(proxy, filterOptions.proxyFormat);

    const requestProxy = 'http://' + (username !== null ? username + ':' + password + '@' : '') + address + ':' + port;

    const messageEnd        = ': Proxy ' + proxy + ' ';
    const genericMessage    = 'GENERIC' + messageEnd;
    const failureMessage    = 'FAILURE' + messageEnd;
    const successMessage    = 'SUCCESS' + messageEnd;

    let responseTime = null;

    for(let i = 0; i < filterOptions.tests; i++){
        try{
            const startTime = Date.now();
            const {statusCode, body} = await request({
                url: testUrl,
                proxy: requestProxy,
                resolveWithFullResponse: true,
                timeout: filterOptions.maxResponseTime + 1000
            });

            // step 1: check response time
            const endTime = Date.now();
            responseTime = endTime - startTime;
            if(responseTime > filterOptions.maxResponseTime){
                console.log(failureMessage + 'responded after ' + (responseTime / 1000) + 's');
                return {success: false, responseTime: responseTime};
            }


            // step 2: check status code
            if(filterOptions.acceptedResponseCodes.indexOf(statusCode) === -1){
                console.log(failureMessage + 'has status code ' + statusCode);
                return {success: false, responseTime: responseTime};
            }

            // step 3: check body
            for(let x = 0; x < filterOptions.bodyContains.length; x++){
                const containValue = filterOptions.bodyContains[x];
                if(body.indexOf(containValue) === -1){
                    console.log(failureMessage + 'does not contain "' + containValue + '" in body');
                    return {success: false, responseTime: responseTime};
                }
            }

            console.log(successMessage + 'succeeded at attempt ' + (i + 1) + '/' + filterOptions.tests);
        }catch(error){
            // step 0?
            console.log(failureMessage + 'threw an error');
            return {success: false, responseTime: responseTime};
        }
        await new Promise((resolve)=>{setTimeout(resolve, filterOptions.testsDelay);});
    }

    return {success: true, responseTime: responseTime};
}

async function run(){
    const filteredProxies = [];
    const filteredProxiesWithResponseTime = [];
    let proxyIndex = 0;
    let finished = false;
    while(!finished){
        const promises = [];
        for(let i = proxyIndex; i < proxyIndex + batchAmount; i++){
            const proxy = proxies[proxyIndex++];
            if(proxy === undefined){
                finished = true;
                break;
            }
            const promise = test(proxy);
            promises.push(promise);
            promise.then((result)=>{
                const {success, responseTime} = result;
                if(success){
                    filteredProxies.push(proxy);
                    filteredProxiesWithResponseTime.push({proxy: proxy, responseTime: responseTime});
                }
            });
        }
        await Promise.all(promises);
    }

    fs.writeFileSync(__dirname + '/config/filtered-proxies.txt', filteredProxies.join('\n'), 'utf8');
    console.log('Finished. Filtered proxies in filtered-proxies.txt\nSummary:');

    const orderedFilteredProxiesWithResponseTime = filteredProxiesWithResponseTime.sort((a, b)=>{return a.responseTime - b.responseTime;});
    for(let i = 0; i < orderedFilteredProxiesWithResponseTime.length; i++){
        const {proxy, responseTime} = orderedFilteredProxiesWithResponseTime[i];
        console.log((responseTime / 1000).toFixed(6) + ': ' + proxy);
    }
    console.log('Tested in total: ' + tested);
}
run();

// identifier(s) should be in the following format:
// {
//     resultName: 'name of key that will be used in the returned object',
//     identifier: 'identifier that will be searched for in the format'
// }
function breakDown(string, format, identifiers){
    for(let i = 0; i < identifiers.length; i++){
        const identifier = identifiers[i];
        identifier.index = format.indexOf(identifier.identifier);
    }
    identifiers = identifiers.sort((a, b)=>{return a.index - b.index})          // sort the indexes in ascending order
                             .filter((object)=>{return object.index !== -1});   // remove the indexes that weren't found (not found means index value of -1)

    if(identifiers.length > 0){
        identifiers[0].leftSideCharacters = format.slice(0, identifiers[0].index);
        for(let i = 0; i < identifiers.length; i++){ // this will handle all the characters between the identifiers
            const {identifier, index} = identifiers[i];

            if(i + 1 !== identifiers.length){ // this is not the last array item
                const nextIndex = identifiers[i + 1].index; // we can use i + 1 because it will never return undefined because we skip the last item
                const characters = format.slice(index + identifier.length, nextIndex);
                identifiers[i].rightSideCharacters = characters;
                identifiers[i + 1].leftSideCharacters = characters;
            }
        }
        const previousItem = identifiers[identifiers.length - 1];
        previousItem.rightSideCharacters = format.slice(previousItem.index + previousItem.identifier.length);
    }

    let finished = false;
    const brokenDown = {};
    for(let i = 0; i < identifiers.length; i++){
        const item = identifiers[i];
        const {resultName, leftSideCharacters, rightSideCharacters} = identifiers[i];
        const previousItem = identifiers[i - 1];

        const leftSideCharactersIndex = previousItem === undefined ? leftSideCharacters.length : previousItem.rightSideCharactersIndex + previousItem.rightSideCharacters.length;
        let rightSideCharactersIndex = rightSideCharacters === '' ? string.length : string.indexOf(rightSideCharacters, leftSideCharactersIndex);
        if(rightSideCharactersIndex === -1){
            rightSideCharactersIndex = string.length;
        }
        item.rightSideCharactersIndex = rightSideCharactersIndex;

        if(finished){
            brokenDown[resultName] = null;
            continue;
        }

        brokenDown[resultName] = string.slice(leftSideCharactersIndex, rightSideCharactersIndex);

        if(rightSideCharactersIndex === string.length){
            finished = true;
        }
    }

    return brokenDown;
}

function getProxyDetails(proxy, format){
    return breakDown(proxy, format, [
        {
            resultName: 'address',
            identifier: '#address#'
        },
        {
            resultName: 'port',
            identifier: '#port#'
        },
        {
            resultName: 'username',
            identifier: '#username#'
        },
        {
            resultName: 'password',
            identifier: '#password#'
        }
    ]);
}