domotz-remote-pawn/src/network_tools/ooklaSpeedTest.js

Domotz Agent

'use strict';

var async = require('async');

var MODULE = "OOKLA-TEST";
var MAX_SPEEDTEST_ATTEMPTS = 4;
var PARALLEL_DOWNLOAD_FAIL = "FALLBACK";

function speedTest() {

    var q = require('q');
    var request = require('requestretry');
    var http = require('http');
    var url = require('url');

    http.globalAgent.maxSockets = Infinity;

    var DOWNLOAD_FILE_SIZES = [1000, 2000, 3000]; // a jpg file of size NxN
    var UPLOAD_FILE_SIZES = [1000000, 2000000, 5000000]; // 1, 2, 5 MBytes

    var MAX_THREADS = 4;
    var MIN_EXP_DURATION = 0.5 * 1e9;

    var RAMP_PERCENTAGE = 10;
    var DEFAULT_TIMEOUT = 20000;
    var MIN_ALLOWED_DOWNLOAD_BIT_RATE = 5 * 1e6;
    var MIN_ALLOWED_UPLOAD_BIT_RATE = 1e6;
    var UPLOAD_FILE_THRESHOLD = 1000;
    var SHORT_TEST_FILE = 'random2000x2000.jpg';

    var USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ' +
        '(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36';

    var commonHeaders = {
        'user-agent': USER_AGENT,
        'Cache-Control': 'no-cache',
        'Pragma': 'no-cache',
        'Referer': 'http://www.speedtest.net/'
    };


    var allTaskStarted = null;
    var accumulator = 0;
    var firstDownloadEnded = false;
    var flows = {};

    function fromIntervalToNs(time) {
        return (time[0] * 1e9) + time[1];
    }

    function generateRandomData(size) {
        var d = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n';
        var c = 1;
        while (d.length < size) {
            d += "a" + c + '=' + d;
            c++;
        }
        return d.substr(0, size);
    }

    function requestToServer(address, cb) {
        console.log("%s - (requestToServer): Sending request to %s", MODULE, address);

        var randomString = (new Date()).getTime();
        var options = {
            url: address + '?x=' + randomString + '&y=1',
            method: 'GET',
            headers: commonHeaders,
            timeout: DEFAULT_TIMEOUT
        };

        var start = process.hrtime();
        request(options, function callback(error, response, body) {
            try {
                if (error) {
                    cb(error);
                    return;
                }

                var statusCode = response.statusCode;
                if (statusCode > 299 || statusCode < 200) {
                    error = {
                        message: body,
                        status: statusCode
                    };
                    cb(error);
                    return;
                }
                var nsLatency = fromIntervalToNs(process.hrtime(start));

                var sample = {
                    size: response.socket.bytesRead,
                    time: nsLatency
                };
                cb(null, sample);

            } catch (err) {
                console.log("%s - (requestToServer): exception: %s", err.message);
            }
        });
    }

    function downloadFile(server, stopIfTooSlow, cb) {
        stopIfTooSlow = typeof stopIfTooSlow !== 'undefined' ? stopIfTooSlow : false;
        console.log("%s - (downloadFile): Downloading file from %s", MODULE, server);

        requestToServer(server, function (err, res) {
            if (err) {
                cb("%s - (downloadFile): Error downloading file: %s", MODULE, err.message);
                return;
            }
            var rate = res.size / (res.time * 1e-9);
            console.log("%s - (downloadFile): HTTP download rate: %s", MODULE, (rate * 1e-6).toFixed(2));
            var result = {
                rate: rate,
                size: res.size,
                server: server,
                time: res.time
            };
            if (stopIfTooSlow && ((rate * 8) < MIN_ALLOWED_DOWNLOAD_BIT_RATE)) {
                console.log("%s - (downloadFile): Stop process, download too slow", MODULE);
                /**
                 * Return an error if the download duration is too high.
                 * Then skip downloading the remaining file sizes.
                 */
                cb(result);
            } else {
                cb(null, result);
            }
        });
    }


    function onChunkReceived(id, len, threads) {
        if (!flows.hasOwnProperty(id)) {
            flows[id] = true;
        }

        if (Object.keys(flows).length === threads) {
            accumulator += len;
            if (allTaskStarted === null) {
                allTaskStarted = process.hrtime();
            }
        }
    }

    function onSingleDownloadEnd(promise, n) {
        if (!firstDownloadEnded) {
            /* the first of the parallel download processes has completed */
            firstDownloadEnded = true;
            if (allTaskStarted !== null) {
                /* all the other processes already reached the ramp up threshold, we compute the final download rate */
                var duration = process.hrtime(allTaskStarted);
                var nsDuration = fromIntervalToNs(duration);
                console.log("%s - (onSingleDownloadEnd): Task #%s. " +
                    "Parallel phase duration = %s s, " +
                    "Download size = %s MB, Rate = %s MB/s.", MODULE, n,
                    (nsDuration * 1e-9).toFixed(2), (accumulator * 1e-6).toFixed(2),
                    ((accumulator * 8) / (nsDuration * 1e-3)).toFixed(2));
                promise.resolve({nsDuration: nsDuration, size: accumulator});
            } else {
                console.log("%s - (onSingleDownloadEnd): Task #%s. Warning, " +
                    "First task ended before all started", MODULE, n);
                promise.reject(PARALLEL_DOWNLOAD_FAIL);
            }
        }
    }

    function singleDownloadProcess(n, address, size, promise, threads, callback) {
        var randomString = (new Date()).getTime();
        var len = 0, partialBytes = 0;
        var startTime = null, partialStartTime = null;
        var isStarted = false;
        var threshold = (size * RAMP_PERCENTAGE) / 100;
        var fullUrl = address + '?x=' + randomString + '&y=1';
        var options = url.parse(fullUrl);

        options.headers = commonHeaders;

        /**
         * @param response: http.IncomingMessage
         *          <https://nodejs.org/api/http.html#http_class_http_incomingmessage>
         */
        function onFirstResponse(response) {

            console.log("%s - (singleDownloadProcess): Task #%s. Received first " +
                "chunk of bytes. Response Code: %s", MODULE, n, response.statusCode);

            startTime = process.hrtime();
            if (response.statusCode !== 200) {
                callback(response.statusCode);
                return;
            }

            response.on('data', function (d) {
                len += d.length;
                if (len > threshold && !firstDownloadEnded) {
                    onChunkReceived(n, d.length, threads);
                    partialBytes += d.length;
                    if (!isStarted) {
                        partialStartTime = process.hrtime();
                        isStarted = true;
                    }
                }
                if (firstDownloadEnded) {
                    req.abort();
                }
            });

            response.on('end', function () {
                if (startTime !== null && partialStartTime !== null) {
                    var partialLatency = process.hrtime(partialStartTime);
                    var partialNsLatency = fromIntervalToNs(partialLatency);

                    var rate = partialBytes / (partialNsLatency * 1e-9);
                    console.log("%s - (singleDownloadProcess): Task #%s. Duration: %s.",
                        MODULE, n, (partialNsLatency * 1e-9).toFixed(2));

                    onSingleDownloadEnd(promise, n);
                    callback(null, {rate: rate, latency: partialNsLatency, bytes: partialBytes});

                } else {
                    callback("end before partial start");
                    console.log("%s - (singleDownloadProcess): Error end before partial start", MODULE);
                }
            });
        }

        console.log("%s - (singleDownloadProcess): Task #%s. Url: %s", MODULE, n, fullUrl);

        var req = http.request(options, onFirstResponse);

        req.setTimeout(DEFAULT_TIMEOUT, function () {
            console.log("%s - (singleDownloadProcess): Timeout occurred", MODULE);
            callback("timeout error");
            req.abort();
        });

        req.on('error', function (e) {
            callback("error");
            req.abort();
            console.log("%s - (singleDownloadProcess): Error %s", MODULE, e.code);
        });

        req.end();
    }

    function uploadRandomFile(address, size, stopIfTooSlow, cb) {
        stopIfTooSlow = typeof stopIfTooSlow !== 'undefined' ? stopIfTooSlow : false;
        console.log("%s - (uploadRandomFile): From URL: %s. File size: %s",
            MODULE, address, size);
        var options = {};
        options.uri = url.parse(address);
        options.headers = JSON.parse(JSON.stringify(commonHeaders));
        options.method = 'POST';
        options.headers['Content-Type'] = 'application/x-www-form-urlencoded';

        var uploadStartTime = null;

        var dataBlock = generateRandomData(size);

        options.headers['Content-Length'] = size;
        options.body = dataBlock;

        uploadStartTime = process.hrtime();
        request(options, function (error, response) {

            if (error) {
                console.log("%s - (uploadRandomFile): upload request error %s",
                    MODULE, error.message);
                cb(null);
                return;
            }
            if (uploadStartTime !== null && response.statusCode === 200 && response.body) {
                var bodySize = response.body.replace('&', '').split('=')[1];

                console.log("%s - (uploadRandomFile): Response Code = %s. Body size = %s",
                    MODULE, response.statusCode, bodySize);
                if (!bodySize || Math.abs(bodySize - size) > UPLOAD_FILE_THRESHOLD) {
                    console.log("%s - (uploadRandomFile): Upload response size from " +
                        "server is not valid %s != %s", MODULE, bodySize, size);
                    cb(null);
                    return;
                }

                var latency = process.hrtime(uploadStartTime);
                var nsLatency = fromIntervalToNs(latency);

                var rate = dataBlock.length / (nsLatency * 1e-9);
                var result = {
                    size: dataBlock.length,
                    nsLatency: nsLatency,
                    rate: rate
                };
                if (stopIfTooSlow && ((rate * 8) < MIN_ALLOWED_UPLOAD_BIT_RATE)) {
                    cb(result);
                } else {
                    cb(null, result);
                }

            } else {
                console.log("%s - (uploadRandomFile): Bad status code %s or missing body",
                    MODULE, response.statusCode);
                cb(null);
            }
        });
    }


    function downloadProcess(address, bytes, threads, promise) {
        var iterations = typeof threads === 'undefined' ? MAX_THREADS : threads;
        console.log("%s - (downloadProcess): Parallel download phase: #%s concurrent tasks.",
            MODULE, iterations);
        async.times(iterations, function (n, next) {
            singleDownloadProcess(n, address, bytes, promise, threads, function (err, res) {
                next(err, res);
            });
        }, function (err, samples) {
            if (err) {
                promise.reject("Parallel download process failed");
                console.log(MODULE + err);
            }
        });
    }

    function download(server, interactive, threads) {
        threads = typeof threads === 'undefined' ? MAX_THREADS : threads;
        var deferred = q.defer();
        var downloadDeferred = q.defer();
        var fileSizes = DOWNLOAD_FILE_SIZES;

        var urlsVector = [];

        var functions = [];
        for (var i = 0; i < fileSizes.length; i++) {
            var size = fileSizes[i];
            urlsVector.push(url.resolve(server.url, 'random' + size + 'x' + size + '.jpg'));
            functions.push(downloadFile.bind(null, url.resolve(server.url, 'random' + size + 'x' + size + '.jpg'), true));
        }

        var parallelDownload = function (server, fallbackSize, fallbackTime) {
            downloadProcess(server, fallbackSize, threads, downloadDeferred);
            downloadDeferred.promise.then(function (result) {
                if (result.nsDuration < MIN_EXP_DURATION || (fallbackSize / fallbackTime) > (result.size / result.nsDuration)) {
                    console.log("%s - (download): Warning, fallback download results. Data too short", MODULE);
                    if (interactive) {
                        //resolve fallback results
                        deferred.resolve({nsDuration: fallbackTime, size: fallbackSize});
                    } else {
                        deferred.reject(PARALLEL_DOWNLOAD_FAIL);
                    }

                } else {
                    //call the success callback
                    deferred.resolve(result);
                }
            }).catch(function (error) {
                console.log("%s - (download): Error => %s. Fallback download results.",
                    MODULE, error.message);

                if (interactive) {
                    //resolve fallback results
                    deferred.resolve({nsDuration: fallbackTime, size: fallbackSize});
                } else {
                    deferred.reject(error);
                }
            });
        };

        async.series(functions, function (err, results) {
            if (err !== null) {
                if (err.server && err.size && err.time) {
                    console.log("%s - (download): Serial test stopped due to slow download", MODULE);
                    parallelDownload(err.server, err.size, err.time);
                } else {
                    deferred.reject(err);
                }
            } else {
                var maxRate = 0;
                var index = 0;
                for (var i = 0; i < results.length; i++) {
                    if (results[i] && results[i].rate && results[i].rate > maxRate) {
                        maxRate = results[i].rate;
                        index = i;
                    }
                }
                parallelDownload(urlsVector[index], results[index].size, results[index].time);
            }
        });
        return deferred.promise;
    }

    function upload(server, interactive) {
        var address = server.url;
        var deferred = q.defer();

        var functions = [];
        for (var i = 0; i < UPLOAD_FILE_SIZES.length; i++) {
            functions.push(uploadRandomFile.bind(null, address, UPLOAD_FILE_SIZES[i], true));
        }

        async.series(functions, function (err, results) {
            console.log("%s - (upload): Completed", MODULE);

            if (err !== null && !err.rate && !err.size && !err.nsLatency) {
                console.log("%s - (upload): upload phase error");
                deferred.reject(err);
            } else if (err !== null && err.rate && err.size && err.nsLatency) {
                console.log("%s - (upload): Returning partial results. " +
                    "Rate %s. Size %s.", MODULE, err.rate, err.size);
                deferred.resolve({
                    rate: err.rate,
                    size: err.size,
                    nsDuration: err.nsLatency
                });
            }
            else {
                var maxRate = 0;
                var index = 0;
                for (var i = 0; i < results.length; i++) {
                    if (results[i] !== undefined && results[i] && results[i].rate && results[i].rate > maxRate) {
                        maxRate = results[i].rate;
                        index = i;
                    }
                }
                if (results[index]) {
                    console.log("%s - (upload): returning results object", MODULE);
                    deferred.resolve({
                        rate: results[index].rate,
                        size: results[index].size,
                        nsDuration: results[index].nsLatency
                    });
                } else {
                    console.log("%s - (upload): error parsing the results", MODULE);
                    deferred.reject(err);
                }

            }
        });
        return deferred.promise;
    };

    function sortBestServers(serverList) {
        var deferred = q.defer();

        var functions = [];
        for (var i = 0; i < serverList.length; i++) {
            var serverUrl = url.resolve(serverList[i].url, SHORT_TEST_FILE);
            var blob = downloadFile.bind(null, serverUrl, false);
            functions.push(blob);
        }

        async.series(functions, function (err, results) {
            console.log('%s - (sortBestServers): Raw results => ', MODULE,
                JSON.stringify(results));
            if (err !== null && results.length === 0) {
                deferred.reject(err);
            } else {
                for (var i = 0; i < results.length; i++) {
                    if (results[i] !== undefined) {
                        serverList[i].rate = results[i].rate;
                    }
                }
                serverList.sort(function (a, b) {
                    return b.rate - a.rate;
                });
                console.log('%s - (sortBestServers): sorted servers => ', MODULE,
                    JSON.stringify(serverList));

                deferred.resolve(serverList);
            }
        });
        return deferred.promise;
    }

    return {
        performSpeedTest: function (server, interactive) {
            allTaskStarted = null;
            accumulator = 0;
            firstDownloadEnded = false;
            flows = {};

            var deferred = q.defer();
            console.log("%s - (performSpeedTest)", MODULE);
            upload(server, interactive).then(function (uploadResults) {
                console.log("%s - (performSpeedTest): download test", MODULE);
                download(server, interactive).then(function (downloadResult) {
                    var result = {
                        raw: [],
                        server: server,
                        'speed.upload': (uploadResults.size * 8 / (uploadResults.nsDuration * 1e-9)).toFixed(0),
                        'speed.download': (downloadResult.size * 8 / (downloadResult.nsDuration * 1e-9)).toFixed(0),
                        MaxRTT: "0",
                        MinRTT: "0",
                        rttsec: "0.0"
                    };
                    deferred.resolve(result);
                }).catch(function (error) {
                    deferred.reject(error);
                    console.log("%s - (performSpeedTest): Error performing " +
                        "download %s", MODULE, typeof error === 'string' ? error : error.message);
                });
            }).catch(function (error) {
                console.log("%s - (performSpeedTest): Error performing upload " +
                    "test %s", MODULE, typeof error === 'string' ? error : error.message);
                deferred.reject(error);
            });
            return deferred.promise;
        },
        sortBestServers: sortBestServers,
        /* test code */
        downloadProcess: downloadProcess,
        requestToServer: requestToServer,
        downloadFile: downloadFile,
        singleDownloadProcess: singleDownloadProcess,
        uploadRandomFile: uploadRandomFile
        /* test code */
    };
}

function main(message, callback) {
    var app = speedTest();
    if (message.bestServer) {
        console.log("%s - Interactive mode", MODULE);
        app.performSpeedTest(message.bestServer, true).then(function (result) {
            /* do not resolve the server as it should not be stored in the cache */
            var data = {result: result, error: null};
            callback(data);
        }).catch(function (error) {
            console.log("%s - Error performing interactive speed test. " +
                "Error: %s", MODULE, error.message);
            console.log(error.stack);
            callback({error: error});
        });
    } else {
        console.log("%s - Automatic mode", MODULE);
        app.sortBestServers(message.serverList).then(function (sortedServerList) {

            console.log("%s - (sortBestServers): SUCCESS", MODULE);

            var blacklist = [];
            async.retry({
                times: MAX_SPEEDTEST_ATTEMPTS,
                interval: 1000
            }, function (retryCallback) {
                var bestServer = sortedServerList[0];
                console.log('%s - Having servers: %s. Trying speedTest with ' +
                    'best server: ', MODULE, JSON.stringify(sortedServerList),
                    JSON.stringify(bestServer));

                app.performSpeedTest(bestServer, false).then(function (result) {
                    console.log('%s - Finish', MODULE);
                    retryCallback(null, result);
                }).catch(function (error) {
                    console.log("%s - Error performing non interactive " +
                        "speed test => %s", MODULE, error);

                    if (error !== PARALLEL_DOWNLOAD_FAIL) { //do not blacklist the server if parallel phase failed
                        console.log("%s - Adding server to blacklist: " +
                            "%s", MODULE, JSON.stringify(bestServer));
                        blacklist.push(sortedServerList.splice(0, 1)[0]);
                    }
                    retryCallback(error);
                });
            }, function (error, result) {
                if (error === null) {
                    console.log('%s: NO error. SpeedTest Result: %s', MODULE,
                        JSON.stringify(result));
                    callback({
                        result: result,
                        error: null,
                        server: sortedServerList[0],
                        blacklist: blacklist
                    });
                    console.log('%s: Called report', MODULE);
                } else {
                    callback({error: error, blacklist: blacklist});
                }
            });

        }).catch(function (error) {
            console.log("%s: Error performing non interactive " +
                "speed test => %s", MODULE, error.message);
            callback({error: error, blacklist: message.serverList});
        });
    }
}

function eventManager() {
    process.on('message', function (message) {
        main(message, function (response) {
            process.send(response);
        });
    });

    process.on('SIGTERM', function () {
        console.log('%s: Got SIGTERM signal.', MODULE);
        process.exit(0);
    });
}

if (process.env.DPLATFORM === 'openwrt') {
    console.log('%s: Not launching Speed Test as External Process', MODULE);
} else {
    eventManager();
}

module.exports.main = main;
module.exports.factory = speedTest;
module.exports._eventManager = eventManager;