domotz-remote-pawn/src/network_tools/speed_test/ooklaSpeedTest.js

Domotz Agent

var MAX_SPEEDTEST_ATTEMPTS = 4;
var PARALLEL_DOWNLOAD_FAIL = 'FALLBACK';

module.exports.factory = function (resourceLocator) {
    const q = resourceLocator.q;
    const http = resourceLocator.http;
    const request = resourceLocator.request;
    const url = resourceLocator.url;
    const myConsole = resourceLocator.log.decorateLogs();
    const async = resourceLocator.async;

    http.globalAgent.maxSockets = Infinity;

    var DOWNLOAD_FILE_SIZES = [1000, 2000, 3000]; // a jpg file of size NxN
    var UPLOAD_FILE_SIZES = [1000000, 2000000, 5000000]; // 1, 2, 5 MBytes

    var MAX_THREADS = 4;
    var MIN_EXP_DURATION = 0.5 * 1e9;

    var RAMP_PERCENTAGE = 10;
    var DEFAULT_TIMEOUT = 20000;
    var MIN_ALLOWED_DOWNLOAD_BIT_RATE = 5 * 1e6;
    var MIN_ALLOWED_UPLOAD_BIT_RATE = 1e6;
    var UPLOAD_FILE_THRESHOLD = 1000;
    var SHORT_TEST_FILE = 'random2000x2000.jpg';

    var USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36';

    var commonHeaders = {
        'user-agent': USER_AGENT,
        'Cache-Control': 'no-cache',
        Pragma: 'no-cache',
        Referer: 'http://www.speedtest.net/',
    };

    var allTaskStarted = null;
    var accumulator = 0;
    var firstDownloadEnded = false;
    var flows = {};
    var pingInExecute = false;

    function _fromIntervalToNs(time) {
        return time[0] * 1e9 + time[1];
    }

    function _generateRandomData(size) {
        var d = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\n';
        var c = 1;
        while (d.length < size) {
            d += 'a' + c + '=' + d;
            c++;
        }
        return d.substr(0, size);
    }

    function requestToServer(address, cb) {
        myConsole.info('(requestToServer): Sending request to %s', address);

        var randomString = new Date().getTime();
        var options = {
            url: address + '?x=' + randomString + '&y=1',
            method: 'GET',
            headers: commonHeaders,
            timeout: DEFAULT_TIMEOUT,
        };

        var start = process.hrtime();

        request(options, function callback(error, response, body) {
            try {
                if (error) {
                    cb(error);
                    return;
                }

                var statusCode = response.statusCode;
                if (statusCode > 299 || statusCode < 200) {
                    error = {
                        message: body,
                        status: statusCode,
                    };
                    cb(error);
                    return;
                }
                var nsLatency = _fromIntervalToNs(process.hrtime(start));

                var bytesRead = response.socket && response.socket.bytesRead;
                var sample = {
                    size: parseInt(response.headers['content-length']) || bytesRead,
                    time: nsLatency,
                };
                cb(null, sample);
            } catch (err) {
                cb(error);
                myConsole.info('(requestToServer): exception: %s', JSON.stringify(err.message));
            }
        });
    }

    function _downloadFile(server, stopIfTooSlow, cb) {
        stopIfTooSlow = typeof stopIfTooSlow !== 'undefined' ? stopIfTooSlow : false;
        myConsole.info('(downloadFile): Downloading file from %s', server);

        requestToServer(server, function (err, res) {
            if (err) {
                cb('(downloadFile): Error downloading file: %s', err.message);
                return;
            }
            var rate = res.size / (res.time * 1e-9);
            myConsole.info('(downloadFile): HTTP download rate: %s', (rate * 1e-6).toFixed(2));
            var result = {
                rate: rate,
                size: res.size,
                server: server,
                time: res.time,
            };
            if (stopIfTooSlow && rate * 8 < MIN_ALLOWED_DOWNLOAD_BIT_RATE) {
                myConsole.info('(downloadFile): Stop process, download too slow');
                /**
                 * Return an error if the download duration is too high.
                 * Then skip downloading the remaining file sizes.
                 */
                cb(result);
            } else {
                cb(null, result);
            }
        });
    }

    function _onChunkReceived(id, len, threads) {
        if (!flows.hasOwnProperty(id)) {
            flows[id] = true;
        }

        if (Object.keys(flows).length === threads) {
            accumulator += len;
            if (allTaskStarted === null) {
                allTaskStarted = process.hrtime();
            }
        }
    }

    function _onSingleDownloadEnd(promise, n) {
        if (!firstDownloadEnded) {
            /* the first of the parallel download processes has completed */
            firstDownloadEnded = true;
            if (allTaskStarted !== null) {
                /* all the other processes already reached the ramp up threshold, we compute the final download rate */
                var duration = process.hrtime(allTaskStarted);
                var nsDuration = _fromIntervalToNs(duration);
                myConsole.info(
                    '(onSingleDownloadEnd): Task #%s. ' + 'Parallel phase duration = %s s, ' + 'Download size = %s MB, Rate = %s MB/s.',
                    n,
                    (nsDuration * 1e-9).toFixed(2),
                    (accumulator * 1e-6).toFixed(2),
                    ((accumulator * 8) / (nsDuration * 1e-3)).toFixed(2)
                );
                promise.resolve({ nsDuration: nsDuration, size: accumulator });
            } else {
                myConsole.info('(onSingleDownloadEnd): Task #%s. Warning, ' + 'First task ended before all started', n);
                promise.reject(PARALLEL_DOWNLOAD_FAIL);
            }
        }
    }

    function _singleDownloadProcess(n, address, size, promise, threads, progressCallback, callback) {
        var randomString = new Date().getTime();
        var len = 0;
        var partialBytes = 0;
        var startTime = null;
        var partialStartTime = null;
        var isStarted = false;
        var threshold = (size * RAMP_PERCENTAGE) / 100;
        var fullUrl = address + '?x=' + randomString + '&y=1';
        var options = url.parse(fullUrl);

        options.headers = commonHeaders;

        /**
         * @param response: http.IncomingMessage
         *          <https://nodejs.org/api/http.html#http_class_http_incomingmessage>
         */
        function onFirstResponse(response) {
            myConsole.info('(singleDownloadProcess): Task #%s. Received first ' + 'chunk of bytes. Response Code: %s', n, response.statusCode);

            startTime = process.hrtime();
            if (response.statusCode !== 200) {
                callback(response.statusCode);
                return;
            }

            response.on('data', function (d) {
                len += d.length;
                if (len > threshold && !firstDownloadEnded) {
                    _onChunkReceived(n, d.length, threads);
                    partialBytes += d.length;
                    if (!isStarted) {
                        partialStartTime = process.hrtime();
                        isStarted = true;
                    }
                }
                if (firstDownloadEnded) {
                    req.abort();
                }
            });

            response.on('end', function () {
                if (startTime !== null && partialStartTime !== null) {
                    var partialLatency = process.hrtime(partialStartTime);
                    var partialNsLatency = _fromIntervalToNs(partialLatency);

                    var rate = partialBytes / (partialNsLatency * 1e-9);
                    myConsole.info('(singleDownloadProcess): Task #%s. Duration: %s.', n, (partialNsLatency * 1e-9).toFixed(2));

                    _onSingleDownloadEnd(promise, n);
                    callback(null, { rate: rate, latency: partialNsLatency, bytes: partialBytes });
                } else {
                    callback('end before partial start');
                    myConsole.info('(singleDownloadProcess): Error end before partial start');
                }
            });
        }

        myConsole.info('(singleDownloadProcess): Task #%s. Url: %s', n, fullUrl);

        if (!pingInExecute) {
            progressCallback('download');
            pingInExecute = true;
        }
        var req = http.request(options, onFirstResponse);

        req.setTimeout(DEFAULT_TIMEOUT, function () {
            myConsole.info('(singleDownloadProcess): Timeout occurred');
            callback('timeout error');
            req.abort();
        });

        req.on('error', function (e) {
            callback('error');
            req.abort();
            myConsole.info('(singleDownloadProcess): Error %s', e.code);
        });

        req.end();
    }

    function _uploadRandomFile(address, size, stopIfTooSlow, progressCallback, cb) {
        stopIfTooSlow = typeof stopIfTooSlow !== 'undefined' ? stopIfTooSlow : false;
        myConsole.info('(uploadRandomFile): From URL: %s. File size: %s', address, size);
        var options = {};
        options.uri = url.parse(address);
        options.headers = JSON.parse(JSON.stringify(commonHeaders));
        options.method = 'POST';
        options.headers['Content-Type'] = 'application/x-www-form-urlencoded';

        var uploadStartTime = null;

        var dataBlock = _generateRandomData(size);

        options.headers['Content-Length'] = size;
        options.body = dataBlock;

        uploadStartTime = process.hrtime();

        if (!pingInExecute) {
            progressCallback('upload');
            pingInExecute = true;
        }
        request(options, function (error, response) {
            if (error) {
                myConsole.info('(uploadRandomFile): upload request error %s', error.message);
                cb(null);
                return;
            }
            if (uploadStartTime !== null && response.statusCode === 200 && response.body) {
                var bodySize = response.body.replace('&', '').split('=')[1];

                myConsole.info('(uploadRandomFile): Response Code = %s. Body size = %s', response.statusCode, bodySize);
                if (!bodySize || Math.abs(bodySize - size) > UPLOAD_FILE_THRESHOLD) {
                    myConsole.info('(uploadRandomFile): Upload response size from server is not valid %s != %s', bodySize, size);
                    cb(null);
                    return;
                }

                var latency = process.hrtime(uploadStartTime);
                var nsLatency = _fromIntervalToNs(latency);

                var rate = dataBlock.length / (nsLatency * 1e-9);
                var result = {
                    size: dataBlock.length,
                    nsLatency: nsLatency,
                    rate: rate,
                };
                if (stopIfTooSlow && rate * 8 < MIN_ALLOWED_UPLOAD_BIT_RATE) {
                    cb(result);
                } else {
                    cb(null, result);
                }
            } else {
                myConsole.warn('(uploadRandomFile): Bad status code %s or missing body', response.statusCode);
                cb(null);
            }
        });
    }

    function _downloadProcess(address, bytes, threads, promise, progressCallback) {
        var iterations = typeof threads === 'undefined' ? MAX_THREADS : threads;
        myConsole.info('(downloadProcess): Parallel download phase: #%s concurrent tasks.', iterations);
        async.times(
            iterations,
            function (n, next) {
                _singleDownloadProcess(n, address, bytes, promise, threads, progressCallback, function (err, res) {
                    next(err, res);
                });
            },
            function (err) {
                if (err) {
                    promise.reject('Parallel download process failed');
                    myConsole.info(err);
                }
            }
        );
    }

    function download(server, interactive, progressCallback, threads) {
        threads = typeof threads === 'undefined' ? MAX_THREADS : threads;
        var deferred = q.defer();
        var downloadDeferred = q.defer();
        var fileSizes = DOWNLOAD_FILE_SIZES;

        var urlsVector = [];

        var functions = [];
        for (var i = 0; i < fileSizes.length; i++) {
            var size = fileSizes[i];
            urlsVector.push(url.resolve(server.url, 'random' + size + 'x' + size + '.jpg'));
            functions.push(_downloadFile.bind(null, url.resolve(server.url, 'random' + size + 'x' + size + '.jpg'), true));
        }

        var parallelDownload = function (server, fallbackSize, fallbackTime) {
            _downloadProcess(server, fallbackSize, threads, downloadDeferred, progressCallback);
            downloadDeferred.promise
                .then(function (result) {
                    if (result.nsDuration < MIN_EXP_DURATION || fallbackSize / fallbackTime > result.size / result.nsDuration) {
                        myConsole.info('(download): Warning, fallback download results. Data too short');
                        if (interactive) {
                            //resolve fallback results
                            deferred.resolve({ nsDuration: fallbackTime, size: fallbackSize });
                        } else {
                            deferred.reject(PARALLEL_DOWNLOAD_FAIL);
                        }
                    } else {
                        //call the success callback
                        deferred.resolve(result);
                    }
                })
                .catch(function (error) {
                    myConsole.info('(download): Error => %s. Fallback download results.', error.message);

                    if (interactive) {
                        //resolve fallback results
                        deferred.resolve({ nsDuration: fallbackTime, size: fallbackSize });
                    } else {
                        deferred.reject(error);
                    }
                });
        };

        async.series(functions, function (err, results) {
            if (err !== null) {
                if (err.server && err.size && err.time) {
                    myConsole.info('(download): Serial test stopped due to slow download');
                    parallelDownload(err.server, err.size, err.time);
                } else {
                    deferred.reject(err);
                }
            } else {
                var maxRate = 0;
                var index = 0;
                for (var i = 0; i < results.length; i++) {
                    if (results[i] && results[i].rate && results[i].rate > maxRate) {
                        maxRate = results[i].rate;
                        index = i;
                    }
                }
                parallelDownload(urlsVector[index], results[index].size, results[index].time);
            }
        });
        return deferred.promise;
    }

    function upload(server, progressCallback) {
        var address = server.url;
        var deferred = q.defer();

        var functions = [];
        for (var i = 0; i < UPLOAD_FILE_SIZES.length; i++) {
            functions.push(_uploadRandomFile.bind(null, address, UPLOAD_FILE_SIZES[i], true, progressCallback));
        }

        async.series(functions, function (err, results) {
            myConsole.info('(upload): Completed');

            if (err !== null && !err.rate && !err.size && !err.nsLatency) {
                myConsole.info('%s - (upload): upload phase error');
                deferred.reject(err);
            } else if (err !== null && err.rate && err.size && err.nsLatency) {
                myConsole.info('(upload): Returning partial results. Rate %s. Size %s.', err.rate, err.size);
                deferred.resolve({
                    rate: err.rate,
                    size: err.size,
                    nsDuration: err.nsLatency,
                });
            } else {
                var maxRate = 0;
                var index = 0;
                for (var i = 0; i < results.length; i++) {
                    if (results[i] !== undefined && results[i] && results[i].rate && results[i].rate > maxRate) {
                        maxRate = results[i].rate;
                        index = i;
                    }
                }
                if (results[index]) {
                    myConsole.info('(upload): returning results object');
                    deferred.resolve({
                        rate: results[index].rate,
                        size: results[index].size,
                        nsDuration: results[index].nsLatency,
                    });
                } else {
                    myConsole.info('(upload): error parsing the results');
                    deferred.reject(err);
                }
            }
        });
        return deferred.promise;
    }

    function sortBestServers(serverList) {
        var deferred = q.defer();

        var functions = [];
        for (var i = 0; i < serverList.length; i++) {
            var serverUrl = url.resolve(serverList[i].url, SHORT_TEST_FILE);
            var blob = _downloadFile.bind(null, serverUrl, false);
            functions.push(blob);
        }

        async.series(functions, function (err, results) {
            myConsole.info('(sortBestServers): Raw results => ', JSON.stringify(results));
            if (err !== null && results.length === 0) {
                deferred.reject(err);
            } else {
                for (var i = 0; i < results.length; i++) {
                    if (results[i] !== undefined) {
                        serverList[i].rate = results[i].rate;
                    }
                }
                serverList.sort(function (a, b) {
                    return b.rate - a.rate;
                });
                myConsole.info('(sortBestServers): sorted servers => ', JSON.stringify(serverList));

                deferred.resolve(serverList);
            }
        });
        return deferred.promise;
    }

    function performSpeedTest(server, interactive, progressCallback) {
        allTaskStarted = null;
        accumulator = 0;
        firstDownloadEnded = false;
        flows = {};

        var deferred = q.defer();
        myConsole.info('(performSpeedTest)');
        upload(server, progressCallback)
            .then(function (uploadResults) {
                myConsole.info('(performSpeedTest): download test');
                pingInExecute = false;
                download(server, interactive, progressCallback)
                    .then(function (downloadResult) {
                        pingInExecute = false;
                        var result = {
                            raw: [],
                            server: server,
                            'speed.upload': ((uploadResults.size * 8) / (uploadResults.nsDuration * 1e-9)).toFixed(0),
                            'speed.download': ((downloadResult.size * 8) / (downloadResult.nsDuration * 1e-9)).toFixed(0),
                            MaxRTT: '0',
                            MinRTT: '0',
                            rttsec: '0.0',
                        };
                        deferred.resolve(result);
                    })
                    .catch(function (error) {
                        pingInExecute = false;
                        deferred.reject(error);
                        myConsole.info('(performSpeedTest): Error performing download %s', typeof error === 'string' ? error : error.message);
                    });
            })
            .catch(function (error) {
                pingInExecute = false;
                myConsole.info('(performSpeedTest): Error performing upload test %s', typeof error === 'string' ? error : error.message);
                deferred.reject(error);
            });
        return deferred.promise;
    }

    function main(message, callback, progressCallback) {
        if (message.bestServer) {
            myConsole.info('Interactive mode');
            performSpeedTest(message.bestServer, true, progressCallback)
                .then(function (result) {
                    /* do not resolve the server as it should not be stored in the cache */
                    var data = { result: result, error: null };
                    callback(data);
                })
                .catch(function (error) {
                    myConsole.info('Error performing interactive speed test. Error: %s', error.message);
                    myConsole.info(error.stack);
                    callback({ error: error });
                });
        } else {
            myConsole.info('Automatic mode');
            sortBestServers(message.serverList)
                .then(function (sortedServerList) {
                    myConsole.info('(sortBestServers): SUCCESS');

                    var blacklist = [];
                    async.retry(
                        {
                            times: MAX_SPEEDTEST_ATTEMPTS,
                            interval: 1000,
                        },
                        function (retryCallback) {
                            var bestServer = sortedServerList[0];
                            myConsole.info(
                                'Having servers: %s. Trying speedTest with best server: ',
                                JSON.stringify(sortedServerList),
                                JSON.stringify(bestServer)
                            );

                            performSpeedTest(bestServer, false, progressCallback)
                                .then(function (result) {
                                    myConsole.info('Finish');
                                    retryCallback(null, result);
                                })
                                .catch(function (error) {
                                    myConsole.info('Error performing non interactive speed test => %s', error);

                                    if (error !== PARALLEL_DOWNLOAD_FAIL) {
                                        //do not blacklist the server if parallel phase failed
                                        myConsole.info('Adding server to blacklist: %s', JSON.stringify(bestServer));
                                        blacklist.push(sortedServerList.splice(0, 1)[0]);
                                    }
                                    retryCallback(error);
                                });
                        },
                        function (error, result) {
                            if (error === null) {
                                myConsole.info('NO error. SpeedTest Result: %s', JSON.stringify(result));
                                callback({
                                    result: result,
                                    error: null,
                                    server: sortedServerList[0],
                                    blacklist: blacklist,
                                });
                                myConsole.info('Called report');
                            } else {
                                callback({ error: error, blacklist: blacklist });
                            }
                        }
                    );
                })
                .catch(function (error) {
                    myConsole.info('Error performing non interactive speed test => %s', error.message);
                    callback({ error: error, blacklist: message.serverList });
                });
        }
    }

    return {
        main: main,
        /* test code */
        performSpeedTest: performSpeedTest,
        sortBestServers: sortBestServers,
        downloadProcess: _downloadProcess,
        requestToServer: requestToServer,
        downloadFile: _downloadFile,
        singleDownloadProcess: _singleDownloadProcess,
        uploadRandomFile: _uploadRandomFile,
        /* test code */
    };
};