domotz-remote-pawn/src/network_tools/speed_test/ooklaServers.js

Domotz Agent

/*jslint node: true */

var CONFIG_URL = 'http://www.speedtest.net/speedtest-config.php';
var SERVERS_URL = 'http://www.speedtest.net/speedtest-servers.php';
var USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, ' + 'like Gecko) Chrome/41.0.2228.0 Safari/537.36';
var SERVER_CACHE_SCOPE = 'ookla-servers';
var FETCH_CACHE_SCOPE = 'fetch-ookla';
var MAX_SERVERS_TO_PING = 4;
var MAX_PING_PER_SERVER = 10;
var DEFAULT_TIMEOUT = 30000;
var PING_TIMEOUT = 10000;

function factory(resourceLocator, resetCache) {
    var myConsole = resourceLocator.log.decorateLogs();

    var lodash = resourceLocator.lodash;
    var request = resourceLocator.request;
    var q = resourceLocator.q;
    var parseXML = resourceLocator.xml2js.parseString;
    var async = resourceLocator.async;
    var url = resourceLocator.url;
    var cacheUtils = resourceLocator.utils.cache;
    var fileUtils = resourceLocator.utils.file;
    var configuration = resourceLocator.configuration;
    var distanceCalculator = resourceLocator.distanceCalculator;
    var blacklistFilePath = process.env.DOMOTZ_CONF_DIR + '/ookla_blacklist.conf';

    var preferredServer = null;
    var clientConfig = null;
    var serversCache = cacheUtils.createCache(SERVER_CACHE_SCOPE);
    var day = new Date().getDay();
    var customBlacklist = [];

    resetCache = typeof resetCache !== 'undefined' ? resetCache : true;

    if (resetCache && day === 0) {
        myConsole.info("(cache): It's sunday, resetting cache, scope = %s", SERVER_CACHE_SCOPE);
        serversCache.reset();
    }

    fileUtils.readFile(blacklistFilePath, function (content) {
        content.split('\n').forEach(function contentFilter(e) {
            myConsole.silly(' Processed elem: %s', content);
            if (e) {
                customBlacklist.push(parseInt(e));
            }
        });
        myConsole.info('(blacklist): Loaded custom blacklist: %s', JSON.stringify(customBlacklist));
    });

    function fromIntervalToNs(time) {
        return time[0] * 1e9 + time[1];
    }

    function getXMLRequest(uri) {
        var deferred = q.defer();
        var options = {};
        options.url = uri;
        options.headers = {};
        options.headers['user-agent'] = USER_AGENT;
        options.timeout = DEFAULT_TIMEOUT;

        request(options, function callback(error, response, body) {
            try {
                if (error) {
                    myConsole.alert('error getting XML data' + JSON.stringify(error));
                    deferred.reject(error);
                    return;
                }

                var statusCode = response.statusCode;
                if (statusCode > 299 || statusCode < 200) {
                    error = {
                        message: body,
                        status: statusCode,
                    };
                    myConsole.alert('error getting XML data' + response.statusCode);
                    deferred.reject(error);
                    return;
                }

                myConsole.debug('getXMLRequest: Parsing xml ' + uri);
                parseXML(body, function (err, xml) {
                    if (err) {
                        myConsole.alert('error parsing XML data' + JSON.stringify(err));
                        deferred.reject(err);
                        return;
                    }
                    myConsole.info('getXMLRequest: Resolving parsed xml ' + uri);
                    deferred.resolve(xml);
                });
            } catch (exception) {
                myConsole.alert('error getting XML data' + JSON.stringify(exception));
                deferred.reject(exception);
            }
        });
        return deferred.promise;
    }

    function getConfiguration() {
        var deferred = q.defer();

        getXMLRequest(CONFIG_URL)
            .then(function (xml) {
                try {
                    var config = xml.settings;

                    var getField = function (name) {
                        return config[name][0].$;
                    };

                    var speedTestConfig = {
                        client: getField('client'),
                        times: getField('times'),
                        download: getField('download'),
                        upload: getField('upload'),
                        ignore_ids: getField('server-config').ignoreids,
                    };
                    deferred.resolve(speedTestConfig);
                } catch (exception) {
                    myConsole.alert('error getting configuration' + JSON.stringify(exception));
                    deferred.reject(exception);
                }
            })
            .catch(function (error) {
                myConsole.error('error getting configuration from XML: ' + JSON.stringify(error));
                deferred.reject(error);
            });
        return deferred.promise;
    }

    /*
     * Executed at startup after a random delay. Fetches the list of servers and saves it in cache.
     */
    function fetch() {
        myConsole.info('(fetch): Fetching server list');
        var deferred = q.defer();

        serversCache.get(
            FETCH_CACHE_SCOPE,
            function () {
                getXMLRequest(SERVERS_URL)
                    .then(function (xml) {
                        var serversObject = xml.settings.servers[0].server;
                        var serversList = [];
                        getConfiguration()
                            .then(function (config) {
                                clientConfig = config;
                                for (var i = 0; i < serversObject.length; i++) {
                                    var serverDescriptor = serversObject[i].$;
                                    if (
                                        config.ignore_ids.indexOf(serverDescriptor.id) === -1 &&
                                        customBlacklist.indexOf(parseInt(serverDescriptor.id)) === -1
                                    ) {
                                        serversList.push(serverDescriptor);
                                    } else {
                                        myConsole.verbose('(fetch): Server %d blacklisted.', serverDescriptor.id);
                                    }
                                }
                                serversCache.set(FETCH_CACHE_SCOPE, serversList);
                                myConsole.info('(fetch): Removed %d invalid servers from the list', serversObject.length - serversList.length);
                                deferred.resolve(serversList);
                            })
                            .catch(function (error) {
                                myConsole.error('(fetch): Error retrieving xml configuration file');
                                deferred.reject(error);
                            });
                    })
                    .catch(function (error) {
                        myConsole.error('(fetch): Error getting servers from XML list: %s', JSON.stringify(error));
                        deferred.reject(error);
                    });
            },
            function (servers) {
                myConsole.info('(fetch): Fetching ookla server list, cache resolved');
                deferred.resolve(servers);
            }
        );
        return deferred.promise;
    }

    function blacklistServers(blacklist) {
        var blacklistIds = lodash.map(blacklist, 'id');
        myConsole.info('blacklist server function: %s', JSON.stringify(blacklistIds));
        serversCache.get(
            FETCH_CACHE_SCOPE,
            function onMiss() {},
            function onHit(servers) {
                var filteredServers = lodash.filter(servers, function (ooklaServer) {
                    return blacklistIds.indexOf(ooklaServer.id) === -1;
                });
                serversCache.set(FETCH_CACHE_SCOPE, filteredServers);
                setServerAsPreferred(null);
                myConsole.info('Blacklisted %d servers', blacklist.length);
            }
        );
    }

    function requestToServer(n, address, cb) {
        var options = {};
        options.url = address;
        options.headers = {};
        options.headers['user-agent'] = USER_AGENT;
        options.timeout = PING_TIMEOUT;
        var start = process.hrtime();
        request(options, function callback(error, response, body) {
            try {
                if (error) {
                    cb(error);
                    return;
                }

                var statusCode = response.statusCode;
                if (statusCode > 299 || statusCode < 200) {
                    error = {
                        message: body,
                        status: statusCode,
                    };
                    cb(error);
                    return;
                }
                var nanoSecondsLatency = fromIntervalToNs(process.hrtime(start));
                cb(null, nanoSecondsLatency);
            } catch (exception) {
                cb(exception);
            }
        });
    }

    function pingServer(server, cb, numberOfPings) {
        var iterations = numberOfPings || MAX_PING_PER_SERVER;
        var address = url.resolve(server.url, 'latency.txt');

        async.timesSeries(
            iterations,
            function (n, next) {
                requestToServer(n, address, function (err, res) {
                    next(err, res);
                });
            },
            function (err, samples) {
                if (err) {
                    myConsole.alert('pingServer: error ' + JSON.stringify(err));
                    cb(null, null);
                    return;
                }

                var min = Math.min.apply(null, samples);
                var max = Math.max.apply(null, samples);
                var sum = 0;
                for (var i = 0; i < samples.length; i++) {
                    sum += samples[i];
                }
                var avg = sum / samples.length;
                var result = {
                    server: address,
                    times: samples,
                    max: max,
                    min: min,
                    avg: avg,
                };
                cb(null, result);
            }
        );
    }

    /* scrolls the servers list and finds the one with the matching ID */
    var findById = function (serverId) {
        var deferred = q.defer();

        fetch()
            .then(function (servers) {
                servers.every(function (server) {
                    if (server.id == serverId) {
                        deferred.resolve(server);
                        return false;
                    }
                    return true;
                });
                deferred.reject({ status: 404, message: 'NOT FOUND' });
            })
            .catch(function () {
                deferred.reject({ status: 404, message: 'NOT FOUND' });
            });
        return deferred.promise;
    };

    /* Given the server list, selects the N nearest ones. Then performs a series of PING operations on each of them.
     * Returns the server with the minimum latency. */
    function getBestServer(serverList) {
        var deferred = q.defer();
        var servers = serverList.slice(0, MAX_SERVERS_TO_PING);

        myConsole.silly('getBestServers, searching in:' + JSON.stringify(servers));

        async.mapSeries(servers, pingServer, function (err, results) {
            if (err !== null) {
                deferred.reject(err);
            } else {
                myConsole.silly('getBestServers, results is:' + JSON.stringify(results));
                var lowest = Number.POSITIVE_INFINITY;
                var minIndex = null;
                for (var i = 0; i < results.length; i++) {
                    if (results[i] !== null && results[i].min < lowest) {
                        lowest = results[i].min;
                        minIndex = i;
                    }
                }

                if (minIndex !== null) {
                    myConsole.info('getBestServer: selected ' + JSON.stringify(servers[minIndex]));
                    deferred.resolve(servers[minIndex]);
                } else {
                    myConsole.alert('Unable to complete any of the requested pings');
                    deferred.reject('Unable to complete any of the requested pings');
                }
            }
        });
        return deferred.promise;
    }

    function sortServers(serverList) {
        var client = {
            lat: NaN,
            lon: NaN,
        };

        if (configuration && configuration.location) {
            myConsole.info('Sorting servers, using Agent geolocation');
            client.lat = configuration.location.latitude;
            client.lon = configuration.location.longitude;
        } else if (clientConfig) {
            myConsole.info('Sorting servers, using Ookla geolocation');
            client.lat = clientConfig.client.lat;
            client.lon = clientConfig.client.lon;
        }

        for (var n = 0; n < serverList.length; n++) {
            var server = serverList[n];
            server.dist = distanceCalculator.distanceTo(server.lat, server.lon, client.lat, client.lon);
        }

        serverList.sort(function (a, b) {
            return a.dist - b.dist;
        });
        return serverList;
    }

    function getNearestServers(maxServers) {
        myConsole.info('getNearestServers.');
        maxServers = typeof maxServers !== 'undefined' ? maxServers : MAX_SERVERS_TO_PING;
        var deferred = q.defer();
        fetch()
            .then(function (serverList) {
                serverList = sortServers(serverList);
                var restrictedList = serverList.slice(0, maxServers);
                myConsole.info('getNearestServers: Resolving nearest servers. Got restricted server list.');
                deferred.resolve(restrictedList);
            })
            .catch(function (error) {
                myConsole.alert('getNearestServers: unable to resolving nearest servers.');
                deferred.reject(error);
            });
        return deferred.promise;
    }

    function findNearestResponding(serverList) {
        var deferred = q.defer();

        if (!preferredServer) {
            myConsole.info('findNearestResponding: preferred server not available.');
            serverList = sortServers(serverList);
            getBestServer(serverList)
                .then(function (bestServer) {
                    bestServer.location = { siteName: bestServer.sponsor };
                    myConsole.info('findNearestResponding: Preferred server is ' + bestServer.sponsor);
                    //Do not save it in the module variable.
                    deferred.resolve(bestServer);
                })
                .catch(function (error) {
                    myConsole.alert('findNearestResponding: Unable to find the nearest responding server ' + JSON.stringify(error));
                    deferred.reject('unable to find the nearest responding server');
                });
        } else {
            myConsole.info('findNearestResponding: The preferred server is already available');
            preferredServer.location = { siteName: preferredServer.sponsor };
            deferred.resolve(preferredServer);
        }
        return deferred.promise;
    }

    function setServerAsPreferred(server) {
        preferredServer = server;
    }

    function getPreferredServer() {
        return preferredServer;
    }

    return {
        fetch: fetch,
        findNearestResponding: findNearestResponding,
        findById: findById,
        getNearestServers: getNearestServers,
        getBestServer: getBestServer,
        setServerAsPreferred: setServerAsPreferred,
        getPreferredServer: getPreferredServer,
        blacklistServers: blacklistServers,
        pingServer: pingServer,
        getConfiguration: getConfiguration,
    };
}

module.exports.factory = factory;