dl/lib/configuration/ForwardingConfigurationManager.js

DreamLab Libs

var core = require('core');
var WebSocket = require('ws');

var Types = core.common.Types;

var Logger = require('../logger').Logger;
var ConfigurationManager = require('./ConfigurationManager.js').ConfigurationManager;

var ForwardingConfigurationManager = function () {
    ConfigurationManager.apply(this, arguments);

    this._endpoints = [];
    this._intervals = {};
    this._reconnectTimers = {};
    this._ephemeral = {};
    this._endpointsLabels = {};
    this._ws = {};

    this._watches = {};
    this._endpointsWatches = {};

    this._pingTimer = null;
    this._pingTimeout = ForwardingConfigurationManager.DEFAULT_PING_TIMEOUT;
    this._pingTimeoutTimers = {};
    this._pingTimeouts = {};
    this._maxPingTimeouts = ForwardingConfigurationManager.MAX_PING_TIMEOUTS;
    this._stats = {};

    this._queue = [];
    this._queueRunning = 0;
    this._queueMax = ForwardingConfigurationManager.MAX_CONCURRENT_QUERIES;
};

ForwardingConfigurationManager.prototype = Object.create(ConfigurationManager.prototype);

ForwardingConfigurationManager.prototype.setMaxConcurrentQueries = function (count) {
    this._queueMax = count;
    return this;
};

ForwardingConfigurationManager.prototype.setPingTimeout = function (timeout) {
    this._pingTimeout = timeout;
    return this;
};

ForwardingConfigurationManager.prototype.setPingInterval = function (interval) {
    var that = this;

    clearInterval(this._pingTimer);

    this._pingTimer = setInterval(function () {
        for (var i = 0, l = that._endpoints.length; i < l; i++) {
            that._ping(that._endpoints[i]);
        }
    }, interval);

    return this;
};

ForwardingConfigurationManager.prototype.setClients = function (endpoints) {
    var oldEndpoints = this._endpoints.slice(0);
    var newEndpoints = [];
    var that = this;

    // connect new endpoints
    endpoints.forEach(function (endpoint) {
        if (Types.isString(endpoint)) {
            newEndpoints.push(endpoint);
            that.addClient(endpoint)
        } else {
            newEndpoints.push(endpoint.url);
            that.addClient(endpoint.url, endpoint.label);
        }
    });

    // disconnect endpoints not longer wanted
    oldEndpoints.forEach(function (endpoint) {
        if (newEndpoints.indexOf(endpoint) === -1) {
            that.removeClient(endpoint);
        }
    });

    return this;
};

ForwardingConfigurationManager.prototype.addClient = function (endpoint, label) {
    console.info('ForwardingConfigurationManager/addClient', endpoint);

    if (this._endpoints.indexOf(endpoint) > -1) {
        console.warn('ForwardingConfigurationManager/addClient', endpoint, 'already exists');
        return false;
    }

    this._endpoints.push(endpoint);
    this._endpointsLabels[endpoint] = label;

    this._connectEndpoints();

    return true;
};

ForwardingConfigurationManager.prototype.removeClient = function (endpoint) {
    console.info('ForwardingConfigurationManager/removeClient', endpoint);

    var index = this._endpoints.indexOf(endpoint);

    if (index === -1) {
        console.warn('ForwardingConfigurationManager/removeClient', endpoint, 'not exists');
        return false;
    }

    this._endpoints.splice(index, 1);

    this._disconnect(endpoint);

    delete this._intervals[endpoint];
    delete this._stats[endpoint];
    delete this._endpointsLabels[endpoint];

    return true;
};

ForwardingConfigurationManager.prototype.initialize = function () {
    if (this.isReady()) {
        console.warn('ForwardingConfigurationManager/initialize already initialized');
        return;
    }

    console.info('ForwardingConfigurationManager/initialize');

    ConfigurationManager.prototype.initialize.call(this);

    var that = this;

    this._connectEndpoints();

    // on reload reconnect all endpoints since they may have some ephemeral nodes to restore
    this.addEventListener(ForwardingConfigurationManager.Event.RELOADED, function () {
        console.info('ForwardingConfigurationManager/initialize reloading, endpoints:', that._endpoints.length);
        var endpoint;

        for (var i = 0, l = that._endpoints.length; i < l; i++) {
            endpoint = that._endpoints[i];

            console.info('ForwardingConfigurationManager/initialize reloading, endpoint', endpoint);

            that._send(endpoint, {
                method: 'reloaded'
            });
        }
    });

    setInterval(function () {
        Logger.gauge('dl.ForwardingConfigurationManager.queue.running', that._queueRunning);
        Logger.gauge('dl.ForwardingConfigurationManager.queue.waiting', that._queue.length);

        if (that._queueRunning > 0 || that._queue.length > 0) {
            console.info('ForwardingConfigurationManager queue running: %d, waiting: %d',
                that._queueRunning, that._queue.length);
        }
    }, ForwardingConfigurationManager.QUEUE_STATS_INTERVAL);
};

ForwardingConfigurationManager.prototype._ping = function (endpoint) {
    var that = this;

    if (!this._ws[endpoint] || this._ws[endpoint].readyState !== WebSocket.OPEN) {
        console.warn('ForwardingConfigurationManager/_ping', endpoint, 'not connected, skipping...');
        return;
    }

    this._ws[endpoint].ping(Date.now(), {}, true);
    this._pingTimeoutTimers[endpoint] = setTimeout(function () {
        console.warn('ForwardingConfigurationManager/_ping timeout', endpoint);

        if (isNaN(parseInt(that._pingTimeouts[endpoint]))) {
            that._pingTimeouts[endpoint] = 0;
        }

        that._pingTimeouts[endpoint]++;

        if (that._pingTimeouts[endpoint] >= that._maxPingTimeouts) {
            that._scheduleReconnect(endpoint, 0);
        }
    }, this._pingTimeout);
};

ForwardingConfigurationManager.prototype._connectEndpoints = function () {
    var that = this;

    this._endpoints.forEach(function (endpoint) {
        if (!that._ws[endpoint]) {
            that._connect(endpoint);
        }
    });
};

ForwardingConfigurationManager.prototype._scheduleReconnect = function (endpoint, currentInterval) {
    if (currentInterval > ForwardingConfigurationManager.RECONNECT_MAX_INTERVAL) {
        currentInterval = ForwardingConfigurationManager.RECONNECT_MAX_INTERVAL;
    }

    console.log('ForwardingConfigurationManager/_scheduleReconnect: reconnect', endpoint, 'after', currentInterval);

    var that = this;

    this._disconnect(endpoint);

    this._reconnectTimers[endpoint] = setTimeout(function () {
        that._intervals[endpoint] = currentInterval * ForwardingConfigurationManager.RECONNECT_DECAY;
        that._connect(endpoint);
    }, currentInterval);
};

ForwardingConfigurationManager.prototype._onClose = function (endpoint, code, lastInterval) {
    console.info('ForwardingConfigurationManager/_onClose:', endpoint);

    while (this._ephemeral[endpoint] && this._ephemeral[endpoint].length > 0) {
        var key = this._ephemeral[endpoint].shift();
        console.info('ForwardingConfigurationManager/_onClose: removing ephemeral key', key, 'for', endpoint);
        this.remove(key, {}, function () {});
    }

    delete this._ephemeral[endpoint];

    if (!this._ws[endpoint]) {
        console.info('ForwardingConfigurationManager/_onClose: client', endpoint, 'has been disconnected');
        return;
    }

    this._scheduleReconnect(endpoint, lastInterval);
};

ForwardingConfigurationManager.prototype._addClientWatch = function (endpoint, msgId, key) {
    var that = this;
    var creating = true;

    var onResult = function (cb, rc, data, stat) {
        if (creating) {
            that._send(endpoint, {
                id: msgId,
                method: 'client-watch-init',
                rc: rc || 0,
                data: data,
                stat: stat,
                error: null
            });

            creating = false;
            cb();
        } else {
            that._onWatchNotify(key, rc, data, stat);
        }
    };

    if (this._watches.hasOwnProperty(key)) {
        // already watching this key
        if (this._watches[key].indexOf(endpoint) === -1) {
            console.info('ForwardingConfigurationManager/_addClientWatch: has key, new endpoint',
                         key, '->', msgId, endpoint);
            // new client for this key
            this._watches[key].push(endpoint);
            this._endpointsWatches[endpoint].push(key);
        } else {
            console.info('ForwardingConfigurationManager/_addClientWatch: has key, has endpoint',
                         key, '->', msgId, endpoint);
        }

        // just get
        this._enqueue(function (cb) {
            that.get(key, onResult.bind(that, cb));
        });
    } else {
        console.info('ForwardingConfigurationManager/_addClientWatch: new key', key, '->', msgId, endpoint);
        this._watches[key] = [ endpoint ];
        this._endpointsWatches[endpoint].push(key);

        this._enqueue(function (cb) {
            that.watch(key, onResult.bind(that, cb));
        });
    }
};

ForwardingConfigurationManager.prototype._onWatchNotify = function (key, rc, data, stat) {
    var that = this;
    var endpoints = this._watches[key];

    if (!endpoints.length) {
        console.error('ForwardingConfigurationManager/_onWatchNotify no endpoints to notify for', key);
        return;
    }

    console.info('ForwardingConfigurationManager/_onWatchNotify:', key);

    endpoints.forEach(function (endpoint) {
        console.info('ForwardingConfigurationManager/_onWatchNotify:', key, '->', endpoint);

        that._send(endpoint, {
            method: 'notify',
            params: {
                key: key,
                rc: rc,
                data: data,
                stat: stat,
                error: null
            }
        });
    });
};

ForwardingConfigurationManager.prototype._enqueue = function (fn) {
    var that = this;

    //mam wolne sloty to wykonuje funkcje odrazu
    if (this._queueRunning < this._queueMax) {
        console.info('ForwardingConfigurationManager/enqueue: executing, left:',
                     this._queueRunning, this._queue.length);
        //najpierw wykonuje elementy na kolejce
        var fnx = this._queue.shift();

        if (fnx && fn) {
            this._queue.push(fn);
        } else if (!fnx && fn) {
            fnx = fn;
        }

        if (fnx) {
            this._queueRunning++;
            fnx(function () {
                that._queueRunning--;
                that._enqueue();
            });
        }
    } else if (fn) {
        //dodaje do kolejki jak za duzo naraz wykonuje
        this._queue.push(fn);
        console.info('ForwardingConfigurationManager/enqueue: enqueued, count: %d',
                     this._queueRunning, this._queue.length);
    }
};

ForwardingConfigurationManager.prototype._onMessage = function (endpoint, msg) { // jshint ignore: line
    if (msg.method !== 'get') {
        console.info('ForwardingConfigurationManager/_onMessage:', endpoint, ' -> ', msg.id, msg.method,
                     msg.params ? msg.params.key : '');
    }

    var ephemeral = this._ephemeral[endpoint];
    var that = this;

    var onResult = function (msg, cb, rc, data, stat) {
        that._send(endpoint, {
            id: msg.id,
            method: msg.method,
            rc: rc || 0,
            data: data,
            stat: stat,
            error: null
        });

        cb();
    };

    if (!this._driver.isConnected()) {
        console.warn('ForwardingConfigurationManager/_onMessage: manager not ready yet, endpoint:', endpoint);
        return onResult(msg, function () {}, -4 /* ZCONNECTIONLOSS */);
    }

    var params = msg.params;

    if (!params || !params.key) {
        console.warn('ForwardingConfigurationManager/_onMessage: wrong params, endpoint:', endpoint);
        return onResult(msg, function () {}, -1002, 'Wrong params');
    }

    this._stats[endpoint] = !this._stats[endpoint] ? 1 : ++this._stats[endpoint];

    switch (msg.method) {
        case 'get':
            this._enqueue(function (cb) {
                that.get(params.key, onResult.bind(that, msg, cb));
            });
            break;

        case 'watch':
            that._addClientWatch(endpoint, msg.id, params.key);
            break;

        case 'set':
            this._enqueue(function (cb) {
                that.set(params.key, params.value, params.options, onResult.bind(that, msg, cb));

                if (params.options && params.options.flags === 1 && ephemeral.indexOf(params.key) === -1) {
                    console.info('ForwardingConfigurationManager/_onMessage: creating ephemeral key',
                                 params.key, 'for', endpoint);
                    ephemeral.push(params.key);
                }
            });
            break;

        case 'sync':
            this._enqueue(function (cb) {
                that.sync(params.key, onResult.bind(that, msg, cb));
            });
            break;

        case 'remove':
            this._enqueue(function (cb) {
                that.remove(params.key, params.options, onResult.bind(that, msg, cb));

                var ephemeralIdx = ephemeral.indexOf(params.key);

                if (ephemeralIdx > -1) {
                    console.info('ForwardingConfigurationManager/_onMessage: deleting ephemeral key',
                                 params.key, 'for', endpoint);

                    ephemeral.splice(ephemeralIdx, 1);
                }
            });
            break;

        case 'exists':
            this._enqueue(function (cb) {
                that.exists(params.key, onResult.bind(that, msg, cb));
            });
            break;

        default:
            console.warn('ForwardingConfigurationManager/_onMessage: unknown method', msg.method, 'from', endpoint);
            return onResult(msg, function () {}, -1001, 'Unknown method');
    }
};

ForwardingConfigurationManager.prototype._send = function (endpoint, data) {
    if (this._ws[endpoint] && this._ws[endpoint].readyState === WebSocket.OPEN) {
        console.info('ForwardingConfigurationManager/send: sending to: %s', endpoint, data.id, data.method);
        this._ws[endpoint].send(JSON.stringify(data), { compress: false, binary: false }, function (err) {
            if (err) {
                console.error('ForwardingConfigurationManager/send failed:', err);
            }
        });
    } else {
        console.warn('ForwardingConfigurationManager/send, unable to send: %s, no socket or socket in wrong state',
                     endpoint);
    }
};

ForwardingConfigurationManager.prototype._disconnect = function (endpoint) {
    console.info('ForwardingConfigurationManager/_disconnect', endpoint);

    clearTimeout(this._pingTimeoutTimers[endpoint]);
    clearTimeout(this._reconnectTimers[endpoint]);

    if (this._endpointsWatches.hasOwnProperty(endpoint)) {
        for (var i = 0, l = this._endpointsWatches[endpoint].length; i < l; i++) {
            var key = this._endpointsWatches[endpoint][i];
            var idx = this._watches[key].indexOf(endpoint);

            console.info('ForwardingConfigurationManager/_disconnect removing watch', key, '->', endpoint);
            this._watches[key].splice(idx, 1);
        }
    }

    delete this._endpointsWatches[endpoint];
    delete this._pingTimeoutTimers[endpoint];
    delete this._pingTimeouts[endpoint];
    delete this._reconnectTimers[endpoint];

    if (this._ws.hasOwnProperty(endpoint)) {
        var ws = this._ws[endpoint];
        delete this._ws[endpoint];

        ws.close();
    }
};

ForwardingConfigurationManager.prototype._connect = function (endpoint) {
    if (!this.isReady()) {
        console.warn('ForwardingConfigurationManager/_connect', endpoint, 'not ready');
        return;
    }

    console.info('ForwardingConfigurationManager/_connect', endpoint);

    var that = this;
    var endpointStr = this._endpointsLabels[endpoint] || 'UNKNOWN';
    var currentInterval = this._intervals[endpoint] || ForwardingConfigurationManager.RECONNECT_INTERVAL;
    var ws = this._ws[endpoint] = new WebSocket(endpoint);

    this._ephemeral[endpoint] = [];
    this._endpointsWatches[endpoint] = [];

    ws.on('open', function () {
        console.info('ForwardingConfigurationManager/_connect: connected to', endpoint);
        currentInterval = ForwardingConfigurationManager.RECONNECT_INTERVAL;
    });

    ws.on('pong', function (data) {
        var pingDate = parseInt(data);
        var latency = Date.now() - pingDate;

        Logger.gauge(['dl.ForwardingConfigurationManager.endpoints', endpointStr, 'latency'], latency);
        Logger.gauge(['dl.ForwardingConfigurationManager.endpoints', endpointStr, 'requests'], that._stats[endpoint]);

        if (latency > 100 || that._stats[endpoint] > 0) {
            console.info('ForwardingConfigurationManager/ping endpoint:', endpoint, 'latency:', Date.now() - pingDate,
                'reqs:', that._stats[endpoint]);
        }

        that._pingTimeouts[endpoint] = 0;
        that._stats[endpoint] = 0;
        clearTimeout(that._pingTimeoutTimers[endpoint]);
    });

    ws.once('close', function (code) {
        that._onClose(endpoint, code, currentInterval);
    });

    ws.on('error', function (err) {
        Logger.counter(['dl.ForwardingConfigurationManager.endpoints', endpointStr, 'errors']);

        console.error('ForwardingConfigurationManager/_connect: error in connection to', endpoint, err);

        if (ws) {
            ws = null;
            that._scheduleReconnect(endpoint, currentInterval);
        }
    });

    ws.on('message', function (data) {
        try {
            data = JSON.parse(data);
        } catch (ex) {
            console.error('ForwardingConfigurationManager/_connect: failed to parse data from', endpoint);
            return;
        }

        that._onMessage(endpoint, data);
    });
};

ForwardingConfigurationManager.DEFAULT_PING_INTERVAL = 5000;
ForwardingConfigurationManager.DEFAULT_PING_TIMEOUT = 2500;
ForwardingConfigurationManager.MAX_PING_TIMEOUTS = 2;
ForwardingConfigurationManager.RECONNECT_INTERVAL = 500;
ForwardingConfigurationManager.RECONNECT_DECAY = 1.5;
ForwardingConfigurationManager.RECONNECT_MAX_INTERVAL = 5000;
ForwardingConfigurationManager.MAX_CONCURRENT_QUERIES = 20;
ForwardingConfigurationManager.QUEUE_STATS_INTERVAL = 5000;

ForwardingConfigurationManager.Event = ConfigurationManager.Event;

exports.ForwardingConfigurationManager = ForwardingConfigurationManager;