UNPKG

core

Version:

Core Libs

788 lines (664 loc) 22.9 kB
var Pool = require('../pattern/Pool.js').Pool; var CredentialsProvider = require('../credentials/CredentialsProvider.js').CredentialsProvider; var Event = require('../event/Event').Event; var EventDispatcher = require('../event/EventDispatcher').EventDispatcher; var redis = require('redis'); var instances = []; var emptyFn = function () {}; var Redis = function (cp) { return Redis.getInstance(cp); }; Redis.getInstance = function (cp) { for (var i = 0, max = instances.length; i < max; i++) { if (instances[i][0] === cp) { return instances[i][1]; } } var obj = Object.create(Redis.prototype); EventDispatcher.call(obj); obj._debugEnabled = true; obj._options = { db: false, updateTimeout: 1000, defaultConnectionsCount: 1, redisConfig: { 'hbInterval': false, 'hbTimeout': false, 'max_attempts': 0, 'return_buffers': true, 'enable_offline_queue': false } }; obj._hosts = {}; obj._credentials = null; obj._readyClients = 0; obj._isReady = false; obj._onReadyCallback = false; obj._sortedHosts = []; cp.addEventListener(CredentialsProvider.Event.LOAD, function (e) { obj._onCredentialsChange(null, e.data); }, obj); cp.get(function () { obj._onCredentialsChange.apply(obj, arguments); }); obj._cp = cp; instances.push([cp, obj]); // in the old version it was running find out why?Maybe some bug with cp obj._updateTimeout = setTimeout(obj._onCredentialsChange.bind(obj), obj._options.updateTimeout); return obj; }; Redis.prototype = Object.create(EventDispatcher.prototype); /** * Enable/disable debug mode for redisClient * * @return {void} */ Redis.prototype._debug = function () { if (!this._debugEnabled) { return; } console.log.apply(console, arguments); }; /** * Gracefully kill unused redis numConnectionss using pool release and clean host object * * @return {void} */ Redis.prototype._cleanDeletedHosts = function () { var hosts = Object.keys(this._hosts); for (var i = 0, hostsL = hosts.length; i < hostsL; i++) { var server = this._hosts[hosts[i]]; if (!server.toDelete) { continue; } var clientCount = server.pool.getKnownLength(); if (clientCount === 0) { delete this._hosts[hosts[i]]; continue; } for (var j = 0; j < clientCount; j++) { this._deleteClient(hosts[i]); } } }; /** * Check if all connections are ready. * * @return {bool} - true if ready, otherwise false */ Redis.prototype.isReady = function () { return this._isReady; }; /** * Add onReady Function * * @param {Function} callback funct to be called when all clients are connected * @return {void} */ Redis.prototype.onReady = function (callback) { if (typeof callback !== 'function') { return; } this._onReadyCallback = callback; }; /** * Update redis clients to config given from credentialProvider * * @param {Object} - Err * @param {Object} - Credentials in CredentialProvider format * @return {void} */ Redis.prototype._onCredentialsChange = function (error, credentials) { var that = this; if (error) { console.error('Redis Credentials error', error); return; } if (this._updateTimeout) { clearTimeout(this._updateTimeout); this._updateTimeout = null; } this._cleanDeletedHosts(); if (credentials) { if (credentials.hasOwnProperty('connect_timeout')) { this._options.redisConfig.connect_timeout = credentials.connect_timeout; } if (this._options.hasOwnProperty('db')) { this._options.db = credentials.db; } else { this._options.db = false; } this._options.redisConfig.enable_offline_queue = credentials.enable_offline_queue || false; redis.debug_mode = that._debugEnabled = credentials.debug || false; this._options.redisConfig.hbInterval = credentials.hbInterval || false; this._options.redisConfig.hbTimeout = credentials.hbTimeout || false; } //parse credentials to class format credentials = this._normalizeCredentials(credentials); if (!credentials || typeof credentials !== 'object') { if (typeof this._credentials !== 'object') { console.error('Redis no credentials available'); return; } credentials = this._credentials; } else { this._credentials = credentials; } var credentialKeys = Object.keys(credentials); var hostsKeys = Object.keys(this._hosts); credentialKeys.forEach(function (i) { if (hostsKeys.indexOf(i) < 0) { that._hosts[i] = { connections: credentials[i], pool: new Pool(), active: 0, errors: 0, toDelete: false }; } }); hostsKeys.forEach(function (i) { if (credentialKeys.indexOf(i) < 0) { that._hosts[i].toDelete = true; } }); var hosts = Object.keys(this._hosts); for (var j = 0, hostsL = hosts.length; j < hostsL; j++) { if (credentials[hosts[j]]) { this._hosts[hosts[j]].connections = credentials[hosts[j]]; } var connected = this._hosts[hosts[j]].pool.getKnownLength(); if (connected !== this._hosts[hosts[j]].connections) { this._manageConnections(hosts[j], this._hosts[hosts[j]].connections - connected); } } //sort host for getShard method (backward compatibility) this._sortedHosts = Object.keys(this._hosts).sort(function (a, b) { var hostA = a.split(':'); var hostB = b.split(':'); if (hostA[0] !== hostB[0]) { return hostA[0] > hostB[0] ? 1 : -1; } return hostA[1] - hostB[1]; }); }; /** * Normalize params from credentialsProvider to class format * * @param {Object} * @return {Object} - object with {ip:port => numConnections} */ Redis.prototype._normalizeCredentials = function (items) { if (!items || !items.hosts) { return false; } var result = {}; if (typeof items.connections === 'undefined') { items.connections = this._options.defaultConnectionsCount; } items.connections = parseInt(items.connections); var connections = isNaN(items.connections) ? 0 : items.connections; for (var i = 0; i < items.hosts.length; i++) { if (items.hosts[i].host && items.hosts[i].port) { //if we have two ore more same entries in config we add it to existing entry if (result[items.hosts[i].host + ':' + items.hosts[i].port]) { result[items.hosts[i].host + ':' + items.hosts[i].port] += connections; } else { result[items.hosts[i].host + ':' + items.hosts[i].port] = connections; } } } return result; }; /** * Add/delete connections to redis * * @param {string} - ip:port string * @param {int} - number of connections * @return {void} */ Redis.prototype._manageConnections = function (addr, connections) { if (!connections) { return; } if (!this._validateHost(addr)) { console.error('Bad hostname adres:'.addr); return; } var toRemove = false; if (connections < 0) { toRemove = true; connections = (connections * -1); } for (var i = 0; i < connections; i++) { if (!toRemove) { this._createClient(addr); } else { this._deleteClient(addr); } } }; /** * close single redis client * * @param {string} - host with port * @return {void} */ Redis.prototype._deleteClient = function (addr) { if (!this._validateHost(addr)) { console.error('Bad hostname adres:', addr); return; } this._hosts[addr].pool.get(function (client) { client.quit(function (err) { if (err) { client.connection_gone('delete_client'); } }); }); }; /** * Simple hostname:ip string validation * * @param {string} addr - ip:port * @return {bool} v-alidation result */ Redis.prototype._validateHost = function (addr) { if (!addr || typeof addr !== 'string' || addr.indexOf(':') === -1 || addr.split(':').length !== 2) { return false; } return true; }; /** * Check if all connections are established at start and run onReady callback (only runs at client start) * * @return {void} */ Redis.prototype._readyCheck = function () { var that = this; var readyTmp = 0; if (this._isReady) { return; } that._readyClients++; Object.keys(this._hosts).forEach(function (host) { readyTmp += that._hosts[host].connections; }); if (readyTmp === this._readyClients) { this._isReady = true; this.dispatchEvent(new Event(Redis.Event.READY)); if (typeof this._onReadyCallback === 'function') { this._onReadyCallback(); } } }; /** * Add redis client to pool * * @param {string} - ip:port * @return {void} */ Redis.prototype._createClient = function (addr) { var that = this; var countActive = false; if (!this._validateHost(addr)) { console.error('Bad hostname adres:', addr); return; } if (!that._hosts[addr]) { console.error('Host not found in pools'); return; } var host = addr.split(':', 2); var client = redis.createClient(host[1], host[0], that._options.redisConfig); that._heartbeat(client); client.on('connect', function () { that._debug('Redis client connected to', addr); }); client.on('ready', function () { //increment connection active counter and add flag that connection is connected //redis default flag cant be use because on end event its set to false that._hosts[addr].active++; countActive = true; if (that._options.db || that._options.db === 0) { client.select(that._options.db, function (error) { if (error) { console.error('Redis client error', error); that._hosts[addr].errors++; } else { that._debug('Redis client', addr, 'ready'); that._readyCheck(); } }); } else { that._debug('Redis client', addr, 'ready'); that._readyCheck(); } }); client.on('error', function (error) { console.error('Redis client error:', addr, 'Error:', error); that._hosts[addr].errors++; that._hosts[addr].pool.markForDeletion(client); that._hosts[addr].pool.release(client); client.connection_gone('error'); }); client.on('end', function () { //disable client reconnect by set closing var client.closing = true; //if connection was active and it disconnects remove it from active counter if (countActive) { that._hosts[addr].active--; countActive = false; } if (!that._hosts[addr]) { console.error('Cannot find host', addr, ' onRedisClientEnd'); return; } if (that._hbTimeout) { clearTimeout(that._hbTimeout); } that._hosts[addr].pool.markForDeletion(client); that._hosts[addr].pool.release(client); client.end(); //when client ends check connections count if (!that._updateTimeout) { that._updateTimeout = setTimeout(that._onCredentialsChange.bind(that), that._options.updateTimeout); } that._debug('client :', addr, 'disconnected'); }); that._hosts[addr].pool.add(client); }; /** * Add hearbeat to redis client * * @param {Object} client Redis Client object * @return {void} */ Redis.prototype._heartbeat = function (client) { var that = this; if (!client) { return; } if (client._hbTimeout) { clearTimeout(client._hbTimeout); } if (client.ready && client.options.hbInterval) { //add timeout for ping command if (client.options.hbTimeout) { client._hbWait = setTimeout(function () { //send event connection gone //its run flush_and_error <- run all queued callback with error //its run end event (on end we set closing to true because of disable reconnects) client.quit(emptyFn); client.connection_gone('ping'); }, client.options.hbTimeout); } client.ping(function () { clearTimeout(client._hbWait); }); } //set timer for hb //if hbtimer is not set, timer from credentail checking will be used //its because of checking if hb is enabled or not var timer = client.options.hbInterval || this._options.updateTimeout; if (!timer) { return; } client._hbTimeout = setTimeout(function () { that._heartbeat(client); }, timer); }; /** * Returns object with connection statistics * * @return {Object} */ Redis.prototype.getStats = function () { var result = {}; var hosts = Object.keys(this._hosts); for (var j = 0, hostsL = hosts.length; j < hostsL; j++) { var host = this._hosts[hosts[j]]; //do not show deleted connections in stats if (host.toDelete) { continue; } //active key deprecated result[hosts[j]] = { known: host.pool.getKnownLength(), spare: host.pool.getSpareLength(), waiting: host.pool.getWaitingLength(), active: host.active, errors: host.errors, queued: host.pool.getQueued() }; // reset error counter host.errors = 0; } return result; }; /** * Generate crc32 hash from string * * @param {String} - string to hash * @return {string} hashed key */ Redis.prototype._hashKey = function (key) { var digest = crc32(key); var activeHosts = 0; var hosts = Object.keys(this._hosts); for (var i = 0, hl = hosts.length; i < hl; i++) { if (!this._hosts[hosts[i]].toDelete) { activeHosts++; } } if (!activeHosts) { return undefined; } else { return digest % activeHosts; } }; /** * Destroy all connections and disable update interval * * @return {void} */ Redis.prototype.destroy = function () { var that = this; clearTimeout(this._updateTimeout); for (var host in this._hosts) { var server = that._hosts[host]; var clientsCount = server.pool.getKnownLength(); for (var j = 0; j < clientsCount; j++) { that._deleteClient(); } } }; /** * Get redis hostname assigned to key * * @param {string} - * @return {string} - redis key */ Redis.prototype._getShard = function (key) { var pool = this._hashKey(key); var hostsL = Object.keys(this._hosts).length; for (var i = 0; i < hostsL; i++) { if (pool > hostsL) { pool = 0; } if (!this._sortedHosts[pool] || !this._hosts[this._sortedHosts[pool]] || this._hosts[this._sortedHosts[pool]].toDelete) { pool++; continue; } if (this._hosts[this._sortedHosts[pool]].pool.getKnownLength() > 0) { return this._sortedHosts[pool]; } console.info('Dont know any active client for shard', this._sortedHosts[pool]); pool++; } }; /** * Returns redis client by keys * * @param {string} key key for shard * @param {Function} callback call with err, client, host * @return {void} */ Redis.prototype.getConnectionByKey = function (key, callback) { var host = this._getShard(key); if (!callback) { console.error('Callback is required'); return; } if (host === undefined) { if (callback) { callback('All shards are down', null); } return; } this._hosts[host].pool.get(function (client) { callback(null, client, host); }); }; /** * Function ovverides redis client to add shard support * * @param * @return {void} */ var cmd = function (cmd) { return function () { var that = this; var tmp = new Array(arguments.length); for (var i = 0; i < arguments.length; i++) { tmp[i] = arguments[i]; } var key = tmp[0]; var cb = tmp.pop(); if (typeof cb !== 'function') { tmp.push(cb); cb = function (err) { if (err) { console.error('Redis/Error cmd=%s error="%s"', cmd, err); } }; } this.getConnectionByKey(key, function (err, client, host) { if (err) { return cb(err, null); } if (client === Pool.ERROR.MAX_QUEUED_REACHED) { cb(Pool.ERROR.MAX_QUEUED_REACHED, null); return; } tmp.push(function (err, data) { if (err) { console.warn('Redis/Error cmd=%s error="%s"', cmd, err); that._hosts[host].errors++; } that._hosts[host].pool.release(client); cb(err, data); }); client[cmd].apply(client, tmp); }); }; }; CMDS = []; var CMDS = ['get', 'set', 'setnx', 'setex', 'append', 'strlen', 'del', 'exists', 'setbit', 'getbit', 'setrange', 'getrange', 'substr', 'incr', 'decr', 'mget', 'rpush', 'lpush', 'rpushx', 'lpushx', 'linsert', 'rpop', 'lpop', 'brpop', 'brpoplpush', 'blpop', 'llen', 'lindex', 'lset', 'lrange', 'ltrim', 'lrem', 'rpoplpush', 'sadd', 'srem', 'smove', 'sismember', 'scard', 'spop', 'srandmember', 'sinter', 'sinterstore', 'sunion', 'sunionstore', 'sdiff', 'sdiffstore', 'smembers', 'zadd', 'zincrby', 'zrem', 'zremrangebyscore', 'zremrangebyrank', 'zunionstore', 'zinterstore', 'zrange', 'zrangebyscore', 'zrevrangebyscore', 'zcount', 'zrevrange', 'zcard', 'zscore', 'zrank', 'zrevrank', 'hset', 'hsetnx', 'hget', 'hmset', 'hmget', 'hincrby', 'hdel', 'hlen', 'hkeys', 'hvals', 'hgetall', 'hexists', 'incrby', 'decrby', 'getset', 'mset', 'msetnx', 'randomkey', 'select', 'move', 'rename', 'renamenx', 'expire', 'expireat', 'keys', 'dbsize', 'auth', 'ping', 'echo', 'save', 'bgsave', 'bgrewriteaof', 'shutdown', 'lastsave', 'type', 'multi', 'exec', 'discard', 'sync', 'flushdb', 'flushall', 'sort', 'info', 'monitor', 'ttl', 'persist', 'slaveof', 'debug', 'config', 'subscribe', 'unsubscribe', 'psubscribe', 'punsubscribe', 'publish', 'watch', 'unwatch', 'cluster', 'restore', 'migrate', 'dump', 'object', 'client', 'eval', 'evalsha' ]; var i; for (i = 0; i < CMDS.length; i++) { Redis.prototype[CMDS[i]] = cmd(CMDS[i]); } var crc32 = function (str, hex) { var crc = ~0; var i, l; for (i = 0, l = str.length; i < l; i++) { crc = (crc >>> 8) ^ crc32tab[(crc ^ str.charCodeAt(i)) & 0xff]; } crc = Math.abs(crc ^ -1); return hex ? crc.toString(16) : crc; }; var crc32tab = [ 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d ]; Redis.Event = {}; Redis.Event.READY = 'Redis.Event.READY'; exports.Redis = Redis;