core
Version:
Core Libs
788 lines (664 loc) • 22.9 kB
JavaScript
var Pool = require('../pattern/Pool.js').Pool;
var CredentialsProvider = require('../credentials/CredentialsProvider.js').CredentialsProvider;
var Event = require('../event/Event').Event;
var EventDispatcher = require('../event/EventDispatcher').EventDispatcher;
var redis = require('redis');
var instances = [];
var emptyFn = function () {};
var Redis = function (cp) {
return Redis.getInstance(cp);
};
Redis.getInstance = function (cp) {
for (var i = 0, max = instances.length; i < max; i++) {
if (instances[i][0] === cp) {
return instances[i][1];
}
}
var obj = Object.create(Redis.prototype);
EventDispatcher.call(obj);
obj._debugEnabled = true;
obj._options = {
db: false,
updateTimeout: 1000,
defaultConnectionsCount: 1,
redisConfig: {
'hbInterval': false,
'hbTimeout': false,
'max_attempts': 0,
'return_buffers': true,
'enable_offline_queue': false
}
};
obj._hosts = {};
obj._credentials = null;
obj._readyClients = 0;
obj._isReady = false;
obj._onReadyCallback = false;
obj._sortedHosts = [];
cp.addEventListener(CredentialsProvider.Event.LOAD, function (e) {
obj._onCredentialsChange(null, e.data);
}, obj);
cp.get(function () {
obj._onCredentialsChange.apply(obj, arguments);
});
obj._cp = cp;
instances.push([cp, obj]);
// in the old version it was running find out why?Maybe some bug with cp
obj._updateTimeout = setTimeout(obj._onCredentialsChange.bind(obj), obj._options.updateTimeout);
return obj;
};
Redis.prototype = Object.create(EventDispatcher.prototype);
/**
* Enable/disable debug mode for redisClient
*
* @return {void}
*/
Redis.prototype._debug = function () {
if (!this._debugEnabled) {
return;
}
console.log.apply(console, arguments);
};
/**
* Gracefully kill unused redis numConnectionss using pool release and clean host object
*
* @return {void}
*/
Redis.prototype._cleanDeletedHosts = function () {
var hosts = Object.keys(this._hosts);
for (var i = 0, hostsL = hosts.length; i < hostsL; i++) {
var server = this._hosts[hosts[i]];
if (!server.toDelete) {
continue;
}
var clientCount = server.pool.getKnownLength();
if (clientCount === 0) {
delete this._hosts[hosts[i]];
continue;
}
for (var j = 0; j < clientCount; j++) {
this._deleteClient(hosts[i]);
}
}
};
/**
* Check if all connections are ready.
*
* @return {bool} - true if ready, otherwise false
*/
Redis.prototype.isReady = function () {
return this._isReady;
};
/**
* Add onReady Function
*
* @param {Function} callback funct to be called when all clients are connected
* @return {void}
*/
Redis.prototype.onReady = function (callback) {
if (typeof callback !== 'function') {
return;
}
this._onReadyCallback = callback;
};
/**
* Update redis clients to config given from credentialProvider
*
* @param {Object} - Err
* @param {Object} - Credentials in CredentialProvider format
* @return {void}
*/
Redis.prototype._onCredentialsChange = function (error, credentials) {
var that = this;
if (error) {
console.error('Redis Credentials error', error);
return;
}
if (this._updateTimeout) {
clearTimeout(this._updateTimeout);
this._updateTimeout = null;
}
this._cleanDeletedHosts();
if (credentials) {
if (credentials.hasOwnProperty('connect_timeout')) {
this._options.redisConfig.connect_timeout = credentials.connect_timeout;
}
if (this._options.hasOwnProperty('db')) {
this._options.db = credentials.db;
} else {
this._options.db = false;
}
this._options.redisConfig.enable_offline_queue = credentials.enable_offline_queue || false;
redis.debug_mode = that._debugEnabled = credentials.debug || false;
this._options.redisConfig.hbInterval = credentials.hbInterval || false;
this._options.redisConfig.hbTimeout = credentials.hbTimeout || false;
}
//parse credentials to class format
credentials = this._normalizeCredentials(credentials);
if (!credentials || typeof credentials !== 'object') {
if (typeof this._credentials !== 'object') {
console.error('Redis no credentials available');
return;
}
credentials = this._credentials;
} else {
this._credentials = credentials;
}
var credentialKeys = Object.keys(credentials);
var hostsKeys = Object.keys(this._hosts);
credentialKeys.forEach(function (i) {
if (hostsKeys.indexOf(i) < 0) {
that._hosts[i] = {
connections: credentials[i],
pool: new Pool(),
active: 0,
errors: 0,
toDelete: false
};
}
});
hostsKeys.forEach(function (i) {
if (credentialKeys.indexOf(i) < 0) {
that._hosts[i].toDelete = true;
}
});
var hosts = Object.keys(this._hosts);
for (var j = 0, hostsL = hosts.length; j < hostsL; j++) {
if (credentials[hosts[j]]) {
this._hosts[hosts[j]].connections = credentials[hosts[j]];
}
var connected = this._hosts[hosts[j]].pool.getKnownLength();
if (connected !== this._hosts[hosts[j]].connections) {
this._manageConnections(hosts[j], this._hosts[hosts[j]].connections - connected);
}
}
//sort host for getShard method (backward compatibility)
this._sortedHosts = Object.keys(this._hosts).sort(function (a, b) {
var hostA = a.split(':');
var hostB = b.split(':');
if (hostA[0] !== hostB[0]) {
return hostA[0] > hostB[0] ? 1 : -1;
}
return hostA[1] - hostB[1];
});
};
/**
* Normalize params from credentialsProvider to class format
*
* @param {Object}
* @return {Object} - object with {ip:port => numConnections}
*/
Redis.prototype._normalizeCredentials = function (items) {
if (!items || !items.hosts) {
return false;
}
var result = {};
if (typeof items.connections === 'undefined') {
items.connections = this._options.defaultConnectionsCount;
}
items.connections = parseInt(items.connections);
var connections = isNaN(items.connections) ? 0 : items.connections;
for (var i = 0; i < items.hosts.length; i++) {
if (items.hosts[i].host && items.hosts[i].port) {
//if we have two ore more same entries in config we add it to existing entry
if (result[items.hosts[i].host + ':' + items.hosts[i].port]) {
result[items.hosts[i].host + ':' + items.hosts[i].port] += connections;
} else {
result[items.hosts[i].host + ':' + items.hosts[i].port] = connections;
}
}
}
return result;
};
/**
* Add/delete connections to redis
*
* @param {string} - ip:port string
* @param {int} - number of connections
* @return {void}
*/
Redis.prototype._manageConnections = function (addr, connections) {
if (!connections) {
return;
}
if (!this._validateHost(addr)) {
console.error('Bad hostname adres:'.addr);
return;
}
var toRemove = false;
if (connections < 0) {
toRemove = true;
connections = (connections * -1);
}
for (var i = 0; i < connections; i++) {
if (!toRemove) {
this._createClient(addr);
} else {
this._deleteClient(addr);
}
}
};
/**
* close single redis client
*
* @param {string} - host with port
* @return {void}
*/
Redis.prototype._deleteClient = function (addr) {
if (!this._validateHost(addr)) {
console.error('Bad hostname adres:', addr);
return;
}
this._hosts[addr].pool.get(function (client) {
client.quit(function (err) {
if (err) {
client.connection_gone('delete_client');
}
});
});
};
/**
* Simple hostname:ip string validation
*
* @param {string} addr - ip:port
* @return {bool} v-alidation result
*/
Redis.prototype._validateHost = function (addr) {
if (!addr || typeof addr !== 'string' || addr.indexOf(':') === -1 || addr.split(':').length !== 2) {
return false;
}
return true;
};
/**
* Check if all connections are established at start and run onReady callback (only runs at client start)
*
* @return {void}
*/
Redis.prototype._readyCheck = function () {
var that = this;
var readyTmp = 0;
if (this._isReady) {
return;
}
that._readyClients++;
Object.keys(this._hosts).forEach(function (host) {
readyTmp += that._hosts[host].connections;
});
if (readyTmp === this._readyClients) {
this._isReady = true;
this.dispatchEvent(new Event(Redis.Event.READY));
if (typeof this._onReadyCallback === 'function') {
this._onReadyCallback();
}
}
};
/**
* Add redis client to pool
*
* @param {string} - ip:port
* @return {void}
*/
Redis.prototype._createClient = function (addr) {
var that = this;
var countActive = false;
if (!this._validateHost(addr)) {
console.error('Bad hostname adres:', addr);
return;
}
if (!that._hosts[addr]) {
console.error('Host not found in pools');
return;
}
var host = addr.split(':', 2);
var client = redis.createClient(host[1], host[0], that._options.redisConfig);
that._heartbeat(client);
client.on('connect', function () {
that._debug('Redis client connected to', addr);
});
client.on('ready', function () {
//increment connection active counter and add flag that connection is connected
//redis default flag cant be use because on end event its set to false
that._hosts[addr].active++;
countActive = true;
if (that._options.db || that._options.db === 0) {
client.select(that._options.db, function (error) {
if (error) {
console.error('Redis client error', error);
that._hosts[addr].errors++;
} else {
that._debug('Redis client', addr, 'ready');
that._readyCheck();
}
});
} else {
that._debug('Redis client', addr, 'ready');
that._readyCheck();
}
});
client.on('error', function (error) {
console.error('Redis client error:', addr, 'Error:', error);
that._hosts[addr].errors++;
that._hosts[addr].pool.markForDeletion(client);
that._hosts[addr].pool.release(client);
client.connection_gone('error');
});
client.on('end', function () {
//disable client reconnect by set closing var
client.closing = true;
//if connection was active and it disconnects remove it from active counter
if (countActive) {
that._hosts[addr].active--;
countActive = false;
}
if (!that._hosts[addr]) {
console.error('Cannot find host', addr, ' onRedisClientEnd');
return;
}
if (that._hbTimeout) {
clearTimeout(that._hbTimeout);
}
that._hosts[addr].pool.markForDeletion(client);
that._hosts[addr].pool.release(client);
client.end();
//when client ends check connections count
if (!that._updateTimeout) {
that._updateTimeout = setTimeout(that._onCredentialsChange.bind(that), that._options.updateTimeout);
}
that._debug('client :', addr, 'disconnected');
});
that._hosts[addr].pool.add(client);
};
/**
* Add hearbeat to redis client
*
* @param {Object} client Redis Client object
* @return {void}
*/
Redis.prototype._heartbeat = function (client) {
var that = this;
if (!client) {
return;
}
if (client._hbTimeout) {
clearTimeout(client._hbTimeout);
}
if (client.ready && client.options.hbInterval) {
//add timeout for ping command
if (client.options.hbTimeout) {
client._hbWait = setTimeout(function () {
//send event connection gone
//its run flush_and_error <- run all queued callback with error
//its run end event (on end we set closing to true because of disable reconnects)
client.quit(emptyFn);
client.connection_gone('ping');
}, client.options.hbTimeout);
}
client.ping(function () {
clearTimeout(client._hbWait);
});
}
//set timer for hb
//if hbtimer is not set, timer from credentail checking will be used
//its because of checking if hb is enabled or not
var timer = client.options.hbInterval || this._options.updateTimeout;
if (!timer) {
return;
}
client._hbTimeout = setTimeout(function () {
that._heartbeat(client);
}, timer);
};
/**
* Returns object with connection statistics
*
* @return {Object}
*/
Redis.prototype.getStats = function () {
var result = {};
var hosts = Object.keys(this._hosts);
for (var j = 0, hostsL = hosts.length; j < hostsL; j++) {
var host = this._hosts[hosts[j]];
//do not show deleted connections in stats
if (host.toDelete) {
continue;
}
//active key deprecated
result[hosts[j]] = {
known: host.pool.getKnownLength(),
spare: host.pool.getSpareLength(),
waiting: host.pool.getWaitingLength(),
active: host.active,
errors: host.errors,
queued: host.pool.getQueued()
};
// reset error counter
host.errors = 0;
}
return result;
};
/**
* Generate crc32 hash from string
*
* @param {String} - string to hash
* @return {string} hashed key
*/
Redis.prototype._hashKey = function (key) {
var digest = crc32(key);
var activeHosts = 0;
var hosts = Object.keys(this._hosts);
for (var i = 0, hl = hosts.length; i < hl; i++) {
if (!this._hosts[hosts[i]].toDelete) {
activeHosts++;
}
}
if (!activeHosts) {
return undefined;
} else {
return digest % activeHosts;
}
};
/**
* Destroy all connections and disable update interval
*
* @return {void}
*/
Redis.prototype.destroy = function () {
var that = this;
clearTimeout(this._updateTimeout);
for (var host in this._hosts) {
var server = that._hosts[host];
var clientsCount = server.pool.getKnownLength();
for (var j = 0; j < clientsCount; j++) {
that._deleteClient();
}
}
};
/**
* Get redis hostname assigned to key
*
* @param {string} -
* @return {string} - redis key
*/
Redis.prototype._getShard = function (key) {
var pool = this._hashKey(key);
var hostsL = Object.keys(this._hosts).length;
for (var i = 0; i < hostsL; i++) {
if (pool > hostsL) {
pool = 0;
}
if (!this._sortedHosts[pool] || !this._hosts[this._sortedHosts[pool]] || this._hosts[this._sortedHosts[pool]].toDelete) {
pool++;
continue;
}
if (this._hosts[this._sortedHosts[pool]].pool.getKnownLength() > 0) {
return this._sortedHosts[pool];
}
console.info('Dont know any active client for shard', this._sortedHosts[pool]);
pool++;
}
};
/**
* Returns redis client by keys
*
* @param {string} key key for shard
* @param {Function} callback call with err, client, host
* @return {void}
*/
Redis.prototype.getConnectionByKey = function (key, callback) {
var host = this._getShard(key);
if (!callback) {
console.error('Callback is required');
return;
}
if (host === undefined) {
if (callback) {
callback('All shards are down', null);
}
return;
}
this._hosts[host].pool.get(function (client) {
callback(null, client, host);
});
};
/**
* Function ovverides redis client to add shard support
*
* @param
* @return {void}
*/
var cmd = function (cmd) {
return function () {
var that = this;
var tmp = new Array(arguments.length);
for (var i = 0; i < arguments.length; i++) {
tmp[i] = arguments[i];
}
var key = tmp[0];
var cb = tmp.pop();
if (typeof cb !== 'function') {
tmp.push(cb);
cb = function (err) {
if (err) {
console.error('Redis/Error cmd=%s error="%s"', cmd, err);
}
};
}
this.getConnectionByKey(key, function (err, client, host) {
if (err) {
return cb(err, null);
}
if (client === Pool.ERROR.MAX_QUEUED_REACHED) {
cb(Pool.ERROR.MAX_QUEUED_REACHED, null);
return;
}
tmp.push(function (err, data) {
if (err) {
console.warn('Redis/Error cmd=%s error="%s"', cmd, err);
that._hosts[host].errors++;
}
that._hosts[host].pool.release(client);
cb(err, data);
});
client[cmd].apply(client, tmp);
});
};
};
CMDS = [];
var CMDS = ['get', 'set', 'setnx', 'setex', 'append', 'strlen', 'del', 'exists',
'setbit', 'getbit', 'setrange', 'getrange', 'substr', 'incr', 'decr', 'mget', 'rpush',
'lpush', 'rpushx', 'lpushx', 'linsert', 'rpop', 'lpop', 'brpop', 'brpoplpush', 'blpop',
'llen', 'lindex', 'lset', 'lrange', 'ltrim', 'lrem', 'rpoplpush', 'sadd', 'srem', 'smove',
'sismember', 'scard', 'spop', 'srandmember', 'sinter', 'sinterstore', 'sunion', 'sunionstore',
'sdiff', 'sdiffstore', 'smembers', 'zadd', 'zincrby', 'zrem', 'zremrangebyscore', 'zremrangebyrank',
'zunionstore', 'zinterstore', 'zrange', 'zrangebyscore', 'zrevrangebyscore', 'zcount', 'zrevrange',
'zcard', 'zscore', 'zrank', 'zrevrank', 'hset', 'hsetnx', 'hget', 'hmset', 'hmget', 'hincrby', 'hdel', 'hlen',
'hkeys', 'hvals', 'hgetall', 'hexists', 'incrby', 'decrby', 'getset', 'mset', 'msetnx', 'randomkey', 'select',
'move', 'rename', 'renamenx', 'expire', 'expireat', 'keys', 'dbsize', 'auth', 'ping', 'echo', 'save',
'bgsave', 'bgrewriteaof', 'shutdown', 'lastsave', 'type', 'multi', 'exec', 'discard', 'sync', 'flushdb',
'flushall', 'sort', 'info', 'monitor', 'ttl', 'persist', 'slaveof', 'debug', 'config', 'subscribe',
'unsubscribe',
'psubscribe', 'punsubscribe', 'publish', 'watch', 'unwatch', 'cluster', 'restore', 'migrate', 'dump', 'object',
'client', 'eval', 'evalsha'
];
var i;
for (i = 0; i < CMDS.length; i++) {
Redis.prototype[CMDS[i]] = cmd(CMDS[i]);
}
var crc32 = function (str, hex) {
var crc = ~0;
var i, l;
for (i = 0, l = str.length; i < l; i++) {
crc = (crc >>> 8) ^ crc32tab[(crc ^ str.charCodeAt(i)) & 0xff];
}
crc = Math.abs(crc ^ -1);
return hex ? crc.toString(16) : crc;
};
var crc32tab = [
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
];
Redis.Event = {};
Redis.Event.READY = 'Redis.Event.READY';
exports.Redis = Redis;