redis-json
Version:
A wrapper library to store JSON Objects in redis-hashsets and retrieve it back as JSON objects
662 lines (648 loc) • 23.4 kB
JavaScript
'use strict';
var util = require('util');
/*! *****************************************************************************
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
this file except in compliance with the License. You may obtain a copy of the
License at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions
and limitations under the License.
***************************************************************************** */
function __awaiter(thisArg, _arguments, P, generator) {
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
}
function rejected(value) {
try {
step(generator["throw"](value));
} catch (e) {
reject(e);
}
}
function step(result) {
result.done ? resolve(result.value) : new P(function (resolve) {
resolve(result.value);
}).then(fulfilled, rejected);
}
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
}
/**
* Return the given key if it's a string else
* parses it into number
*
* @param key
*
* @returns a string if it cannot be parsed to a number
* else returns the parsed number
*/
function parseKey(key) {
const numKey = Number(key);
return isNaN(numKey) || isHex(key) ? decodeKey(key) : numKey;
}
/**
* Encapsulate '.' in the given key, such
* that a '.' in the key is NOT misinterpreted
* during unflattening of the object
*
* @param key
*/
function encodeKey(key) {
return key.replace(/\./g, '/.');
}
/**
* Recover the actual key which was encoded earlier.
* This is done to allow a '.' in the key
*
* @param key
*/
function decodeKey(key) {
return key ? key.replace(/\/\./g, '.') : key;
}
const splitKey = (() => {
const keySplitReg = /(?<!\/)\./;
/**
* Splits the the given key based
* on the delimiter ('.')
*
* @param key
*/
return (key) => {
return key.split(keySplitReg);
};
})();
/**
* Checks the key is hex string or not.
*
* @param key
*/
function isHex(key) {
return key ? Boolean(key.match(/^0x[0-9a-f]+$/i)) : false;
}
var TYPE;
(function (TYPE) {
TYPE["OBJECT"] = "0";
TYPE["STRING"] = "1";
TYPE["NUMBER"] = "2";
TYPE["BOOLEAN"] = "3";
TYPE["FUNCTION"] = "4";
TYPE["UNDEFINED"] = "5";
TYPE["SYMBOL"] = "6";
})(TYPE || (TYPE = {}));
/**
* Returns true if the constructor name is known
* to us. Ex: Object, Array
*/
const isKnownContructor = (() => {
const knownConstructors = {
Object: true,
Array: true,
};
return (constructorName) => knownConstructors[constructorName];
})();
/**
* Returns true if the given value's
* type need to be skipped during storage.
* For ex: Symbol -> Since symbols are private,
* we DO NOT encourage them to be stored, hence
* we are skipping from storing the same.
*
* In case you've forked this library and want to
* add more type, then this is the place for you 🙂
*/
const isSkippedType = (() => {
const skippedType = {
symbol: true,
};
return (val) => !!skippedType[typeof val];
})();
/**
* Returns a shorter form of the type of the
* value that can be stored in redis.
* This also handles custom Classes by using
* their constructor names directly.
*
* @param val Value whose type needs to be computed
*/
const getTypeOf = (() => {
const shortTypes = {
object: TYPE.OBJECT,
string: TYPE.STRING,
number: TYPE.NUMBER,
boolean: TYPE.BOOLEAN,
function: TYPE.FUNCTION,
undefined: TYPE.UNDEFINED,
symbol: TYPE.SYMBOL,
};
return (val) => {
if (typeof val === 'object') {
// If the val is `null`
if (!val) {
return TYPE.OBJECT;
}
const constructorName = val.constructor.name;
return isKnownContructor(constructorName)
// if the val is {} or []
? TYPE.OBJECT
// if the val is Date or other custom classes / object
: constructorName;
}
return shortTypes[typeof val] || TYPE.STRING /** this is a fallback, just in case */;
};
})();
/**
* Returns the stringified version of the given value.
* However note that this method needs to take care,
* such that special values like undefined, null, false, true
* etc are also stringified correctly for storage.
*
* In case of a custom class / object, this method would
* call the provided stringifier (if any available), else
* would use `String(val)`
*
* @param val Value to be evaluated
* @param stringifier Custom stringifiers
*
* @returns Stringified value. If null is returned, then such a value must NOT
* be stored
*/
const getValueOf = (val, stringifier = {}) => {
var _a;
if (typeof val === 'object') {
// if the val is null
if (!val) {
return 'null';
}
const constructorName = (_a = val === null || val === void 0 ? void 0 : val.constructor) === null || _a === void 0 ? void 0 : _a.name;
return isKnownContructor(constructorName)
// if the val is {} or []
? JSON.stringify(val)
// if the val is Date or other custom classes / object
: stringifier[constructorName]
? stringifier[constructorName](val)
: String(val);
}
return String(val);
};
/**
* Converts the given value to the specified type.
* Also note that, if a custom className type is
* detected, then the provided custom Parser will
* be called (if any available), else will return
* the value as is.
*/
const getTypedVal = (() => {
const internalParsers = {
[TYPE.STRING]: val => val,
[TYPE.NUMBER]: Number,
[TYPE.BOOLEAN]: (val) => val === 'true',
[TYPE.FUNCTION]: val => val,
[TYPE.UNDEFINED]: () => undefined,
[TYPE.SYMBOL]: (val) => val,
[TYPE.OBJECT]: (() => {
const valMap = {
'{}': () => ({}),
'[]': () => [],
'null': () => null,
};
return (val) => valMap[val]();
})(),
};
return (type, val, parser = {}) => {
return internalParsers[type]
? internalParsers[type](val)
: parser[type]
? parser[type](val)
: val;
};
})();
const getDefaultResult = () => ({
data: {},
typeInfo: {},
arrayInfo: {},
});
/**
* @internal
*
* Class for flattening and unflattening an object / array
*
* This could've been a simple function but is rather a class
* because we are instantiating it during the constructor phase of
* JSONCache class by calling it with stringifier & parser options.
*/
class Flattener {
constructor(stringifier = {}, parser = {}) {
this.stringifier = stringifier;
this.parser = parser;
}
/**
* Flattens the given object and converts it
* to a dept of 1
*
* @param obj Object to be flattened
*/
flatten(obj) {
return this.traverse(obj, '', getDefaultResult());
}
/**
* Unflattens the given object to its original
* format and also applies the necessary types
* that it originally had
*
* @param flattened Flattened object
*/
unflatten(flattened) {
const typedData = this.mergeTypes(flattened);
let result;
Object.entries(typedData).some(([key, val]) => {
// if the key is '', it means that
// the flattened object / array is empty
if (!key) {
if (Object.keys(typedData).length <= 1) {
result = val;
return true;
}
else {
// when the initial data is {'': {}} and
// later a prop is added to the same, then
// the data would be {'': {}, prop: {...}}
// hence we need to continue the loop when
// the keys.length > 1
return false;
}
}
const splittedKeys = splitKey(key);
if (!result) {
result = typeof parseKey(splittedKeys[0]) === 'number' ? [] : {};
}
this.scaffoldStructure(result, splittedKeys, val);
return false;
});
return result;
}
/***********************************
* PRIVATE METHODS - Flatten helpers
**********************************/
traverse(target, basePath, result) {
if (!(target instanceof Object))
return result;
if (Array.isArray(target)) {
result.arrayInfo[basePath] = true;
}
const entries = Object.entries(target);
if (entries.length > 0) {
entries.forEach(([key, val]) => {
const encodedKey = encodeKey(key);
const path = appendPath(basePath, encodedKey);
if (val instanceof Object) {
this.traverse(val, path, result);
}
else {
this.assignResult(result, path, val);
}
});
}
else {
this.assignResult(result, basePath, target);
}
return result;
}
assignResult(result, path, val) {
if (!isSkippedType(val)) {
result.data[path] = getValueOf(val, this.stringifier);
result.typeInfo[path] = getTypeOf(val);
}
}
/*************************************
* PRIVATE METHODS - Unflatten helpers
*************************************/
mergeTypes(result) {
const { data, typeInfo } = result;
return Object.entries(data).reduce((merged, [path, val]) => {
merged[path] = getTypedVal(typeInfo[path], val, this.parser);
return merged;
}, {});
}
scaffoldStructure(tree, splittedKeys, val) {
// Loop until k1 has reached end of split
for (let i = 0, len = splittedKeys.length; i < len; i++) {
const k1 = parseKey(splittedKeys[i]);
const k2 = parseKey(splittedKeys[i + 1]);
if (typeof k2 === 'undefined') {
tree[k1] = val;
}
else {
const isObj = typeof tree[k1] === 'object';
if (!isObj)
tree[k1] = typeof k2 === 'number' ? [] : {};
tree = tree[k1];
}
}
}
}
function appendPath(basePath, key) {
return basePath ? `${basePath}.${key}` : key;
}
const Config = {
SCAN_COUNT: 100,
};
/**
* JSONCache eases the difficulties in storing a JSON in redis.
*
* It stores the JSON in hashset for simpler get and set of required
* fields. It also allows you to override/set specific fields in
* the JSON without rewriting the whole JSON tree. Which means that it
* is literally possible to `Object.deepAssign()`.
*
* Everytime you store an object, JSONCache would store two hashset
* in Redis, one for data and the other for type information. This helps
* during retrieval of data, to restore the type of data which was originally
* provided. All these workaround are needed because Redis DOES NOT support
* any other data type apart from String.
*
* Well the easiest way is to store an object in Redis is
* JSON.stringify(obj) and store the stringified result.
* But this can cause issue when the obj is
* too huge or when you would want to retrieve only specific fields
* from the JSON but do not want to parse the whole JSON.
* Also note that this method would end up in returing all the
* fields as strings and you would have no clue to identify the type of
* field.
*/
class JSONCache {
/**
* Intializes JSONCache instance
* @param redisClient RedisClient instance(Preferred ioredis - cient).
* It supports any redisClient instance that has
* `'hmset' | 'hmget' | 'hgetall' | 'expire' | 'del' | 'keys'`
* methods implemented
* @param options Options for controlling the prefix
*/
constructor(redisClient, options = {}) {
this.options = options;
this.options.prefix = typeof options.prefix === 'string' ? options.prefix : 'jc:';
this.redisClientInt = {
hmset: util.promisify(redisClient.hmset).bind(redisClient),
hmget: util.promisify(redisClient.hmget).bind(redisClient),
hgetall: util.promisify(redisClient.hgetall).bind(redisClient),
expire: util.promisify(redisClient.expire).bind(redisClient),
del: util.promisify(redisClient.del).bind(redisClient),
scan: util.promisify(redisClient.scan).bind(redisClient),
hincrbyfloat: util.promisify(redisClient.hincrbyfloat).bind(redisClient),
multi: (commands) => {
return new Promise((resolve, reject) => {
redisClient.multi(commands).exec((err, results) => {
if (err)
reject(err);
else
resolve(results);
});
});
},
};
this.flattener = new Flattener(options.stringifier, options.parser);
}
/**
* Flattens the given json object and
* stores it in Redis hashset
*
* @param key Redis key
* @param obj JSON object to be stored
* @param options
*/
set(key, obj, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
const flattened = this.flattener.flatten(obj);
const commands = yield this.getKeysToBeRemoved(key, flattened);
this.addSetCommands(key, flattened, commands, options.expire);
yield this.execCommand(commands, options.transaction);
});
}
get(key, ...fields) {
return __awaiter(this, void 0, void 0, function* () {
const [data, typeInfo] = yield Promise.all([
this.redisClientInt.hgetall(this.getKey(key)),
this.redisClientInt.hgetall(this.getTypeKey(key)),
]);
// Empty object is returned when
// the given key is not present
// in the cache
if (!(data && typeInfo)) {
return undefined;
}
const dataKeysLen = Object.keys(data).length;
const typeInfoKeysLen = Object.keys(typeInfo).length;
if (dataKeysLen !== typeInfoKeysLen || dataKeysLen === 0)
return undefined;
let result;
if (fields.length > 0) {
let dataKeys;
result = fields.reduce((res, field) => {
if (field in data) {
res.data[field] = data[field];
res.typeInfo[field] = typeInfo[field];
}
else {
const searchKey = `${field}.`;
(dataKeys || (dataKeys = Object.keys(data))).forEach(flattenedKey => {
if (flattenedKey.startsWith(searchKey)) {
res.data[flattenedKey] = data[flattenedKey];
res.typeInfo[flattenedKey] = typeInfo[flattenedKey];
}
});
}
return res;
}, { data: {}, typeInfo: {} });
}
else {
result = { data, typeInfo, arrayInfo: {} };
}
return this.flattener.unflatten(result);
});
}
/**
* Replace the entire hashset for the given key
*
* @param key Redis key
* @param obj JSON Object of type T
*/
rewrite(key, obj, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
const commands = [
['del', this.getKey(key)],
['del', this.getTypeKey(key)],
];
const flattened = this.flattener.flatten(obj);
this.addSetCommands(key, flattened, commands, options.expire);
yield this.execCommand(commands, options.transaction);
});
}
/**
* Removes/deletes all the keys in the JSON Cache,
* having the prefix.
*/
clearAll() {
return __awaiter(this, void 0, void 0, function* () {
let cursor = '0';
let keys;
do {
[cursor, keys] = yield this.redisClientInt.scan(cursor, 'MATCH', `${this.options.prefix}*`, 'COUNT', Config.SCAN_COUNT);
if (keys.length > 0) {
yield this.redisClientInt.del(...keys);
}
} while (cursor !== '0');
});
}
/**
* Removes the given key from Redis
*
* Please use this method instead of
* directly using `redis.del` as this method
* ensures that even the corresponding type info
* is removed. It also ensures that prefix is
* added to key, ensuring no other key is
* removed unintentionally
*
* @param key Redis key
*/
del(key, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
const commands = [
['del', this.getKey(key)],
['del', this.getTypeKey(key)],
];
yield this.execCommand(commands, options.transaction);
});
}
/**
* Increments the value of a variable in the JSON
* Note: You can increment multiple variables in the
* same command (Internally it will split it into multiple
* commands on the RedisDB)
*
* @example
* ```JS
* await jsonCache.incr(key, {messages: 10, profile: {age: 1}})
* ```
*
* @param key Redis Cache key
* @param obj Partial object specifying the path to the required
* variable along with value
*/
incr(key, obj, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
const flattened = this.flattener.flatten(obj);
const commands = [];
Object.entries(flattened.data).forEach(([path, incrVal]) => {
// This check is needed to avoid redis errors.
// It also helps while the user wants to increment the value
// within an array.
// Ex: rand: [null, null, 1] => this will increment the 3rd index by 1
if (flattened.typeInfo[path] !== TYPE.NUMBER) {
return;
}
commands.push(['hincrbyfloat', this.getKey(key), path, incrVal]);
});
yield this.execCommand(commands, options.transaction);
});
}
/******************
* PRIVATE METHODS
******************/
getKeysToBeRemoved(key, flattened) {
return __awaiter(this, void 0, void 0, function* () {
const commands = [];
// Check if the given obj has arrays and if it does
// then we must remove the current array stored in
// Cache and then set this array in the Cache
if (Object.keys(flattened.arrayInfo).length > 0) {
const currentObj = yield this.get(key);
if (currentObj) {
const currrentObjFlattened = this.flattener.flatten(currentObj).data;
const keysToBeRemoved = [];
// Get all paths matching the parent array path
Object.keys(flattened.arrayInfo).forEach(path => {
Object.keys(currrentObjFlattened).forEach(objPath => {
if (objPath.startsWith(path)) {
keysToBeRemoved.push(objPath);
}
});
});
if (keysToBeRemoved.length > 0) {
commands.push(['hdel', this.getKey(key), ...keysToBeRemoved]);
commands.push(['hdel', this.getTypeKey(key), ...keysToBeRemoved]);
}
}
}
return commands;
});
}
/**
* Returns the redis storage key for storing data
* by prefixing custom string, such that it
* doesn't collide with other keys in usage
*
* @param key Storage key
*/
getKey(key) {
return `${this.options.prefix}${key}`;
}
/**
* Returns the redis storage key for storing
* corresponding types by prefixing custom string,
* such that it doesn't collide with other keys
* in usage
*
* @param key Storage key
*/
getTypeKey(key) {
return `${this.options.prefix}${key}_t`;
}
/**
* Will add Set commands to the given array
* This logic was separated to remove code duplication
* in set & rewrite methods
*
* @param key Storage key
* @param flattened Flattened object containing data & typeInfo
* @param commands List of commands to which set commands has to be appended
* @param expire Redis Key expiry
*/
addSetCommands(key, flattened, commands, expire) {
commands.push(['hmset', this.getKey(key), flattened.data]);
commands.push(['hmset', this.getTypeKey(key), flattened.typeInfo]);
if (expire) {
commands.push(['expire', this.getKey(key), expire]);
commands.push(['expire', this.getTypeKey(key), expire]);
}
return commands;
}
execTransactionCommands(commands, transaction) {
commands.forEach(command => {
const [action, ...args] = command;
transaction[action](...args);
});
}
execCommand(commands, transaction) {
return __awaiter(this, void 0, void 0, function* () {
if (transaction) {
this.execTransactionCommands(commands, transaction);
return transaction;
}
else {
const result = yield this.redisClientInt.multi(commands);
return result;
}
});
}
}
module.exports = JSONCache;