transitory
Version:
In-memory cache with high hit rates via LFU eviction. Supports time-based expiration, automatic loading and metrics.
735 lines • 27.9 kB
JavaScript
"use strict";
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __values = (this && this.__values) || function(o) {
var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
if (m) return m.call(o);
if (o && typeof o.length === "number") return {
next: function () {
if (o && i >= o.length) o = void 0;
return { value: o && o[i++], done: !o };
}
};
throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
};
var __read = (this && this.__read) || function (o, n) {
var m = typeof Symbol === "function" && o[Symbol.iterator];
if (!m) return o;
var i = m.call(o), r, ar = [], e;
try {
while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
}
catch (error) { e = { error: error }; }
finally {
try {
if (r && !r.done && (m = i["return"])) m.call(i);
}
finally { if (e) throw e.error; }
}
return ar;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BoundedCache = void 0;
var AbstractCache_1 = require("../AbstractCache");
var CacheNode_1 = require("../CacheNode");
var RemovalReason_1 = require("../RemovalReason");
var symbols_1 = require("../symbols");
var CountMinSketch_1 = require("./CountMinSketch");
var percentInMain = 0.99;
var percentProtected = 0.8;
var percentOverflow = 0.01;
var adaptiveRestartThreshold = 0.05;
var adaptiveStepPercent = 0.0625;
var adaptiveStepDecayRate = 0.98;
var DATA = Symbol('boundedData');
/**
* Node in a double-linked list used for the segments within the cache.
*/
var BoundedNode = /** @class */ (function (_super) {
__extends(BoundedNode, _super);
function BoundedNode(key, value) {
var _this = _super.call(this, key, value) || this;
_this.hashCode = key === null ? 0 : CountMinSketch_1.CountMinSketch.hash(key);
_this.weight = 1;
_this.location = 0 /* Location.WINDOW */;
return _this;
}
return BoundedNode;
}(CacheNode_1.CacheNode));
/**
* Bounded cache implementation using W-TinyLFU to keep track of data.
*
* See https://arxiv.org/pdf/1512.00727.pdf for details about TinyLFU and
* the W-TinyLFU optimization.
*/
var BoundedCache = /** @class */ (function (_super) {
__extends(BoundedCache, _super);
function BoundedCache(options) {
var _this = _super.call(this) || this;
var maxMain = Math.floor(percentInMain * options.maxSize);
/*
* For weighted caches use an initial sketch size of 256. It will
* grow when the size of the cache approaches that size.
*
* Otherwise set it to a minimum of 128 or the maximum requested size
* of the graph.
*/
var sketchWidth = options.weigher ? 256 : Math.max(options.maxSize, 128);
_this[symbols_1.MAINTENANCE] = _this[symbols_1.MAINTENANCE].bind(_this);
_this[DATA] = {
maxSize: options.weigher ? -1 : options.maxSize,
removalListener: options.removalListener || null,
weigher: options.weigher || null,
weightedMaxSize: options.maxSize,
weightedSize: 0,
sketch: CountMinSketch_1.CountMinSketch.uint8(sketchWidth, 4),
sketchGrowLimit: sketchWidth,
values: new Map(),
adaptiveData: {
hits: 0,
misses: 0,
adjustment: 0,
previousHitRate: 0,
stepSize: -adaptiveStepPercent * options.maxSize
},
window: {
head: new BoundedNode(null, null),
size: 0,
maxSize: options.maxSize - maxMain
},
protected: {
head: new BoundedNode(null, null),
size: 0,
maxSize: Math.floor(maxMain * percentProtected)
},
probation: {
head: new BoundedNode(null, null),
},
maintenanceTimeout: null,
forceEvictionLimit: options.maxSize + Math.max(Math.floor(options.maxSize * percentOverflow), 5),
maintenanceInterval: 5000
};
return _this;
}
Object.defineProperty(BoundedCache.prototype, "maxSize", {
/**
* Get the maximum size this cache can be.
*
* @returns
* maximum size of the cache
*/
get: function () {
return this[DATA].maxSize;
},
enumerable: false,
configurable: true
});
Object.defineProperty(BoundedCache.prototype, "size", {
/**
* Get the current size of the cache.
*
* @returns
* items currently in the cache
*/
get: function () {
return this[DATA].values.size;
},
enumerable: false,
configurable: true
});
Object.defineProperty(BoundedCache.prototype, "weightedSize", {
/**
* Get the weighted size of all items in the cache.
*
* @returns
* the weighted size of all items in the cache
*/
get: function () {
return this[DATA].weightedSize;
},
enumerable: false,
configurable: true
});
/**
* Store a value tied to the specified key. Returns the previous value or
* `null` if no value currently exists for the given key.
*
* @param key -
* key to store value under
* @param value -
* value to store
* @returns
* current value or `null`
*/
BoundedCache.prototype.set = function (key, value) {
var data = this[DATA];
var old = data.values.get(key);
// Create a node and add it to the backing map
var node = new BoundedNode(key, value);
data.values.set(key, node);
if (data.weigher) {
node.weight = data.weigher(key, value);
}
// Update our weight
data.weightedSize += node.weight;
if (old) {
// Remove the old node
old.remove();
// Adjust weight
data.weightedSize -= old.weight;
// Update weights of where the node belonged
switch (old.location) {
case 1 /* Location.PROTECTED */:
// Node was protected, reduce the size
data.protected.size -= old.weight;
break;
case 0 /* Location.WINDOW */:
// Node was in window, reduce window size
data.window.size -= old.weight;
break;
}
}
// Check if we reached the grow limit of the sketch
if (data.weigher && data.values.size >= data.sketchGrowLimit) {
var sketchWidth = data.values.size * 2;
data.sketch = CountMinSketch_1.CountMinSketch.uint8(sketchWidth, 4);
data.sketchGrowLimit = sketchWidth;
}
// Append the new node to the window space
node.appendToTail(data.window.head);
data.window.size += node.weight;
// Register access to the key
data.sketch.update(node.hashCode);
// Schedule eviction
if (data.weightedSize >= data.forceEvictionLimit) {
this[symbols_1.MAINTENANCE]();
}
else if (!data.maintenanceTimeout) {
data.maintenanceTimeout = setTimeout(this[symbols_1.MAINTENANCE], data.maintenanceInterval);
}
// Return the value we replaced
if (old) {
this[symbols_1.TRIGGER_REMOVE](key, old.value, RemovalReason_1.RemovalReason.REPLACED);
return old.value;
}
else {
return null;
}
};
/**
* Get the cached value for the specified key if it exists. Will return
* the value or `null` if no cached value exist. Updates the usage of the
* key.
*
* @param key -
* key to get
* @returns
* current value or `null`
*/
BoundedCache.prototype.getIfPresent = function (key) {
var data = this[DATA];
var node = data.values.get(key);
if (!node) {
// This value does not exist in the cache
data.adaptiveData.misses++;
return null;
}
// Keep track of the hit
data.adaptiveData.hits++;
// Register access to the key
data.sketch.update(node.hashCode);
switch (node.location) {
case 0 /* Location.WINDOW */:
// In window cache, mark as most recently used
node.moveToTail(data.window.head);
break;
case 2 /* Location.PROBATION */:
// In SLRU probation segment, move to protected
node.location = 1 /* Location.PROTECTED */;
node.moveToTail(data.protected.head);
// Plenty of room, keep track of the size
data.protected.size += node.weight;
while (data.protected.size > data.protected.maxSize) {
/*
* There is now too many nodes in the protected segment
* so demote the least recently used.
*/
var lru = data.protected.head.next;
lru.location = 2 /* Location.PROBATION */;
lru.moveToTail(data.probation.head);
data.protected.size -= lru.weight;
}
break;
case 1 /* Location.PROTECTED */:
// SLRU protected segment, mark as most recently used
node.moveToTail(data.protected.head);
break;
}
return node.value;
};
/**
* Peek to see if a key is present without updating the usage of the
* key. Returns the value associated with the key or `null` if the key
* is not present.
*
* In many cases `has(key)` is a better option to see if a key is present.
*
* @param key -
* the key to check
* @returns
* value associated with key or `null`
*/
BoundedCache.prototype.peek = function (key) {
var data = this[DATA];
var node = data.values.get(key);
return node ? node.value : null;
};
/**
* Delete a value in the cache. Returns the deleted value or `null` if
* there was no value associated with the key in the cache.
*
* @param key -
* the key to delete
* @returns
* deleted value or `null`
*/
BoundedCache.prototype.delete = function (key) {
var data = this[DATA];
var node = data.values.get(key);
if (node) {
// Remove the node from its current list
node.remove();
switch (node.location) {
case 1 /* Location.PROTECTED */:
// Node was protected, reduce the size
data.protected.size -= node.weight;
break;
case 0 /* Location.WINDOW */:
// Node was in window, reduce window size
data.window.size -= node.weight;
break;
}
// Reduce overall weight
data.weightedSize -= node.weight;
// Remove from main value storage
data.values.delete(key);
this[symbols_1.TRIGGER_REMOVE](key, node.value, RemovalReason_1.RemovalReason.EXPLICIT);
if (!data.maintenanceTimeout) {
data.maintenanceTimeout = setTimeout(this[symbols_1.MAINTENANCE], data.maintenanceInterval);
}
return node.value;
}
return null;
};
/**
* Check if the given key exists in the cache.
*
* @param key -
* key to check
* @returns
* `true` if value currently exists, `false` otherwise
*/
BoundedCache.prototype.has = function (key) {
var data = this[DATA];
return data.values.has(key);
};
/**
* Clear the cache removing all of the entries cached.
*/
BoundedCache.prototype.clear = function () {
var e_1, _a;
var data = this[DATA];
var oldValues = data.values;
data.values = new Map();
try {
for (var oldValues_1 = __values(oldValues), oldValues_1_1 = oldValues_1.next(); !oldValues_1_1.done; oldValues_1_1 = oldValues_1.next()) {
var _b = __read(oldValues_1_1.value, 2), key = _b[0], node = _b[1];
this[symbols_1.TRIGGER_REMOVE](key, node.value, RemovalReason_1.RemovalReason.EXPLICIT);
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (oldValues_1_1 && !oldValues_1_1.done && (_a = oldValues_1.return)) _a.call(oldValues_1);
}
finally { if (e_1) throw e_1.error; }
}
data.weightedSize = 0;
data.window.head.remove();
data.window.size = 0;
data.probation.head.remove();
data.protected.head.remove();
data.protected.size = 0;
if (data.maintenanceTimeout) {
clearTimeout(data.maintenanceTimeout);
data.maintenanceTimeout = null;
}
};
/**
* Get all of the keys in the cache as an array. Can be used to iterate
* over all of the values in the cache, but be sure to protect against
* values being removed during iteration due to time-based expiration if
* used.
*
* @returns
* snapshot of keys
*/
BoundedCache.prototype.keys = function () {
this[symbols_1.MAINTENANCE]();
return Array.from(this[DATA].values.keys());
};
/**
* Request clean up of the cache by removing expired entries and
* old data. Clean up is done automatically a short time after sets and
* deletes, but if your cache uses time-based expiration and has very
* sporadic updates it might be a good idea to call `cleanUp()` at times.
*
* A good starting point would be to call `cleanUp()` in a `setInterval`
* with a delay of at least a few minutes.
*/
BoundedCache.prototype.cleanUp = function () {
this[symbols_1.MAINTENANCE]();
};
Object.defineProperty(BoundedCache.prototype, "metrics", {
/**
* Get metrics for this cache. Returns an object with the keys `hits`,
* `misses` and `hitRate`. For caches that do not have metrics enabled
* trying to access metrics will throw an error.
*/
get: function () {
throw new Error('Metrics are not supported by this cache');
},
enumerable: false,
configurable: true
});
BoundedCache.prototype[symbols_1.TRIGGER_REMOVE] = function (key, value, cause) {
var data = this[DATA];
// Trigger any extended remove listeners
var onRemove = this[symbols_1.ON_REMOVE];
if (onRemove) {
onRemove(key, value, cause);
}
// Trigger the removal listener
if (data.removalListener) {
data.removalListener(key, value, cause);
}
};
BoundedCache.prototype[symbols_1.MAINTENANCE] = function () {
/*
* Trigger the onMaintenance listener if one exists. This is done
* before eviction occurs so that extra layers have a chance to
* apply their own eviction rules.
*
* This can be things such as things being removed because they have
* been expired which in turn might cause eviction to be unnecessary.
*/
var onMaintenance = this[symbols_1.ON_MAINTENANCE];
if (onMaintenance) {
onMaintenance();
}
var data = this[DATA];
/*
* Evict the least recently used node in the window space to the
* probation segment until we are below the maximum size.
*/
var evictedToProbation = 0;
while (data.window.size > data.window.maxSize) {
var first = data.window.head.next;
first.moveToTail(data.probation.head);
first.location = 2 /* Location.PROBATION */;
data.window.size -= first.weight;
evictedToProbation++;
}
/*
* Evict nodes for real until we are below our maximum size.
*/
while (data.weightedSize > data.weightedMaxSize) {
var probation = data.probation.head.next;
var evictedCandidate = evictedToProbation === 0 ? data.probation.head : data.probation.head.previous;
var hasProbation = probation !== data.probation.head;
var hasEvicted = evictedCandidate !== data.probation.head;
var toRemove = void 0;
if (!hasProbation && !hasEvicted) {
// TODO: Probation queue is empty, how is this handled?
break;
}
else if (!hasEvicted) {
toRemove = probation;
}
else if (!hasProbation) {
toRemove = evictedCandidate;
evictedToProbation--;
}
else {
/*
* Estimate how often the two nodes have been accessed to
* determine which of the keys should actually be evicted.
*
* Also protect against hash collision attacks where the
* frequency of an node in the cache is raised causing the
* candidate to never be admitted into the cache.
*/
var removeCandidate = void 0;
var freqEvictedCandidate = data.sketch.estimate(evictedCandidate.hashCode);
var freqProbation = data.sketch.estimate(probation.hashCode);
if (freqEvictedCandidate > freqProbation) {
removeCandidate = false;
}
else if (freqEvictedCandidate < data.sketch.slightlyLessThanHalfMaxSize) {
/*
* If the frequency of the candidate is slightly less than
* half it can be admitted without going through randomness
* checks.
*
* The idea here is that will reduce the number of random
* admittances.
*/
removeCandidate = true;
}
else {
/*
* Make it a 1 in 1000 chance that the candidate is not
* removed.
*
* TODO: Should this be lower or higher? Please open an issue if you have thoughts on this
*/
removeCandidate = Math.floor(Math.random() * 1000) >= 1;
}
toRemove = removeCandidate ? evictedCandidate : probation;
evictedToProbation--;
}
if (toRemove.key === null) {
throw new Error('Cache issue, problem with removal');
}
data.values.delete(toRemove.key);
toRemove.remove();
data.weightedSize -= toRemove.weight;
this[symbols_1.TRIGGER_REMOVE](toRemove.key, toRemove.value, RemovalReason_1.RemovalReason.SIZE);
}
// Perform adaptive adjustment of size of window cache
adaptiveAdjustment(data);
if (data.maintenanceTimeout) {
clearTimeout(data.maintenanceTimeout);
data.maintenanceTimeout = null;
}
};
return BoundedCache;
}(AbstractCache_1.AbstractCache));
exports.BoundedCache = BoundedCache;
/**
* Perform adaptive adjustment. This will do a simple hill climb and attempt
* to find the best balance between the recency and frequency parts of the
* cache.
*
* This is based on the work done in Caffeine and the paper Adaptive Software
* Cache Management by Gil Einziger, Ohad Eytan, Roy Friedman and Ben Manes.
*
* This implementation does work in chunks so that not too many nodes are
* moved around at once. At every maintenance interval it:
*
* 1) Checks if there are enough samples to calculate a new adjustment.
* 2)
* Takes the current adjustment and increases or decreases the window in
* chunks. At every invocation it currently moves a maximum of 1000 nodes
* around.
*
* @param data -
*/
function adaptiveAdjustment(data) {
/*
* Calculate the new adaptive adjustment. This might result in a
* recalculation or it may skip touching the adjustment.
*/
calculateAdaptiveAdjustment(data);
var a = data.adaptiveData.adjustment;
if (a > 0) {
// Increase the window size if the adjustment is positive
increaseWindowSegmentSize(data);
}
else if (a < 0) {
// Decrease the window size if the adjustment is negative
decreaseWindowSegmentSize(data);
}
}
/**
* Evict nodes from the protected segment to the probation segment if there
* are too many nodes in the protected segment.
*
* @param data -
*/
function evictProtectedToProbation(data) {
/*
* Move up to 1000 nodes from the protected segment to the probation one
* if the segment is over max size.
*/
var i = 0;
while (i++ < 1000 && data.protected.size > data.protected.maxSize) {
var lru = data.protected.head.next;
if (lru === data.protected.head)
break;
lru.location = 2 /* Location.PROBATION */;
lru.moveToTail(data.probation.head);
data.protected.size -= lru.weight;
}
}
/**
* Calculate the adjustment to the window size. This will check if there is
* enough samples to do a step and if so perform a simple hill climbing to
* find the new adjustment.
*
* @param data -
* @returns
* `true` if an adjustment occurred, `false` otherwise
*/
function calculateAdaptiveAdjustment(data) {
var adaptiveData = data.adaptiveData;
var requestCount = adaptiveData.hits + adaptiveData.misses;
if (requestCount < data.sketch.resetAfter) {
/*
* Skip adjustment if the number of gets in the cache has not reached
* the same size as the sketch reset.
*/
return false;
}
var hitRate = adaptiveData.hits / requestCount;
var hitRateDiff = hitRate - adaptiveData.previousHitRate;
var amount = hitRateDiff >= 0 ? adaptiveData.stepSize : -adaptiveData.stepSize;
var nextStep;
if (Math.abs(hitRateDiff) >= adaptiveRestartThreshold) {
nextStep = adaptiveStepPercent * data.weightedMaxSize * (amount >= 0 ? 1 : -1);
}
else {
nextStep = adaptiveStepDecayRate * amount;
}
// Store the adjustment, step size and previous hit rate for the next step
adaptiveData.adjustment = Math.floor(amount);
adaptiveData.stepSize = nextStep;
adaptiveData.previousHitRate = hitRate;
// Reset the sample data
adaptiveData.misses = 0;
adaptiveData.hits = 0;
return true;
}
/**
* Increase the size of the window segment. This will change increase the max
* size of the window segment and decrease the max size of the protected
* segment. The method will then move nodes from the probation and protected
* segment the window segment.
*
* @param data -
*/
function increaseWindowSegmentSize(data) {
if (data.protected.maxSize === 0) {
// Can't increase the window size anymore
return;
}
var amountLeftToAdjust = Math.min(data.adaptiveData.adjustment, data.protected.maxSize);
data.protected.maxSize -= amountLeftToAdjust;
data.window.maxSize += amountLeftToAdjust;
/*
* Evict nodes from the protected are to the probation area now that it
* is smaller.
*/
evictProtectedToProbation(data);
/*
* Transfer up to 1000 node into the window segment.
*/
for (var i = 0; i < 1000; i++) {
var lru = data.probation.head.next;
if (lru === data.probation.head || lru.weight > amountLeftToAdjust) {
/*
* Either got the probation head or the node was to big to fit.
* Move on and check in the protected area.
*/
lru = data.protected.head.next;
if (lru === data.protected.head) {
// No more values to remove
break;
}
}
if (lru.weight > amountLeftToAdjust) {
/*
* The node weight exceeds what is left of the adjustment.
*/
break;
}
amountLeftToAdjust -= lru.weight;
// Remove node from its current segment
if (lru.location === 1 /* Location.PROTECTED */) {
// If its protected reduce the size
data.protected.size -= lru.weight;
}
// Move to the window segment
lru.moveToTail(data.window.head);
data.window.size += lru.weight;
lru.location = 0 /* Location.WINDOW */;
}
/*
* Keep track of the adjustment amount that is left. The next maintenance
* invocation will look at this and attempt to adjust for it.
*/
data.protected.maxSize += amountLeftToAdjust;
data.window.maxSize -= amountLeftToAdjust;
data.adaptiveData.adjustment = amountLeftToAdjust;
}
/**
* Decrease the size of the window. This will increase the size of the
* protected segment while decreasing the size of the window segment. Nodes
* will be moved from the window segment into the probation segment, where
* they are later moved to the protected segment when they are accessed.
*
* @param data -
*/
function decreaseWindowSegmentSize(data) {
if (data.window.maxSize <= 1) {
// Can't decrease the size of the window anymore
return;
}
var amountLeftToAdjust = Math.min(-data.adaptiveData.adjustment, Math.max(data.window.maxSize - 1, 0));
data.window.maxSize -= amountLeftToAdjust;
data.protected.maxSize += amountLeftToAdjust;
/*
* Transfer upp to 1000 nodes from the window segment into the probation
* segment.
*/
for (var i = 0; i < 1000; i++) {
var lru = data.window.head.next;
if (lru === data.window.head) {
// No more nodes in the window segment, can't adjust anymore
break;
}
if (lru.weight > amountLeftToAdjust) {
/*
* The node weight exceeds what is left of the change. Can't move
* it around.
*/
break;
}
amountLeftToAdjust -= lru.weight;
// Remove node from the window
lru.moveToTail(data.probation.head);
lru.location = 2 /* Location.PROBATION */;
data.window.size -= lru.weight;
}
/*
* Keep track of the adjustment amount that is left. The next maintenance
* invocation will look at this and attempt to adjust for it.
*/
data.window.maxSize += amountLeftToAdjust;
data.protected.maxSize -= amountLeftToAdjust;
data.adaptiveData.adjustment = -amountLeftToAdjust;
}
//# sourceMappingURL=BoundedCache.js.map