cache-manager-fs-binary-ts
Version:
File system store for node-cache-manager with binary data as files
560 lines • 29 kB
JavaScript
"use strict";
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __read = (this && this.__read) || function (o, n) {
var m = typeof Symbol === "function" && o[Symbol.iterator];
if (!m) return o;
var i = m.call(o), r, ar = [], e;
try {
while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
}
catch (error) { e = { error: error }; }
finally {
try {
if (r && !r.done && (m = i["return"])) m.call(i);
}
finally { if (e) throw e.error; }
}
return ar;
};
var __values = (this && this.__values) || function(o) {
var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
if (m) return m.call(o);
if (o && typeof o.length === "number") return {
next: function () {
if (o && i >= o.length) o = void 0;
return { value: o && o[i++], done: !o };
}
};
throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
exports.__esModule = true;
exports.fsBinaryStore = void 0;
var fast_glob_1 = __importDefault(require("fast-glob"));
var crypto_1 = require("crypto");
var fs_1 = require("fs");
var promises_1 = require("fs/promises");
var path_1 = require("path");
var util_1 = require("util");
var zlib_1 = require("zlib");
var slugify = function (str) {
return str
.toLowerCase()
.trim()
.replace(/[^\w\s-]/g, '')
.replace(/[\s_-]+/g, '-')
.replace(/^-+|-+$/g, '');
};
function fsBinaryStore(args) {
var _a, _b;
var options = __assign(__assign({}, args), { ttl: (_a = args === null || args === void 0 ? void 0 : args.ttl) !== null && _a !== void 0 ? _a : 60, path: (_b = args === null || args === void 0 ? void 0 : args.path) !== null && _b !== void 0 ? _b : 'cache', isCacheable: (args === null || args === void 0 ? void 0 : args.isCacheable) || (function (value) { return value !== undefined && value !== null && (typeof value == 'object' && value.binary && (typeof value.binary == 'string' || Object.keys(value.binary).length > 0)); }) });
// check storage directory for existence (or create it)
if (!(0, fs_1.existsSync)(options.path)) {
(0, fs_1.mkdirSync)(options.path);
}
var store = {
collection: {},
currentsize: 0,
options: options,
get: function (key) {
return __awaiter(this, void 0, void 0, function () {
var data;
return __generator(this, function (_a) {
data = this.collection[key];
if (!data) {
// not found
return [2 /*return*/, undefined];
}
if (data.expires < Date.now()) {
// delete the element from the store
return [2 /*return*/, this.del(key).then(function () { return undefined; })];
}
return [2 /*return*/, (0, promises_1.readFile)(data.filename)
.then(this.unzipIfNeeded)
.then(function (buffer) { return JSON.parse(buffer.toString()).value; })];
});
});
},
mget: function () {
var args = [];
for (var _i = 0; _i < arguments.length; _i++) {
args[_i] = arguments[_i];
}
return __awaiter(this, void 0, void 0, function () {
var _this = this;
return __generator(this, function (_a) {
return [2 /*return*/, args.map(function (x) { return _this.get(x); })];
});
});
},
mset: function (args, ttl) {
return __awaiter(this, void 0, void 0, function () {
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, Promise.all(args.map(function (_a) {
var _b = __read(_a, 2), key = _b[0], value = _b[1];
if (!options.isCacheable(value)) {
throw new Error("no cacheable value ".concat(JSON.stringify(value)));
}
return _this.set(key, value, ttl);
}))];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
},
mdel: function () {
var args = [];
for (var _i = 0; _i < arguments.length; _i++) {
args[_i] = arguments[_i];
}
return __awaiter(this, void 0, void 0, function () {
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, Promise.all(args.map(function (x) { return _this.del(x); }))];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
},
set: function (key, data, ttl) {
return __awaiter(this, void 0, void 0, function () {
var fileName, filePath, metaData, binarySize, binary, binkey, path, stream;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!options.isCacheable(data)) {
throw new Error("no cacheable value ".concat(JSON.stringify(data)));
}
ttl = (ttl || ttl === 0) ? ttl : options.ttl;
fileName = "cache_".concat((0, crypto_1.randomUUID)(), ".dat");
filePath = (0, path_1.resolve)((0, path_1.join)(options.path, fileName));
metaData = {
key: key,
value: data,
expires: Date.now() + (ttl * 1000),
filename: filePath,
size: 0
};
binarySize = 0;
if (typeof data.binary == 'string' || Buffer.isBuffer(data.binary)) {
binary = Buffer.from(data.binary);
binarySize += binary.byteLength;
metaData.value.binary = filePath.replace(/\.dat$/, '.bin');
data.binary = metaData.value.binary;
}
else {
binary = data.binary;
delete data.binary;
data.binary = {};
for (binkey in binary) {
path = metaData.filename.replace(/\.dat$/, '_' + slugify(binkey) + '.bin');
metaData.value.binary[binkey] = path;
data.binary[binkey] = path;
// calculate the size of the binary data
binarySize += Buffer.from(binary[binkey]).byteLength;
}
}
stream = JSON.stringify(metaData);
metaData.size = stream.length + binarySize;
if (options.maxsize && metaData.size > options.maxsize) {
throw new Error('Item size too big.');
}
// remove the key from the cache (if it already existed, this updates also the current size of the store)
return [4 /*yield*/, this.del(key)
.then(function () { return _this.freeupspace(); })
.then(function () { return Promise.all(binary
? !Buffer.isBuffer(binary)
? Object.entries(binary).map(function (_a) {
var _b = __read(_a, 2), k = _b[0], v = _b[1];
return (0, promises_1.writeFile)(metaData.value.binary[k], v);
})
: [(0, promises_1.writeFile)(metaData.value.binary, binary)]
: []); })
.then(function () { return _this.zipIfNeeded(Buffer.from(stream)); })
.then(function (processedStream) { return (0, promises_1.writeFile)(metaData.filename, processedStream); })
.then(function () {
// remove data value from memory
var metaDataWithOptionalValue = metaData;
metaDataWithOptionalValue.value = undefined;
delete metaDataWithOptionalValue.value;
_this.currentsize += metaData.size;
// place element with meta info in internal collection
_this.collection[metaData.key] = metaData;
})];
case 1:
// remove the key from the cache (if it already existed, this updates also the current size of the store)
_a.sent();
return [2 /*return*/];
}
});
});
},
del: function (key) {
return __awaiter(this, void 0, void 0, function () {
var metaData;
var _this = this;
return __generator(this, function (_a) {
metaData = this.collection[key];
if (!metaData) {
return [2 /*return*/];
}
// check if the filename is set
if (!metaData.filename) {
return [2 /*return*/];
}
// check for existence of the file
return [2 /*return*/, (0, promises_1.readFile)(metaData.filename)
.then(this.unzipIfNeeded)
.then(function (metaExtraContent) { return __awaiter(_this, void 0, void 0, function () {
var metaData, _a, _b, e_1_1, _c;
var e_1, _d;
return __generator(this, function (_e) {
switch (_e.label) {
case 0:
try {
metaData = JSON.parse(metaExtraContent.toString());
}
catch (e) {
throw new Error('Parsing meta error');
}
_e.label = 1;
case 1:
_e.trys.push([1, 14, , 15]);
if (!!(typeof metaData.value.binary == 'string')) return [3 /*break*/, 10];
_e.label = 2;
case 2:
_e.trys.push([2, 7, 8, 9]);
_a = __values(Object.keys(metaData.value.binary)), _b = _a.next();
_e.label = 3;
case 3:
if (!!_b.done) return [3 /*break*/, 6];
key = _b.value;
return [4 /*yield*/, (0, promises_1.unlink)(metaData.value.binary[key])];
case 4:
_e.sent();
_e.label = 5;
case 5:
_b = _a.next();
return [3 /*break*/, 3];
case 6: return [3 /*break*/, 9];
case 7:
e_1_1 = _e.sent();
e_1 = { error: e_1_1 };
return [3 /*break*/, 9];
case 8:
try {
if (_b && !_b.done && (_d = _a["return"])) _d.call(_a);
}
finally { if (e_1) throw e_1.error; }
return [7 /*endfinally*/];
case 9: return [3 /*break*/, 12];
case 10: return [4 /*yield*/, (0, promises_1.unlink)(metaData.value.binary)];
case 11:
_e.sent();
_e.label = 12;
case 12: return [4 /*yield*/, (0, promises_1.unlink)(metaData.filename)];
case 13:
_e.sent();
return [3 /*break*/, 15];
case 14:
_c = _e.sent();
return [3 /*break*/, 15];
case 15: return [2 /*return*/];
}
});
}); })["catch"]()["finally"](function () {
_this.currentsize -= metaData.size;
_this.collection[key] = null;
delete _this.collection[key];
})];
});
});
},
ttl: function (key) {
var _a;
return __awaiter(this, void 0, void 0, function () {
var now, expires;
return __generator(this, function (_b) {
now = Date.now();
expires = (_a = this.collection[key]) === null || _a === void 0 ? void 0 : _a.expires;
return [2 /*return*/, expires
? expires < now
? 0
: expires - now
: options.ttl];
});
});
},
keys: function () {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
return [2 /*return*/, Object.keys(this.collection)];
});
});
},
reset: function () {
return __awaiter(this, void 0, void 0, function () {
var keys;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.keys()];
case 1:
keys = _a.sent();
if (keys.length == 0) {
return [2 /*return*/];
}
return [4 /*yield*/, Promise.all(keys.map(function (key) { return _this.del(key); }))];
case 2:
_a.sent();
return [2 /*return*/];
}
});
});
},
isCacheableValue: options.isCacheable,
name: 'fsBinary',
cleanExpired: function () {
return __awaiter(this, void 0, void 0, function () {
var _a, _b, _i, key, entry;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
_a = [];
for (_b in this.collection)
_a.push(_b);
_i = 0;
_c.label = 1;
case 1:
if (!(_i < _a.length)) return [3 /*break*/, 4];
key = _a[_i];
entry = this.collection[key];
if (!(entry.expires <= Date.now())) return [3 /*break*/, 3];
return [4 /*yield*/, this.del(entry.key)];
case 2:
_c.sent();
_c.label = 3;
case 3:
_i++;
return [3 /*break*/, 1];
case 4: return [2 /*return*/];
}
});
});
},
cleancache: function () {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
// clean all current used files
return [4 /*yield*/, this.reset()];
case 1:
// clean all current used files
_a.sent();
// check, if other files still resist in the cache and clean them, too
return [4 /*yield*/, (0, promises_1.readdir)(options.path)
.then(function (files) { return Promise.all(files
.map(function (file) { return (0, path_1.join)(options.path, file); })
.filter(function (filePath) { return (0, fs_1.statSync)(filePath).isFile(); })
.map(function (filePath) { return (0, promises_1.unlink)(filePath)["catch"](); })); })];
case 2:
// check, if other files still resist in the cache and clean them, too
_a.sent();
return [2 /*return*/];
}
});
});
},
zipIfNeeded: function (data) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
if (options.zip) {
return [2 /*return*/, (0, util_1.promisify)(zlib_1.deflate)(data)];
}
return [2 /*return*/, data];
});
});
},
unzipIfNeeded: function (data) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!options.zip) return [3 /*break*/, 2];
return [4 /*yield*/, (0, util_1.promisify)(zlib_1.unzip)(data)];
case 1: return [2 /*return*/, _a.sent()];
case 2: return [2 /*return*/, data];
}
});
});
},
intializefill: function () {
return __awaiter(this, void 0, void 0, function () {
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, (0, promises_1.readdir)(options.path)
.then(function (files) { return Promise.all(files
.map(function (file) { return (0, path_1.resolve)((0, path_1.join)(options.path, file)); })
.filter(function (filePath) { return (0, fs_1.statSync)(filePath).isFile(); })
.filter(function (filePath) { return /\.dat$/.test(filePath); })
.map(function (filePath) {
return (0, promises_1.readFile)(filePath)
.then(_this.unzipIfNeeded)
.then(function (data) {
// get the json out of the data
var diskData;
try {
diskData = JSON.parse(data.toString());
}
catch (_a) {
return (0, promises_1.unlink)(filePath)
.then(function () { return (0, fast_glob_1["default"])(filePath.replace(/\.dat$/, '*.bin'), { onlyFiles: true, absolute: true }); })
.then(function (files) { return Promise.all(files.map(function (f) { return (0, promises_1.unlink)(f)["catch"](); })); })
.then(function () {
return;
});
}
// update the size in the metadata - this value isn't correctly stored in the file
// diskData.size = data.length;
// update collection size
_this.currentsize += diskData.size;
// and put the entry in the store
_this.collection[diskData.key] = diskData;
// check for expiry - in this case we instantly delete the entry
if (diskData.expires < Date.now()) {
return store.del(diskData.key);
}
});
})); })];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
},
freeupspacehelper: function (tuples) {
return __awaiter(this, void 0, void 0, function () {
var tuple, key;
var _this = this;
return __generator(this, function (_a) {
// check, if we have any entry to process
if (tuples.length === 0) {
return [2 /*return*/];
}
tuple = tuples.pop();
if (tuple) {
key = tuple[0];
// delete an entry from the store
return [2 /*return*/, store.del(key).then(function () {
// stop processing when enouth space has been cleaned up
if (options.maxsize == null || (_this.currentsize <= options.maxsize)) {
return;
}
// ok - we need to free up more space
return _this.freeupspacehelper(tuples);
})];
}
return [2 /*return*/];
});
});
},
freeupspace: function () {
return __awaiter(this, void 0, void 0, function () {
var tuples, key;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!options.maxsize) {
return [2 /*return*/];
}
if (!(this.currentsize > options.maxsize)) return [3 /*break*/, 2];
return [4 /*yield*/, store.cleanExpired()];
case 1:
_a.sent();
_a.label = 2;
case 2:
// when the space usage is too high, remove the oldest entries until we gain enough disks pace
if (this.currentsize <= options.maxsize) {
return [2 /*return*/];
}
tuples = [];
for (key in this.collection) {
tuples.push([key, this.collection[key].expires]);
}
tuples.sort(function (a, b) {
var a1 = a[1];
var b1 = b[1];
return a1 < b1 ? 1 : (a1 > b1 ? -1 : 0);
});
return [2 /*return*/, this.freeupspacehelper(tuples)];
}
});
});
}
};
if (!options.preventfill) {
store.intializefill().then(options.fillcallback);
}
return store;
}
exports.fsBinaryStore = fsBinaryStore;
//# sourceMappingURL=index.js.map