keyv-file
Version:
File storage adapter for Keyv, using msgpack to serialize data fast and small.
259 lines (258 loc) • 8.03 kB
JavaScript
'use strict';
Object.defineProperty(exports, "__esModule", { value: true });
exports.KeyvFile = exports.defaultOpts = void 0;
const tslib_1 = require("tslib");
const os = tslib_1.__importStar(require("os"));
const fs = tslib_1.__importStar(require("fs"));
const fsp = tslib_1.__importStar(require("fs/promises"));
const events_1 = tslib_1.__importDefault(require("events"));
const serialize_1 = require("@keyv/serialize");
const path_1 = tslib_1.__importDefault(require("path"));
const separated_file_store_1 = require("./separated-file-store");
tslib_1.__exportStar(require("./make-field"), exports);
exports.defaultOpts = {
deserialize: (val) => (0, serialize_1.defaultDeserialize)(val.toString()),
dialect: 'redis',
expiredCheckDelay: 24 * 3600 * 1000, // ms
filename: `${os.tmpdir()}/keyv-file/default.json`,
serialize: serialize_1.defaultSerialize,
writeDelay: 100, // ms
checkFileLock: false,
separatedFile: false,
};
function isNumber(val) {
return typeof val === 'number';
}
class KeyvFile extends events_1.default {
ttlSupport = true;
namespace;
opts;
_data = new Map();
_lastExpire = 0;
_separated;
constructor(options) {
super();
this.opts = Object.assign({}, exports.defaultOpts, options);
this._separated = new separated_file_store_1.SeparatedFileHelper(this.opts);
if (this.opts.checkFileLock) {
this.acquireFileLock();
}
if (this.opts.separatedFile) {
fs.mkdirSync(this.opts.filename, { recursive: true });
this._lastExpire = this._separated.getLastExpire();
}
else {
this._loadDataSync();
}
}
_loadDataSync() {
try {
const data = this.opts.deserialize(fs.readFileSync(this.opts.filename));
if (!Array.isArray(data.cache)) {
const _cache = data.cache;
data.cache = [];
for (const key in _cache) {
if (_cache.hasOwnProperty(key)) {
data.cache.push([key, _cache[key]]);
}
}
}
this._data = new Map(data.cache);
this._lastExpire = data.lastExpire;
}
catch (e) {
(0, separated_file_store_1.handleIOError)(e);
this._data = new Map();
this._lastExpire = Date.now();
}
}
get _lockFile() {
if (this.opts.separatedFile) {
return this._separated.lockFile;
}
return this.opts.filename + '.lock';
}
acquireFileLock() {
try {
let fd = fs.openSync(this._lockFile, 'wx');
fs.closeSync(fd);
process.on('SIGINT', () => {
this.releaseFileLock();
process.exit(0);
});
process.on('exit', () => {
this.releaseFileLock();
});
}
catch (error) {
console.error(`[keyv-file] There is another process using this file`);
throw error;
}
}
releaseFileLock() {
try {
fs.unlinkSync(this._lockFile);
}
catch (e) {
//pass
(0, separated_file_store_1.handleIOError)(e);
}
}
async get(key) {
if (this.opts.separatedFile) {
let data = await this._separated.get(key);
return this._getWithExpire(key, data);
}
return this.getSync(key);
}
getSync(key) {
if (this.opts.separatedFile) {
let data = this._separated.getSync(key);
return this._getWithExpire(key, data);
}
let ret = void 0;
try {
const data = this._data.get(key);
return this._getWithExpire(key, data);
}
catch (error) {
(0, separated_file_store_1.handleIOError)(error);
}
return ret;
}
async getMany(keys) {
if (this.opts.separatedFile) {
return Promise.all(keys.map((key) => this.get(key)));
}
return keys.map((key) => this.getSync(key));
}
/**
* Note: `await kv.set()` will wait <options.writeDelay> millseconds to save to disk, it would be slow. Please remove `await` if you find performance issues.
* @param key
* @param value
* @param ttl
* @returns
*/
async set(key, value, ttl) {
if (ttl === 0) {
ttl = undefined;
}
value = {
expire: isNumber(ttl) ? Date.now() + ttl : undefined,
value: value,
};
this.clearExpire();
if (this.opts.separatedFile) {
return this._separated.set(key, value);
}
this._data.set(key, value);
return this.save();
}
async delete(key) {
if (this.opts.separatedFile) {
return this._separated.delete(key);
}
const ret = this._data.delete(key);
await this.save();
return ret;
}
async deleteMany(keys) {
if (this.opts.separatedFile) {
let ret = await Promise.all(keys.map((key) => this.delete(key)));
return ret.every((r) => r);
}
let res = keys.every((key) => this._data.delete(key));
await this.save();
return res;
}
async clear() {
if (this.opts.separatedFile) {
await this._separated.clear();
this._lastExpire = 0;
return true;
}
this._data = new Map();
this._lastExpire = Date.now();
return this.save();
}
async has(key) {
const value = await this.get(key);
return value !== undefined;
}
isExpired(data) {
return isNumber(data.expire) && data.expire <= Date.now();
}
_getWithExpire(key, data) {
if (!data) {
return;
}
if (this.isExpired(data)) {
this.delete(key);
return;
}
return data.value;
}
clearExpire() {
const now = Date.now();
if (now - this._lastExpire <= this.opts.expiredCheckDelay) {
return;
}
this._lastExpire = now;
if (this.opts.separatedFile) {
this._separated.clearExpire((key) => this.get(key));
return;
}
for (const key of this._data.keys()) {
const data = this._data.get(key);
this._getWithExpire(key, data);
}
}
async saveToDisk() {
const cache = [];
for (const [key, val] of this._data) {
cache.push([key, val]);
}
const data = this.opts.serialize({
cache,
lastExpire: this._lastExpire,
});
await fsp.mkdir(path_1.default.dirname(this.opts.filename), {
recursive: true,
});
return fsp.writeFile(this.opts.filename, data);
}
_savePromise;
save() {
this.clearExpire();
if (this._savePromise) {
return this._savePromise;
}
this._savePromise = new Promise((resolve, reject) => {
setTimeout(() => {
this.saveToDisk()
.then(resolve, reject)
.finally(() => {
this._savePromise = void 0;
});
}, this.opts.writeDelay);
});
return this._savePromise;
}
disconnect() {
return Promise.resolve();
}
async *iterator(namespace) {
let entries = this.opts.separatedFile ? await this._separated.entries() : this._data.entries();
for (const [key, data] of entries) {
if (key === undefined || data === undefined) {
continue;
}
// Filter by namespace if provided
if (!namespace || key.includes(namespace)) {
yield [key, data.value];
}
}
}
}
exports.KeyvFile = KeyvFile;
exports.default = KeyvFile;