@iobroker/db-objects-jsonl
Version:
The Library contains the JSONL Database classes for File based objects database client and server.
254 lines (253 loc) • 9.03 kB
JavaScript
;
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var objectsInMemJsonlDB_exports = {};
__export(objectsInMemJsonlDB_exports, {
ObjectsInMemoryJsonlDB: () => ObjectsInMemoryJsonlDB
});
module.exports = __toCommonJS(objectsInMemJsonlDB_exports);
var import_db_objects_file = require("@iobroker/db-objects-file");
var import_jsonl_db = require("@alcalzone/jsonl-db");
var import_node_path = __toESM(require("node:path"), 1);
var import_node_fs = __toESM(require("node:fs"), 1);
var import_node_os = __toESM(require("node:os"), 1);
var import_js_controller_common_db = require("@iobroker/js-controller-common-db");
function normalizeJsonlOptions(conf = {}) {
const ret = {
autoCompress: {
// Compress when the number of uncompressed entries has grown a lot
sizeFactor: 2,
sizeFactorMinimumSize: 25e3,
// Compress at least daily to avoid a huge file when DBs have few objects
// but big objects are updated regularly (e.g. the repositories)
intervalMs: 1e3 * 60 * 60 * 23
},
ignoreReadErrors: true,
throttleFS: {
intervalMs: 6e4,
maxBufferedCommands: 1e3
},
lockfile: {
// 5 retries starting at 250ms add up to just above 2s,
// so the DB has 3 more seconds to load all data if it wants to stay within the 5s connectionTimeout
retries: 5,
retryMinTimeoutMs: 250,
// This makes sure the DB stays locked for maximum 2s even if the process crashes
staleMs: 2e3
}
};
if (import_js_controller_common_db.tools.isObject(conf.autoCompress)) {
const ac = conf.autoCompress;
if (typeof ac.sizeFactor === "number" && ac.sizeFactor >= 2 && ac.sizeFactor <= 100) {
ret.autoCompress.sizeFactor = ac.sizeFactor;
}
if (typeof ac.sizeFactorMinimumSize === "number" && ac.sizeFactorMinimumSize >= 0 && ac.sizeFactorMinimumSize <= 1e5) {
ret.autoCompress.sizeFactorMinimumSize = ac.sizeFactorMinimumSize;
}
}
if (import_js_controller_common_db.tools.isObject(conf.throttleFS)) {
const thr = conf.throttleFS;
if (typeof thr.intervalMs === "number" && thr.intervalMs >= 1e3 && thr.intervalMs <= 36e5) {
ret.throttleFS.intervalMs = thr.intervalMs;
}
if (typeof thr.maxBufferedCommands === "number" && thr.maxBufferedCommands >= 0 && thr.maxBufferedCommands <= 1e5) {
ret.throttleFS.maxBufferedCommands = thr.maxBufferedCommands;
}
}
return ret;
}
class ObjectsInMemoryJsonlDB extends import_db_objects_file.ObjectsInMemoryFileDB {
constructor(settings) {
settings = settings || {};
settings.fileDB = {
fileName: "objects.json",
backupDirName: "backup-objects"
};
const jsonlOptions = normalizeJsonlOptions(settings.connection.jsonlOptions);
settings.jsonlDB = {
fileName: "objects.jsonl"
};
super(settings);
this._db = new import_jsonl_db.JsonlDB(import_node_path.default.join(this.dataDir, settings.jsonlDB.fileName), jsonlOptions);
}
async open() {
if (!await this._maybeMigrateFileDB()) {
await this._db.open();
}
this.dataset = new Proxy(this._db, {
/**
* @param target
* @param prop
*/
get(target, prop) {
return target.get(prop);
},
/**
* @param target
* @param prop
*/
has(target, prop) {
return target.has(prop);
},
/**
* @param target
* @param prop
* @param value
*/
set(target, prop, value) {
target.set(prop, value);
return true;
},
/**
* @param target
* @param prop
*/
deleteProperty(target, prop) {
return target.delete(prop);
},
ownKeys(target) {
return [...target.keys()];
},
/**
* @param target
* @param prop
*/
getOwnPropertyDescriptor(target, prop) {
if (!target.has(prop)) {
return void 0;
}
return {
configurable: true,
enumerable: true,
writable: true,
value: target.get(prop)
};
}
});
if (this.settings.backup && this.settings.backup.period && !this.settings.backup.disabled) {
this._backupInterval = setInterval(() => {
this.saveBackup();
}, this.settings.backup.period);
}
}
/**
* Checks if an existing file DB should be migrated to JSONL
*
* @returns true if the file DB was migrated. false if not.
* If this returns true, the jsonl DB was opened and doesn't need to be opened again.
*/
async _maybeMigrateFileDB() {
const jsonlFileName = import_node_path.default.join(this.dataDir, this.settings.jsonlDB.fileName);
const jsonFileName = import_node_path.default.join(this.dataDir, this.settings.fileDB.fileName);
const bakFileName = import_node_path.default.join(this.dataDir, `${this.settings.fileDB.fileName}.bak`);
let jsonlTimeStamp = 0;
let jsonTimeStamp = 0;
let bakTimeStamp = 0;
try {
const stat = import_node_fs.default.statSync(jsonlFileName);
if (stat.isFile()) {
jsonlTimeStamp = stat.mtime;
}
} catch {
}
try {
const stat = import_node_fs.default.statSync(jsonFileName);
if (stat.isFile()) {
jsonTimeStamp = stat.mtime;
}
} catch {
}
try {
const stat = import_node_fs.default.statSync(bakFileName);
if (stat.isFile()) {
bakTimeStamp = stat.mtime;
}
} catch {
}
let importFilename;
if (jsonTimeStamp > 0 && jsonTimeStamp >= bakTimeStamp && jsonTimeStamp >= jsonlTimeStamp) {
importFilename = jsonFileName;
} else if (bakTimeStamp > 0 && bakTimeStamp >= jsonTimeStamp && bakTimeStamp >= jsonlTimeStamp) {
importFilename = bakFileName;
} else {
return false;
}
await this._db.open();
this._db.clear();
await this._db.importJson(importFilename);
if (import_node_fs.default.existsSync(jsonFileName)) {
try {
import_node_fs.default.renameSync(jsonFileName, `${jsonFileName}.migrated`);
} catch {
}
}
if (import_node_fs.default.existsSync(bakFileName)) {
try {
import_node_fs.default.renameSync(bakFileName, `${bakFileName}.migrated`);
} catch {
}
}
return true;
}
async saveState() {
}
// Is regularly called and stores a compressed backup of the DB
async saveBackup() {
const now = Date.now();
const tmpBackupFileName = import_node_path.default.join(import_node_os.default.tmpdir(), `${this.getTimeStr(now)}_${this.settings.jsonlDB.fileName}`);
const backupFileName = import_node_path.default.join(this.backupDir, `${this.getTimeStr(now)}_${this.settings.jsonlDB.fileName}.gz`);
if (!this._db.isOpen) {
this.log.warn(`${this.namespace} Cannot save backup ${backupFileName}: DB is closed`);
return;
}
try {
if (import_node_fs.default.existsSync(backupFileName)) {
return;
}
await this._db.dump(tmpBackupFileName);
await import_js_controller_common_db.tools.compressFileGZip(tmpBackupFileName, backupFileName, { deleteInput: true });
this.deleteOldBackupFiles(this.settings.jsonlDB.fileName);
} catch (e) {
this.log.error(`${this.namespace} Cannot save backup ${backupFileName}: ${e.message}`);
}
}
async destroy() {
await super.destroy();
if (this._backupInterval) {
clearInterval(this._backupInterval);
}
if (this._db) {
await this._db.close();
}
}
}
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
ObjectsInMemoryJsonlDB
});
//# sourceMappingURL=objectsInMemJsonlDB.js.map