@trap_stevo/star-vault/dist/cjs/HUDControllers/StorageEngine.js

Unleash the future of data management with the ultimate platform for secure, scalable, and dynamic data operations. Power the next generation of applications by combining advanced encryption, revolutionary real-time querying, and seamless synchronization

"use strict";

function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
function _slicedToArray(r, e) { return _arrayWithHoles(r) || _iterableToArrayLimit(r, e) || _unsupportedIterableToArray(r, e) || _nonIterableRest(); }
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _iterableToArrayLimit(r, l) { var t = null == r ? null : "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (null != t) { var e, n, i, u, a = [], f = !0, o = !1; try { if (i = (t = t.call(r)).next, 0 === l) { if (Object(t) !== t) return; f = !1; } else for (; !(f = (e = i.call(t)).done) && (a.push(e.value), a.length !== l); f = !0); } catch (r) { o = !0, n = r; } finally { try { if (!f && null != t["return"] && (u = t["return"](), Object(u) !== u)) return; } finally { if (o) throw n; } } return a; } }
function _arrayWithHoles(r) { if (Array.isArray(r)) return r; }
function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
function _classCallCheck(a, n) { if (!(a instanceof n)) throw new TypeError("Cannot call a class as a function"); }
function _defineProperties(e, r) { for (var t = 0; t < r.length; t++) { var o = r[t]; o.enumerable = o.enumerable || !1, o.configurable = !0, "value" in o && (o.writable = !0), Object.defineProperty(e, _toPropertyKey(o.key), o); } }
function _createClass(e, r, t) { return r && _defineProperties(e.prototype, r), t && _defineProperties(e, t), Object.defineProperty(e, "prototype", { writable: !1 }), e; }
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
var path = require("path");
var fs = require("fs");
var StorageEngine = /*#__PURE__*/function () {
  function StorageEngine(dbPath) {
    var shardCount = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 4;
    var options = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
    _classCallCheck(this, StorageEngine);
    var _options$dirMode = options.dirMode,
      dirMode = _options$dirMode === void 0 ? 448 : _options$dirMode,
      _options$fileMode = options.fileMode,
      fileMode = _options$fileMode === void 0 ? 384 : _options$fileMode;
    this.dbPath = dbPath;
    this.shardCount = shardCount;
    this.fileMode = fileMode;
    this.dirMode = dirMode;
    if (!fs.existsSync(dbPath)) {
      fs.mkdirSync(dbPath, {
        recursive: true,
        mode: this.dirMode
      });
    }
    this.indexMap = new Map();
    this.loadIndexes();
  }
  return _createClass(StorageEngine, [{
    key: "getShardFile",
    value: function getShardFile(collection, shard) {
      return path.join(this.dbPath, "".concat(collection, "_shard_").concat(shard, ".ldb"));
    }
  }, {
    key: "getIndexFile",
    value: function getIndexFile(collection, shard) {
      return path.join(this.dbPath, "".concat(collection, "_shard_").concat(shard, ".idx"));
    }
  }, {
    key: "getShardID",
    value: function getShardID(id) {
      return parseInt(id, 36) % this.shardCount;
    }
  }, {
    key: "write",
    value: function write(collection, id, value) {
      var shard = this.getShardID(id);
      var filePath = this.getShardFile(collection, shard);
      var indexPath = this.getIndexFile(collection, shard);
      var valueBuffer = Buffer.from(JSON.stringify(value));
      var keyBuffer = Buffer.from(id);
      var keyLength = Buffer.alloc(4);
      keyLength.writeUInt32BE(keyBuffer.length);
      var valueLength = Buffer.alloc(4);
      valueLength.writeUInt32BE(valueBuffer.length);
      var recordBuffer = Buffer.concat([keyLength, keyBuffer, valueLength, valueBuffer]);
      var dir = path.dirname(filePath);
      if (!fs.existsSync(dir)) {
        fs.mkdirSync(dir, {
          recursive: true,
          mode: this.dirMode
        });
      }
      if (!fs.existsSync(filePath)) {
        fs.writeFileSync(filePath, Buffer.alloc(0), {
          mode: this.fileMode
        });
      }
      var fd = fs.openSync(filePath, "r+");
      var offset = fs.statSync(filePath).size;
      fs.writeSync(fd, recordBuffer, 0, recordBuffer.length, offset);
      fs.closeSync(fd);
      var keyLenBuf = Buffer.alloc(4);
      keyLenBuf.writeUInt32BE(keyBuffer.length);
      var offsetBuf = Buffer.alloc(8);
      offsetBuf.writeBigUInt64BE(BigInt(offset));
      var idxEntry = Buffer.concat([keyLenBuf, keyBuffer, offsetBuf]);
      fs.appendFileSync(indexPath, idxEntry, {
        mode: this.fileMode
      });
      var indexKey = "".concat(collection, ":").concat(id);
      this.indexMap.set(indexKey, {
        filePath: filePath,
        offset: offset
      });
      return {
        id: id,
        value: value
      };
    }
  }, {
    key: "readAll",
    value: function readAll(collection) {
      var strictMatch = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
      var normalized = collection.replace(/\\/g, "/");
      var actualCollection = normalized.split("/").pop();
      var results = [];
      var _iterator = _createForOfIteratorHelper(this.indexMap.entries()),
        _step;
      try {
        for (_iterator.s(); !(_step = _iterator.n()).done;) {
          var _step$value = _slicedToArray(_step.value, 2),
            indexKey = _step$value[0],
            pointer = _step$value[1];
          var normalizedKey = indexKey.replace(/\\/g, "/");
          var matchesShort = normalizedKey.startsWith(normalized.split("/").pop() + ":");
          var matchesFull = normalizedKey.startsWith(normalized + ":");
          var matches = strictMatch ? matchesShort : !(!matchesShort && !matchesFull);
          if (!matches) {
            continue;
          }
          var id = normalizedKey.split(":")[1];
          var record = this.read(normalizedKey.split(":")[0], id);
          if (record !== null) {
            results.push({
              id: id,
              data: record
            });
          }
        }
      } catch (err) {
        _iterator.e(err);
      } finally {
        _iterator.f();
      }
      return results;
    }
  }, {
    key: "read",
    value: function read(collection, id) {
      var indexKey = "".concat(collection, ":").concat(id);
      var pointer = this.indexMap.get(indexKey);
      if (!pointer) {
        return null;
      }
      var fd = fs.openSync(pointer.filePath, "r");
      var keyLengthBuffer = Buffer.alloc(4);
      fs.readSync(fd, keyLengthBuffer, 0, 4, pointer.offset);
      var keyLength = keyLengthBuffer.readUInt32BE();
      var keyBuffer = Buffer.alloc(keyLength);
      fs.readSync(fd, keyBuffer, 0, keyLength, pointer.offset + 4);
      var valueLengthBuffer = Buffer.alloc(4);
      fs.readSync(fd, valueLengthBuffer, 0, 4, pointer.offset + 4 + keyLength);
      var valueLength = valueLengthBuffer.readUInt32BE();
      var valueBuffer = Buffer.alloc(valueLength);
      fs.readSync(fd, valueBuffer, 0, valueLength, pointer.offset + 8 + keyLength);
      fs.closeSync(fd);
      var actualKey = keyBuffer.toString();
      if (actualKey !== id) {
        return null;
      }
      return JSON.parse(valueBuffer.toString());
    }
  }, {
    key: "delete",
    value: function _delete(collection, ids) {
      var _this = this;
      var idsToDelete = Array.isArray(ids) ? ids : [ids];
      var recordsDeleted = false;
      for (var i = 0; i < this.shardCount; i++) {
        var filePath = this.getShardFile(collection, i);
        var indexPath = this.getIndexFile(collection, i);
        if (!fs.existsSync(filePath)) {
          continue;
        }
        var fd = fs.openSync(filePath, "r");
        var size = fs.statSync(filePath).size;
        var offset = 0;
        var records = [];
        while (offset < size) {
          var keyLengthBuffer = Buffer.alloc(4);
          fs.readSync(fd, keyLengthBuffer, 0, 4, offset);
          var keyLength = keyLengthBuffer.readUInt32BE();
          offset += 4;
          var keyBuffer = Buffer.alloc(keyLength);
          fs.readSync(fd, keyBuffer, 0, keyLength, offset);
          var id = keyBuffer.toString();
          offset += keyLength;
          var valueLengthBuffer = Buffer.alloc(4);
          fs.readSync(fd, valueLengthBuffer, 0, 4, offset);
          var valueLength = valueLengthBuffer.readUInt32BE();
          offset += 4;
          var valueBuffer = Buffer.alloc(valueLength);
          fs.readSync(fd, valueBuffer, 0, valueLength, offset);
          offset += valueLength;
          if (!idsToDelete.includes(id)) {
            records.push({
              id: id,
              value: JSON.parse(valueBuffer.toString())
            });
          } else {
            recordsDeleted = true;
            var indexKey = "".concat(collection, ":").concat(id);
            this.indexMap["delete"](indexKey);
          }
        }
        fs.closeSync(fd);
        if (recordsDeleted) {
          fs.unlinkSync(filePath);
          if (fs.existsSync(indexPath)) {
            fs.unlinkSync(indexPath);
          }
          records.forEach(function (r) {
            return _this.write(collection, r.id, r.value);
          });
        }
      }
      return recordsDeleted;
    }
  }, {
    key: "overwrite",
    value: function overwrite(collection, data) {
      var _this2 = this;
      for (var i = 0; i < this.shardCount; i++) {
        var filePath = this.getShardFile(collection, i);
        var indexPath = this.getIndexFile(collection, i);
        if (fs.existsSync(filePath)) {
          fs.unlinkSync(filePath);
        }
        if (fs.existsSync(indexPath)) {
          fs.unlinkSync(indexPath);
        }
      }
      for (var _i = 0, _Array$from = Array.from(this.indexMap.keys()); _i < _Array$from.length; _i++) {
        var key = _Array$from[_i];
        if (key.startsWith("".concat(collection, ":"))) {
          this.indexMap["delete"](key);
        }
      }
      var lines = data.split("\n").filter(Boolean);
      lines.forEach(function (line) {
        var record = JSON.parse(line);
        _this2.write(collection, record.id, record.data);
      });
    }
  }, {
    key: "deleteCollection",
    value: function deleteCollection(collection) {
      var deletedShards = 0;
      for (var i = 0; i < this.shardCount; i++) {
        var filePath = this.getShardFile(collection, i);
        var indexPath = this.getIndexFile(collection, i);
        if (fs.existsSync(filePath)) {
          fs.unlinkSync(filePath);
          deletedShards++;
          var _iterator2 = _createForOfIteratorHelper(this.indexMap.keys()),
            _step2;
          try {
            for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
              var key = _step2.value;
              if (key.startsWith("".concat(collection, ":"))) {
                this.indexMap["delete"](key);
              }
            }
          } catch (err) {
            _iterator2.e(err);
          } finally {
            _iterator2.f();
          }
        }
        if (fs.existsSync(indexPath)) {
          fs.unlinkSync(indexPath);
        }
      }
      for (var _i2 = 0, _Array$from2 = Array.from(this.indexMap.keys()); _i2 < _Array$from2.length; _i2++) {
        var _key = _Array$from2[_i2];
        if (_key.startsWith("".concat(collection, ":"))) {
          this.indexMap["delete"](_key);
        }
      }
      return deletedShards > 0;
    }
  }, {
    key: "listCollections",
    value: function listCollections() {
      var _this3 = this;
      var dir = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : this.dbPath;
      var basePath = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : "";
      var entries = fs.readdirSync(dir, {
        withFileTypes: true
      });
      var collections = new Set();
      entries.forEach(function (entry) {
        var fullPath = path.join(dir, entry.name);
        if (entry.isDirectory()) {
          var subCollections = _this3.listCollections(fullPath, path.join(basePath, entry.name));
          subCollections.forEach(function (col) {
            return collections.add(col);
          });
        } else if (entry.isFile()) {
          var match = entry.name.match(/^(.+)_shard_\d+\.ldb$/);
          if (match) {
            collections.add(path.join(basePath, match[1]));
          }
        }
      });
      return Array.from(collections);
    }
  }, {
    key: "findIDXFiles",
    value: function findIDXFiles() {
      var _walkDir = function walkDir(dir) {
        var entries = fs.readdirSync(dir, {
          withFileTypes: true
        });
        var results = [];
        var _iterator3 = _createForOfIteratorHelper(entries),
          _step3;
        try {
          for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
            var entry = _step3.value;
            var fullPath = path.join(dir, entry.name);
            if (entry.isDirectory()) {
              results = results.concat(_walkDir(fullPath));
            } else if (entry.isFile() && entry.name.endsWith(".idx")) {
              results.push(fullPath);
            }
          }
        } catch (err) {
          _iterator3.e(err);
        } finally {
          _iterator3.f();
        }
        return results;
      };
      return _walkDir(this.dbPath);
    }
  }, {
    key: "rebuildIDXFromMap",
    value: function rebuildIDXFromMap() {
      var idxFiles = this.findIDXFiles();
      var _iterator4 = _createForOfIteratorHelper(idxFiles),
        _step4;
      try {
        for (_iterator4.s(); !(_step4 = _iterator4.n()).done;) {
          var filePath = _step4.value;
          try {
            fs.unlinkSync(filePath);
          } catch (e) {
            console.warn("[StarVault] ~ Did not delete old index file ".concat(filePath, " ~"), e.message);
          }
        }
      } catch (err) {
        _iterator4.e(err);
      } finally {
        _iterator4.f();
      }
      var _iterator5 = _createForOfIteratorHelper(this.indexMap.entries()),
        _step5;
      try {
        for (_iterator5.s(); !(_step5 = _iterator5.n()).done;) {
          var _step5$value = _slicedToArray(_step5.value, 2),
            mapKey = _step5$value[0],
            _step5$value$ = _step5$value[1],
            _filePath = _step5$value$.filePath,
            offset = _step5$value$.offset;
          var id = mapKey.split(":").pop();
          var collection = path.relative(this.dbPath, _filePath).replace(/\\/g, "/").split("_shard_")[0];
          var shard = this.getShardID(id);
          var indexPath = this.getIndexFile(collection, shard);
          var keyBuf = Buffer.from(id);
          var keyLenBuf = Buffer.alloc(4);
          keyLenBuf.writeUInt32BE(keyBuf.length);
          var offsetBuf = Buffer.alloc(8);
          offsetBuf.writeBigUInt64BE(BigInt(offset));
          var idxEntry = Buffer.concat([keyLenBuf, keyBuf, offsetBuf]);
          fs.appendFileSync(indexPath, idxEntry, {
            mode: this.fileMode
          });
        }
      } catch (err) {
        _iterator5.e(err);
      } finally {
        _iterator5.f();
      }
      console.log("[StarVault] Index files rebuilt from current data.");
    }
  }, {
    key: "loadIndexesFromData",
    value: function loadIndexesFromData() {
      var _walkDir2 = function walkDir(dir) {
        var entries = fs.readdirSync(dir, {
          withFileTypes: true
        });
        var ldbFiles = [];
        var _iterator6 = _createForOfIteratorHelper(entries),
          _step6;
        try {
          for (_iterator6.s(); !(_step6 = _iterator6.n()).done;) {
            var entry = _step6.value;
            var fullPath = path.join(dir, entry.name);
            if (entry.isDirectory()) {
              ldbFiles = ldbFiles.concat(_walkDir2(fullPath));
            } else if (entry.isFile() && entry.name.endsWith(".ldb")) {
              ldbFiles.push(fullPath);
            }
          }
        } catch (err) {
          _iterator6.e(err);
        } finally {
          _iterator6.f();
        }
        return ldbFiles;
      };
      var ldbFiles = _walkDir2(this.dbPath);
      var _iterator7 = _createForOfIteratorHelper(ldbFiles),
        _step7;
      try {
        for (_iterator7.s(); !(_step7 = _iterator7.n()).done;) {
          var filePath = _step7.value;
          var fileName = path.basename(filePath);
          var collection = fileName.split("_shard_")[0];
          var offset = 0;
          var fd = fs.openSync(filePath, "r");
          var size = fs.statSync(filePath).size;
          while (offset < size) {
            try {
              var keyLengthBuffer = Buffer.alloc(4);
              fs.readSync(fd, keyLengthBuffer, 0, 4, offset);
              var keyLength = keyLengthBuffer.readUInt32BE();
              offset += 4;
              var keyBuffer = Buffer.alloc(keyLength);
              fs.readSync(fd, keyBuffer, 0, keyLength, offset);
              var id = keyBuffer.toString();
              offset += keyLength;
              var valueLengthBuffer = Buffer.alloc(4);
              fs.readSync(fd, valueLengthBuffer, 0, 4, offset);
              var valueLength = valueLengthBuffer.readUInt32BE();
              offset += 4 + valueLength;
              var indexOffset = offset - (4 + keyLength + 4 + valueLength);
              var mapKey = "".concat(collection, ":").concat(id);
              this.indexMap.set(mapKey, {
                filePath: filePath,
                offset: indexOffset
              });
            } catch (error) {
              console.log("Did not index record at ".concat(offset, " in ").concat(filePath, ":"), error.message);
              break;
            }
          }
          fs.closeSync(fd);
        }
      } catch (err) {
        _iterator7.e(err);
      } finally {
        _iterator7.f();
      }
    }
  }, {
    key: "loadIndexesFromIDX",
    value: function loadIndexesFromIDX(idxFiles) {
      var _iterator8 = _createForOfIteratorHelper(idxFiles),
        _step8;
      try {
        for (_iterator8.s(); !(_step8 = _iterator8.n()).done;) {
          var filePath = _step8.value;
          var fileName = path.basename(filePath);
          var relPath = path.relative(this.dbPath, filePath).replace(/\\/g, "/");
          var collection = relPath.split("_shard_")[0];
          var shardPart = fileName.split("_shard_")[1];
          var shard = parseInt(shardPart.split(".")[0]);
          var ldbPath = this.getShardFile(collection, shard);
          var fd = fs.openSync(filePath, "r");
          var size = fs.statSync(filePath).size;
          var offset = 0;
          while (offset < size) {
            try {
              var keyLenBuf = Buffer.alloc(4);
              fs.readSync(fd, keyLenBuf, 0, 4, offset);
              var keyLen = keyLenBuf.readUInt32BE();
              var keyBuf = Buffer.alloc(keyLen);
              fs.readSync(fd, keyBuf, 0, keyLen, offset + 4);
              var id = keyBuf.toString();
              var offsetBuf = Buffer.alloc(8);
              fs.readSync(fd, offsetBuf, 0, 8, offset + 4 + keyLen);
              var fileOffset = Number(offsetBuf.readBigUInt64BE());
              var mapKey = "".concat(collection, ":").concat(id);
              this.indexMap.set(mapKey, {
                filePath: ldbPath,
                offset: fileOffset
              });
              offset += 4 + keyLen + 8;
            } catch (error) {
              console.warn("Corrupt .idx entry at ".concat(offset, " in ").concat(filePath, ":"), error.message);
              break;
            }
          }
          fs.closeSync(fd);
        }
      } catch (err) {
        _iterator8.e(err);
      } finally {
        _iterator8.f();
      }
    }
  }, {
    key: "loadIndexes",
    value: function loadIndexes() {
      var idxFiles = this.findIDXFiles();
      if (idxFiles.length > 0) {
        this.loadIndexesFromIDX(idxFiles);
      } else {
        console.warn("[StarVault] No index files found. Rebuilding index from current data...");
        this.loadIndexesFromData();
        this.rebuildIDXFromMap();
      }
    }
  }]);
}();
;
module.exports = StorageEngine;