@trap_stevo/star-vault
Version:
Unleash the future of data management with the ultimate platform for secure, scalable, and dynamic data operations. Power the next generation of applications by combining advanced encryption, revolutionary real-time querying, and seamless synchronization
342 lines (341 loc) • 15.2 kB
JavaScript
;
function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
function _slicedToArray(r, e) { return _arrayWithHoles(r) || _iterableToArrayLimit(r, e) || _unsupportedIterableToArray(r, e) || _nonIterableRest(); }
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _iterableToArrayLimit(r, l) { var t = null == r ? null : "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (null != t) { var e, n, i, u, a = [], f = !0, o = !1; try { if (i = (t = t.call(r)).next, 0 === l) { if (Object(t) !== t) return; f = !1; } else for (; !(f = (e = i.call(t)).done) && (a.push(e.value), a.length !== l); f = !0); } catch (r) { o = !0, n = r; } finally { try { if (!f && null != t["return"] && (u = t["return"](), Object(u) !== u)) return; } finally { if (o) throw n; } } return a; } }
function _arrayWithHoles(r) { if (Array.isArray(r)) return r; }
function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
function _classCallCheck(a, n) { if (!(a instanceof n)) throw new TypeError("Cannot call a class as a function"); }
function _defineProperties(e, r) { for (var t = 0; t < r.length; t++) { var o = r[t]; o.enumerable = o.enumerable || !1, o.configurable = !0, "value" in o && (o.writable = !0), Object.defineProperty(e, _toPropertyKey(o.key), o); } }
function _createClass(e, r, t) { return r && _defineProperties(e.prototype, r), t && _defineProperties(e, t), Object.defineProperty(e, "prototype", { writable: !1 }), e; }
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
var path = require("path");
var fs = require("fs");
var StorageEngine = /*#__PURE__*/function () {
function StorageEngine(dbPath) {
var shardCount = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 4;
var options = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : {};
_classCallCheck(this, StorageEngine);
var _options$dirMode = options.dirMode,
dirMode = _options$dirMode === void 0 ? 448 : _options$dirMode,
_options$fileMode = options.fileMode,
fileMode = _options$fileMode === void 0 ? 384 : _options$fileMode;
this.dbPath = dbPath;
this.shardCount = shardCount;
this.fileMode = fileMode;
this.dirMode = dirMode;
if (!fs.existsSync(dbPath)) {
fs.mkdirSync(dbPath, {
recursive: true,
mode: this.dirMode
});
}
this.indexMap = new Map();
this.loadIndexes();
}
return _createClass(StorageEngine, [{
key: "getShardFile",
value: function getShardFile(collection, shard) {
return path.join(this.dbPath, "".concat(collection, "_shard_").concat(shard, ".ldb"));
}
}, {
key: "getShardID",
value: function getShardID(id) {
return parseInt(id, 36) % this.shardCount;
}
}, {
key: "write",
value: function write(collection, id, value) {
var shard = this.getShardID(id);
var filePath = this.getShardFile(collection, shard);
var valueBuffer = Buffer.from(JSON.stringify(value));
var keyBuffer = Buffer.from(id);
var keyLength = Buffer.alloc(4);
keyLength.writeUInt32BE(keyBuffer.length);
var valueLength = Buffer.alloc(4);
valueLength.writeUInt32BE(valueBuffer.length);
var recordBuffer = Buffer.concat([keyLength, keyBuffer, valueLength, valueBuffer]);
var dir = path.dirname(filePath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, {
recursive: true,
mode: this.dirMode
});
}
if (!fs.existsSync(filePath)) {
fs.writeFileSync(filePath, Buffer.alloc(0), {
mode: this.fileMode
});
}
var fd = fs.openSync(filePath, "r+");
var offset = fs.statSync(filePath).size;
fs.writeSync(fd, recordBuffer, 0, recordBuffer.length, offset);
fs.closeSync(fd);
var indexKey = "".concat(collection, ":").concat(id);
this.indexMap.set(indexKey, {
filePath: filePath,
offset: offset
});
return {
id: id,
value: value
};
}
}, {
key: "readAll",
value: function readAll(collection) {
var normalized = collection.replace(/\\/g, "/");
var actualCollection = normalized.split("/").pop();
var results = [];
var _iterator = _createForOfIteratorHelper(this.indexMap.entries()),
_step;
try {
for (_iterator.s(); !(_step = _iterator.n()).done;) {
var _step$value = _slicedToArray(_step.value, 2),
indexKey = _step$value[0],
pointer = _step$value[1];
var normalizedKey = indexKey.replace(/\\/g, "/");
var matchesShort = normalizedKey.startsWith(normalized.split("/").pop() + ":");
var matchesFull = normalizedKey.startsWith(normalized + ":");
if (!matchesShort && !matchesFull) {
continue;
}
var id = normalizedKey.split(":")[1];
var record = this.read(normalizedKey.split(":")[0], id);
if (record !== null) {
results.push({
id: id,
data: record
});
}
}
} catch (err) {
_iterator.e(err);
} finally {
_iterator.f();
}
return results;
}
}, {
key: "read",
value: function read(collection, id) {
var indexKey = "".concat(collection, ":").concat(id);
var pointer = this.indexMap.get(indexKey);
if (!pointer) {
return null;
}
var fd = fs.openSync(pointer.filePath, "r");
var keyLengthBuffer = Buffer.alloc(4);
fs.readSync(fd, keyLengthBuffer, 0, 4, pointer.offset);
var keyLength = keyLengthBuffer.readUInt32BE();
var keyBuffer = Buffer.alloc(keyLength);
fs.readSync(fd, keyBuffer, 0, keyLength, pointer.offset + 4);
var valueLengthBuffer = Buffer.alloc(4);
fs.readSync(fd, valueLengthBuffer, 0, 4, pointer.offset + 4 + keyLength);
var valueLength = valueLengthBuffer.readUInt32BE();
var valueBuffer = Buffer.alloc(valueLength);
fs.readSync(fd, valueBuffer, 0, valueLength, pointer.offset + 8 + keyLength);
fs.closeSync(fd);
var actualKey = keyBuffer.toString();
if (actualKey !== id) {
return null;
}
return JSON.parse(valueBuffer.toString());
}
}, {
key: "delete",
value: function _delete(collection, ids) {
var _this = this;
var idsToDelete = Array.isArray(ids) ? ids : [ids];
var recordsDeleted = false;
for (var i = 0; i < this.shardCount; i++) {
var filePath = this.getShardFile(collection, i);
if (!fs.existsSync(filePath)) {
continue;
}
var fd = fs.openSync(filePath, "r");
var size = fs.statSync(filePath).size;
var offset = 0;
var records = [];
while (offset < size) {
var keyLengthBuffer = Buffer.alloc(4);
fs.readSync(fd, keyLengthBuffer, 0, 4, offset);
var keyLength = keyLengthBuffer.readUInt32BE();
offset += 4;
var keyBuffer = Buffer.alloc(keyLength);
fs.readSync(fd, keyBuffer, 0, keyLength, offset);
var id = keyBuffer.toString();
offset += keyLength;
var valueLengthBuffer = Buffer.alloc(4);
fs.readSync(fd, valueLengthBuffer, 0, 4, offset);
var valueLength = valueLengthBuffer.readUInt32BE();
offset += 4;
var valueBuffer = Buffer.alloc(valueLength);
fs.readSync(fd, valueBuffer, 0, valueLength, offset);
offset += valueLength;
if (!idsToDelete.includes(id)) {
records.push({
id: id,
value: JSON.parse(valueBuffer.toString())
});
} else {
recordsDeleted = true;
}
}
fs.closeSync(fd);
if (recordsDeleted) {
fs.unlinkSync(filePath);
records.forEach(function (r) {
return _this.write(collection, r.id, r.value);
});
}
}
return recordsDeleted;
}
}, {
key: "overwrite",
value: function overwrite(collection, data) {
var _this2 = this;
for (var i = 0; i < this.shardCount; i++) {
var filePath = this.getShardFile(collection, i);
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
}
}
var lines = data.split("\n").filter(Boolean);
lines.forEach(function (line) {
var record = JSON.parse(line);
_this2.write(collection, record.id, record.data);
});
}
}, {
key: "deleteCollection",
value: function deleteCollection(collection) {
var deletedShards = 0;
for (var i = 0; i < this.shardCount; i++) {
var filePath = this.getShardFile(collection, i);
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
deletedShards++;
var _iterator2 = _createForOfIteratorHelper(this.indexMap.keys()),
_step2;
try {
for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
var key = _step2.value;
if (key.startsWith("".concat(collection, ":"))) {
this.indexMap["delete"](key);
}
}
} catch (err) {
_iterator2.e(err);
} finally {
_iterator2.f();
}
}
}
return deletedShards > 0;
}
}, {
key: "listCollections",
value: function listCollections() {
var _this3 = this;
var dir = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : this.dbPath;
var basePath = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : "";
var entries = fs.readdirSync(dir, {
withFileTypes: true
});
var collections = new Set();
entries.forEach(function (entry) {
var fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
var subCollections = _this3.listCollections(fullPath, path.join(basePath, entry.name));
subCollections.forEach(function (col) {
return collections.add(col);
});
} else if (entry.isFile()) {
var match = entry.name.match(/^(.+)_shard_\d+\.ldb$/);
if (match) {
collections.add(path.join(basePath, match[1]));
}
}
});
return Array.from(collections);
}
}, {
key: "loadIndexes",
value: function loadIndexes() {
var _walkDir = function walkDir(dir) {
var entries = fs.readdirSync(dir, {
withFileTypes: true
});
var ldbFiles = [];
var _iterator3 = _createForOfIteratorHelper(entries),
_step3;
try {
for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
var entry = _step3.value;
var fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
ldbFiles = ldbFiles.concat(_walkDir(fullPath));
} else if (entry.isFile() && entry.name.endsWith(".ldb")) {
ldbFiles.push(fullPath);
}
}
} catch (err) {
_iterator3.e(err);
} finally {
_iterator3.f();
}
return ldbFiles;
};
var ldbFiles = _walkDir(this.dbPath);
var _iterator4 = _createForOfIteratorHelper(ldbFiles),
_step4;
try {
for (_iterator4.s(); !(_step4 = _iterator4.n()).done;) {
var filePath = _step4.value;
var fileName = path.basename(filePath);
var collection = fileName.split("_shard_")[0];
var offset = 0;
var fd = fs.openSync(filePath, "r");
var size = fs.statSync(filePath).size;
while (offset < size) {
try {
var keyLengthBuffer = Buffer.alloc(4);
fs.readSync(fd, keyLengthBuffer, 0, 4, offset);
var keyLength = keyLengthBuffer.readUInt32BE();
offset += 4;
var keyBuffer = Buffer.alloc(keyLength);
fs.readSync(fd, keyBuffer, 0, keyLength, offset);
var id = keyBuffer.toString();
offset += keyLength;
var valueLengthBuffer = Buffer.alloc(4);
fs.readSync(fd, valueLengthBuffer, 0, 4, offset);
var valueLength = valueLengthBuffer.readUInt32BE();
offset += 4 + valueLength;
var indexOffset = offset - (4 + keyLength + 4 + valueLength);
var mapKey = "".concat(collection, ":").concat(id);
this.indexMap.set(mapKey, {
filePath: filePath,
offset: indexOffset
});
} catch (error) {
console.log("Did not index record at ".concat(offset, " in ").concat(filePath, ":"), error.message);
break;
}
}
fs.closeSync(fd);
}
} catch (err) {
_iterator4.e(err);
} finally {
_iterator4.f();
}
}
}]);
}();
;
module.exports = StorageEngine;