axiodb
Version:
A blazing-fast, lightweight, and scalable nodejs package based DBMS for modern application. Supports schemas, encryption, and advanced query capabilities.
61 lines • 3.25 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = ReaderWithWorker;
/* eslint-disable @typescript-eslint/no-explicit-any */
const worker_threads_1 = require("worker_threads");
const path_1 = __importDefault(require("path"));
/**
* Reads data files using worker threads to parallelize the loading process.
* Each worker processes a chunk of the data files list.
*
* @param DataFilesList - An array of file paths to be read.
* @param cryptoInstance - An instance of a crypto library for decryption if needed.
* @param path - The base path where the files are located.
* @param isEncrypted - A boolean indicating if the files are encrypted.
* @returns {Promise<any[]>} - A promise that resolves to an array of loaded data.
*/
function ReaderWithWorker(DataFilesList_1, cryptoInstance_1, path_2, isEncrypted_1) {
return __awaiter(this, arguments, void 0, function* (DataFilesList, cryptoInstance, path, isEncrypted, storeFileName = false) {
const numWorkers = 1; // Use a single worker for simplicity, can be adjusted based on requirements
const chunkSize = Math.ceil(DataFilesList.length / numWorkers);
const workerPath = path_1.default.resolve(__dirname, "../engine/node", "WorkerForDataLoad.engine.js");
const tasks = [];
for (let i = 0; i < numWorkers; i++) {
const start = i * chunkSize;
const end = Math.min(start + chunkSize, DataFilesList.length);
const dataChunk = DataFilesList.slice(start, end);
tasks.push(new Promise((resolve, reject) => {
const worker = new worker_threads_1.Worker(workerPath, {
workerData: {
chunk: dataChunk,
cryptoInstance: cryptoInstance,
path: path,
isEncrypted: isEncrypted,
storeFileName: storeFileName,
},
});
worker.on("message", resolve);
worker.on("error", reject);
worker.on("exit", (code) => {
if (code !== 0)
reject(new Error(`Worker stopped with code ${code}`));
});
}));
}
const results = yield Promise.all(tasks);
return results.flat();
});
}
//# sourceMappingURL=BufferLoaderWithWorker.utils.js.map