axiodb
Version:
The Pure JavaScript Alternative to SQLite. Embedded NoSQL database for Node.js with MongoDB-style queries, zero native dependencies, built-in InMemoryCache, and web GUI. Perfect for desktop apps, CLI tools, and embedded systems. No compilation, no platfor
139 lines • 7.27 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = ReaderWithWorker;
/* eslint-disable @typescript-eslint/no-explicit-any */
const worker_threads_1 = require("worker_threads");
const path_1 = __importDefault(require("path"));
const os_1 = __importDefault(require("os"));
const FileManager_1 = __importDefault(require("../engine/Filesystem/FileManager"));
const Converter_helper_1 = __importDefault(require("../Helper/Converter.helper"));
const Crypto_helper_1 = require("../Helper/Crypto.helper");
/**
* Reads data files using worker threads to parallelize the loading process.
* For small datasets (<100 files), uses direct file reading to avoid worker thread overhead.
* Each worker processes a chunk of the data files list.
*
* @param DataFilesList - An array of file paths to be read.
* @param encryptionKey - Encryption key for decryption if needed.
* @param path - The base path where the files are located.
* @param isEncrypted - A boolean indicating if the files are encrypted.
* @param storeFileName - Whether to include the fileName in the result objects.
* @returns {Promise<any[]>} - A promise that resolves to an array of loaded data.
*/
function ReaderWithWorker(DataFilesList_1, encryptionKey_1, path_2, isEncrypted_1) {
return __awaiter(this, arguments, void 0, function* (DataFilesList, encryptionKey, path, isEncrypted, storeFileName = false) {
// Fast path: Direct file reading for small datasets (<100 files)
// Avoids 300-400ms worker thread initialization overhead
if (DataFilesList.length < 100) {
const fileManager = new FileManager_1.default();
const converter = new Converter_helper_1.default();
const cryptoInstance = isEncrypted && encryptionKey
? new Crypto_helper_1.CryptoHelper(encryptionKey)
: undefined;
// Process all files in parallel for maximum performance
const filePromises = DataFilesList.map((fileName) => __awaiter(this, void 0, void 0, function* () {
try {
const ReadFileResponse = yield fileManager.ReadFile(`${path}/${fileName}`);
if ("data" in ReadFileResponse) {
if (isEncrypted && cryptoInstance) {
const ContentResponse = yield cryptoInstance.decrypt(converter.ToObject(ReadFileResponse.data));
if (storeFileName) {
return {
fileName: fileName,
data: converter.ToObject(ContentResponse),
};
}
else {
return converter.ToObject(ContentResponse);
}
}
else {
if (storeFileName) {
return {
fileName: fileName,
data: converter.ToObject(ReadFileResponse.data),
};
}
else {
return converter.ToObject(ReadFileResponse.data);
}
}
}
else {
console.error(`Failed to read file: ${fileName}`);
return null;
}
}
catch (error) {
console.error(`Error processing file ${fileName}:`, error);
return null;
}
}));
const fileResults = yield Promise.all(filePromises);
return fileResults.filter((r) => r !== null);
}
// Worker thread path for larger datasets
const workerPath = path_1.default.resolve(__dirname, "../engine/node", "WorkerForDataLoad.engine.js");
const tasks = [];
// Use single worker for medium datasets (100-4999 files)
if (DataFilesList.length < 5000) {
tasks.push(new Promise((resolve, reject) => {
const worker = new worker_threads_1.Worker(workerPath, {
workerData: {
chunk: DataFilesList,
encryptionKey: encryptionKey,
path: path,
isEncrypted: isEncrypted,
storeFileName: storeFileName,
},
});
worker.on("message", resolve);
worker.on("error", reject);
worker.on("exit", (code) => {
if (code !== 0)
reject(new Error(`Worker stopped with code ${code}`));
});
}));
}
else {
// For 5000+ files, divide work among multiple workers
for (let i = 0; i < Math.min(os_1.default.cpus().length, DataFilesList.length); i++) {
const start = i * Math.ceil(DataFilesList.length / Math.min(os_1.default.cpus().length, DataFilesList.length));
const end = Math.min(start + Math.ceil(DataFilesList.length / Math.min(os_1.default.cpus().length, DataFilesList.length)), DataFilesList.length);
const dataChunk = DataFilesList.slice(start, end);
tasks.push(new Promise((resolve, reject) => {
const worker = new worker_threads_1.Worker(workerPath, {
workerData: {
chunk: dataChunk,
encryptionKey: encryptionKey,
path: path,
isEncrypted: isEncrypted,
storeFileName: storeFileName,
},
});
worker.on("message", resolve);
worker.on("error", reject);
worker.on("exit", (code) => {
if (code !== 0)
reject(new Error(`Worker stopped with code ${code}`));
});
}));
}
}
const results = yield Promise.all(tasks);
return results.flat();
});
}
//# sourceMappingURL=BufferLoaderWithWorker.utils.js.map