node-downloader-manager
Version:
node-downloader-manager is a simple yet powerful package manager-like download manager built with NodeJs. It allows you to download files sequentially or with a queue-based approach, handling retries and concurrency limits efficiently.
409 lines (408 loc) • 15.9 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const node_fs_1 = __importDefault(require("node:fs"));
const node_path_1 = __importDefault(require("node:path"));
const node_events_1 = require("node:events");
const Queue_js_1 = __importDefault(require("./Queue.js"));
const Thread_js_1 = __importDefault(require("./Thread.js"));
const const_1 = require("../config/const");
class DownloadManager extends node_events_1.EventEmitter {
options;
downloadQueue;
downloadThread;
method;
concurrencyLimit;
downloadFolder;
getFileName;
onAfterDownload;
onBeforeDownload;
log;
overWriteFile;
stream;
requestOptions;
activeDownloads;
currentUrl = "CURRENT_URL";
currentDownloadableUrl;
timeout;
constructor(options = {}) {
super();
this.activeDownloads = new Map();
this.currentDownloadableUrl = new Map();
const { method = "queue", concurrencyLimit = 5, retries = 3, consoleLog = false, downloadFolder = "./downloads", overWriteFile = false, getFileName, onAfterDownload, requestOptions = {}, onBeforeDownload, stream = false, backOff = false, timeout = 30000, maxWorkers = 5, } = options;
this.options = options;
this.method = method;
this.concurrencyLimit = concurrencyLimit;
this.log = consoleLog;
this.downloadFolder = downloadFolder;
this.getFileName = getFileName;
this.onAfterDownload = onAfterDownload;
this.onBeforeDownload = onBeforeDownload;
this.overWriteFile = overWriteFile;
this.requestOptions = requestOptions;
this.stream = stream;
this.timeout = timeout;
if (method === "queue") {
this.downloadQueue = new Queue_js_1.default(concurrencyLimit, retries, consoleLog, backOff);
}
if (method === "thread") {
this.downloadThread = new Thread_js_1.default(maxWorkers, consoleLog);
}
}
logger(message, type = "info") {
if (this.log) {
console[type](`[${process.pid}] : [${new Date().toLocaleString()}] : `, message);
}
}
on(event, callback) {
return super.on(event, callback);
}
emit(event, data) {
return super.emit(event, data);
}
pauseDownload(url = this.currentDownloadableUrl.get(this.currentUrl)) {
const downloadData = this.activeDownloads.get(url);
if (downloadData && !downloadData.paused) {
downloadData.paused = true;
downloadData.stream.close();
this.emit(const_1.emitEvents.paused, {
url,
message: const_1.emitMessages.paused,
});
this.logger(`Download for ${url} paused.`);
}
else {
this.logger(`No active download found for URL: ${url}`, "warn");
}
}
async resumeDownload(url = this.currentDownloadableUrl.get(this.currentUrl)) {
const downloadData = this.activeDownloads.get(url);
if (downloadData?.paused) {
this.emit(const_1.emitEvents.resumed, { url, message: const_1.emitMessages.resumed });
this.logger(`Resuming download for ${url}`);
downloadData.paused = false;
if (this.method === "queue") {
this.enqueueDownloadTask(url, node_path_1.default.basename(downloadData.stream.path), downloadData.downloaded);
}
else if (this.method === "simple") {
this.downloadFile(url, node_path_1.default.basename(downloadData.stream.path), downloadData.downloaded);
}
}
else {
this.logger(`No paused download found for URL: ${url}`, "warn");
}
}
pauseAll() {
this.activeDownloads.forEach((_, url) => this.pauseDownload(url));
this.emit(const_1.emitEvents.pausedAll, { message: const_1.emitMessages.pausedAll });
}
resumeAll() {
this.activeDownloads.forEach((_, url) => this.resumeDownload(url));
this.emit(const_1.emitEvents.resumedAll, { message: const_1.emitMessages.resumedAll });
}
async cancelDownload(url = this.currentDownloadableUrl.get(this.currentUrl)) {
const downloadData = this.activeDownloads.get(url);
if (downloadData) {
downloadData.stream.close();
await node_fs_1.default.promises.unlink(downloadData.stream.path);
this.activeDownloads.delete(url);
this.emit(const_1.emitEvents.cancel, {
message: const_1.emitMessages.cancel,
url,
});
this.logger(`Download canceled and file deleted for URL: ${url}`);
}
}
async cancelAll() {
const cancelPromises = Array.from(this.activeDownloads.keys()).map((url) => this.cancelDownload(url));
const results = await Promise.allSettled(cancelPromises);
results.forEach((result, index) => {
const url = Array.from(this.activeDownloads.keys())[index];
if (result.status === "rejected") {
this.logger(`Failed to cancel download for URL: ${url}. Error: ${result.reason}`, "error");
}
});
this.logger("All cancel operations processed.");
this.emit(const_1.emitEvents.cancel, {
message: const_1.emitMessages.cancelAll,
});
}
normalizeHeaders(headers) {
const normalizedHeaders = {};
if (headers instanceof Headers) {
headers.forEach((value, key) => {
normalizedHeaders[key] = value;
});
}
else if (Array.isArray(headers)) {
headers.forEach(([key, value]) => {
normalizedHeaders[key] = value;
});
}
else if (headers) {
Object.assign(normalizedHeaders, headers);
}
return normalizedHeaders;
}
async streamDownload(res, url, file, fileName, downloadedBytes, totalSize) {
const writeStream = node_fs_1.default.createWriteStream(file, {
flags: downloadedBytes > 0 ? "a" : "w",
});
this.activeDownloads.set(url, {
stream: writeStream,
downloaded: downloadedBytes,
totalSize,
paused: false,
});
this.emit(const_1.emitEvents.progress, {
fileName,
progress: ((downloadedBytes / totalSize) * 100).toFixed(2),
downloaded: downloadedBytes,
totalSize,
message: const_1.emitMessages.progress,
});
const startTime = Date.now();
for await (const chunk of res.body) {
try {
const downloadData = this.activeDownloads.get(url);
if (!downloadData || downloadData.paused) {
break;
}
writeStream.write(chunk);
downloadData.downloaded += chunk.length;
const progress = ((downloadData.downloaded / totalSize) * 100).toFixed(2);
const speed = (downloadData.downloaded /
((Date.now() - startTime) / 1000)).toFixed(2);
this.emit(const_1.emitEvents.progress, {
fileName,
progress,
downloaded: downloadData.downloaded,
totalSize,
speed,
message: const_1.emitMessages.progress,
});
}
catch (error) {
writeStream.close();
this.emit(const_1.emitEvents.error, {
url,
file,
error,
message: const_1.emitMessages.error,
});
this.logger(`Error during stream download: ${error}`, "error");
break;
}
}
writeStream.end();
const downloadData = this.activeDownloads.get(url);
if (downloadData && !downloadData.paused) {
this.activeDownloads.delete(url);
this.emit(const_1.emitEvents.complete, {
fileName,
url,
message: const_1.emitMessages.complete,
});
this.emit(const_1.emitEvents.finished, {
fileName,
url,
message: const_1.emitMessages.finished,
});
this.logger(`Download completed for ${url}. File saved to ${this.downloadFolder}`);
}
}
async downloadFile(url, fileName, downloadedBytes = 0) {
this.emit(const_1.emitEvents.start, {
message: const_1.emitMessages.start,
url,
fileName,
});
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), this.timeout);
try {
this.currentDownloadableUrl.set(this.currentUrl, url);
const file = node_path_1.default.join(this.downloadFolder, fileName);
if (this.onBeforeDownload) {
this.logger("Running before downloaded function");
await this.onBeforeDownload(url, file, fileName);
}
if (!node_fs_1.default.existsSync(file) || this.overWriteFile) {
this.logger(`Download started from ${url}`);
const headers = {
...this.normalizeHeaders(this.requestOptions?.headers),
};
if (downloadedBytes > 0) {
headers["Range"] = `bytes=${downloadedBytes}-`;
}
const res = await fetch(url, {
...this.requestOptions,
headers,
redirect: "follow",
signal: controller.signal,
});
if (res.status !== 200 && res.status !== 206 && res.status !== 304) {
const errMSg = `Download failed from ${url} with status ${res.status}`;
this.emit(const_1.emitEvents.error, {
message: const_1.emitMessages.error,
url,
file,
fileName,
error: { message: errMSg },
});
throw new Error(errMSg);
}
await node_fs_1.default.promises.mkdir(node_path_1.default.dirname(file), { recursive: true });
if (!this.stream) {
await node_fs_1.default.promises.writeFile(file, res.body, { flag: "w" });
this.emit(const_1.emitEvents.complete, {
fileName,
url,
message: const_1.emitMessages.complete,
});
this.emit(const_1.emitEvents.finished, {
fileName,
url,
message: const_1.emitMessages.finished,
});
this.logger(`File ${fileName} downloaded successfully. Downloaded from ${url}`);
}
else {
const totalSize = parseInt(res.headers.get("Content-Length") ?? "0", 10) +
downloadedBytes;
await this.streamDownload(res, url, file, fileName, downloadedBytes, totalSize);
}
}
else {
this.logger(`${fileName} already exists inside ${this.downloadFolder} folder`);
this.emit(const_1.emitEvents.exists, {
message: const_1.emitMessages.exists,
url,
fileName,
});
}
if (this.onAfterDownload) {
this.logger("Running after downloaded function");
await this.onAfterDownload(url, file, fileName);
}
return true;
}
catch (error) {
this.emit(const_1.emitEvents.error, {
message: const_1.emitMessages.error,
url,
fileName,
error,
});
this.logger(`Error downloading ${fileName} from ${url}. Error:- ${error}`, "error");
throw error;
}
finally {
clearTimeout(timeout);
}
}
isValidUrl(url) {
try {
new URL(url);
return true;
}
catch {
return false;
}
}
invalidUrlEmit(url) {
const msg = `${url} is not valid`;
this.logger(msg, "error");
this.emit(const_1.emitEvents.error, { message: msg, url });
}
enqueueDownloadTask(url, fileName, downloadedBytes = 0, priority = 1) {
if (this.isValidUrl(url)) {
this.logger(`${fileName} file downloading task added to Queue`);
const downloadableFile = {
id: `${Date.now()}-${fileName}`,
priority,
retries: 0,
action: () => this.downloadFile(url, fileName, downloadedBytes),
};
this.downloadQueue.enqueue(downloadableFile);
}
else {
this.invalidUrlEmit(url);
}
}
addThreadDownloadTask(url, fileName) {
if (this.isValidUrl(url)) {
const task = {
id: `${Date.now()}-${fileName}`,
url,
fileName,
options: this.options,
};
this.downloadThread.runThreadTask(task);
}
else {
this.invalidUrlEmit(url);
}
}
terminateThreads() {
this.downloadThread.terminateAll();
}
createFileName(url) {
const fileName = this.getFileName
? this.getFileName(url)
: url.split("/").pop() ?? "file";
return fileName;
}
async simpleDownload(urls, fileName) {
if (typeof urls === "string") {
const file = fileName ?? this.createFileName(urls);
await this.downloadFile(urls, file);
}
else {
const downloadChunks = (chunk) => Promise.all(chunk.map((url) => {
const fileName = this.createFileName(url);
return this.downloadFile(url, fileName);
}));
for (let i = 0; i < urls.length; i += this.concurrencyLimit) {
await downloadChunks(urls.slice(i, i + this.concurrencyLimit));
}
}
}
async queueDownload(urls) {
if (typeof urls === "string") {
const fileName = this.createFileName(urls);
this.enqueueDownloadTask(urls, fileName);
}
else {
for await (const url of urls) {
const fileName = this.createFileName(url);
this.enqueueDownloadTask(url, fileName);
}
}
}
async threadDownload(urls) {
if (typeof urls === "string") {
const fileName = this.createFileName(urls);
this.addThreadDownloadTask(urls, fileName);
}
else {
for await (const url of urls) {
const fileName = this.createFileName(url);
this.addThreadDownloadTask(url, fileName);
}
}
}
async download(urls, fileName) {
if (this.method === "simple") {
await this.simpleDownload(urls, fileName);
}
else if (this.method === "queue") {
await this.queueDownload(urls);
}
else if (this.method === "thread") {
await this.threadDownload(urls);
}
}
}
exports.default = DownloadManager;