@emodyz/node-downloader
Version:
Node file downloader
495 lines • 24.6 kB
JavaScript
"use strict";
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createDownloader = exports.Downloader = void 0;
var crypto = __importStar(require("crypto"));
var fs = __importStar(require("fs"));
var path = __importStar(require("path"));
var DownloaderState_1 = require("./enums/DownloaderState");
var node_downloader_helper_1 = require("node-downloader-helper");
var events_1 = require("events");
var Downloader = /** @class */ (function (_super) {
__extends(Downloader, _super);
function Downloader(downloaderOptions) {
if (downloaderOptions === void 0) { downloaderOptions = {}; }
var _this = _super.call(this) || this;
_this.simultaneusDownloads = 5;
_this.maxRetries = 3;
_this.checksumAlgo = 'sha256';
_this.state = DownloaderState_1.DownloaderState.STAND_BY;
_this.lastProgressDispatchedTime = 0;
_this.bytesToDownload = 0;
_this.bytesDownloaded = 0;
_this.bytesToCheck = 0;
_this.bytesChecked = 0;
_this.filesToDownload = 0;
_this.filesDownloaded = 0;
_this.downloadProgress = 0;
_this.checkProgress = 0;
_this.progress = 0;
_this.forceDownload = false;
_this.downloadersQueue = [];
_this.downloadersInProgress = [];
_this.downloaderOptions = downloaderOptions;
return _this;
}
Downloader.prototype.clean = function () {
if (this.state === DownloaderState_1.DownloaderState.DOWNLOADING) {
throw new Error('Cannot clean while downloading.');
}
this.downloadersQueue = [];
this.downloadersInProgress = [];
this.filesToDownload = 0;
this.filesDownloaded = 0;
this.bytesToDownload = 0;
this.bytesDownloaded = 0;
this.bytesToCheck = 0;
this.bytesChecked = 0;
this.downloadProgress = 0;
this.checkProgress = 0;
this.progress = 0;
this.lastProgressDispatchedTime = 0; // TODO see why is it null before
this.state = DownloaderState_1.DownloaderState.STAND_BY;
};
Downloader.prototype.checksumFile = function (queuedDownloader) {
var _this = this;
var filePath = queuedDownloader.filePath;
return new Promise(function (resolve, reject) {
var hash = crypto.createHash(_this.checksumAlgo);
var stream = fs.createReadStream(filePath);
var stats = queuedDownloader.downloader.getStats();
stream.on('error', function (err) { return reject(err); });
stream.on('data', function (chunk) {
if (_this.state === DownloaderState_1.DownloaderState.STOPED) {
stream.close();
resolve(null);
return;
}
hash.update(chunk);
_this.bytesChecked += chunk.length;
_this.dispatchProgress(stats);
});
stream.on('end', function () { return resolve(hash.digest('hex')); });
});
};
Downloader.prototype.dispatchProgress = function (stats) {
var currentTime = new Date().getTime();
var elaspsedTime = currentTime - this.lastProgressDispatchedTime;
this.downloadProgress = (this.bytesDownloaded * 100) / this.bytesToDownload;
this.checkProgress = (this.bytesChecked * 100) / this.bytesToCheck;
this.progress = (this.downloadProgress + this.checkProgress) / 2;
if (this.progress >= 100 || elaspsedTime > 1000) {
this.lastProgressDispatchedTime = currentTime;
this.emit('progress', __assign(__assign({}, stats), {
progressTotal: this.progress,
progressDownload: this.downloadProgress,
progressCheck: this.checkProgress,
}));
}
};
Downloader.prototype.isFileNeedUpdate = function (queuedDownloader) {
return __awaiter(this, void 0, void 0, function () {
var localChecksum, filePath, fileSize, checksum, checksumFilePath;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
localChecksum = null;
filePath = queuedDownloader.filePath;
fileSize = queuedDownloader.fileSize;
checksum = queuedDownloader.checksum;
checksumFilePath = "".concat(filePath, ".").concat(this.checksumAlgo);
if (!fs.existsSync(filePath)) {
return [2 /*return*/, true];
}
if (!fs.existsSync(checksumFilePath)) return [3 /*break*/, 1];
localChecksum = fs.readFileSync(checksumFilePath).toString();
this.bytesChecked += fileSize;
return [3 /*break*/, 4];
case 1:
if (!fs.existsSync(filePath)) return [3 /*break*/, 3];
return [4 /*yield*/, this.checksumFile(queuedDownloader)];
case 2:
localChecksum = _a.sent();
fs.writeFileSync(checksumFilePath, localChecksum);
return [3 /*break*/, 4];
case 3: return [2 /*return*/, true];
case 4:
if (localChecksum !== checksum) {
this.bytesToCheck += fileSize;
}
return [2 /*return*/, localChecksum !== checksum];
}
});
});
};
Downloader.prototype.startDownloader = function (queuedDownloader) {
return __awaiter(this, void 0, void 0, function () {
var lastDownloadedSize, _a;
var _this = this;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
lastDownloadedSize = 0;
this.downloadersInProgress.push(queuedDownloader);
queuedDownloader.downloader.on('progress.throttled', function (stats) {
var totalDownloaded = stats.downloaded - lastDownloadedSize;
_this.bytesDownloaded += totalDownloaded;
if (!queuedDownloader.checksum) {
_this.bytesChecked += totalDownloaded;
}
lastDownloadedSize = stats.downloaded;
_this.dispatchProgress(stats);
});
queuedDownloader.downloader.on('end', function (downloadInfos) { return __awaiter(_this, void 0, void 0, function () {
var checksum;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (this.state === DownloaderState_1.DownloaderState.STOPED) {
return [2 /*return*/];
}
if (!queuedDownloader.checksum) return [3 /*break*/, 4];
return [4 /*yield*/, this.checksumFile(queuedDownloader)];
case 1:
checksum = _a.sent();
// @ts-ignore
if (checksum === null && this.state === DownloaderState_1.DownloaderState.STOPED) {
this.downloaderStopped(queuedDownloader);
return [2 /*return*/];
}
if (!(checksum !== queuedDownloader.checksum)) return [3 /*break*/, 3];
if (!queuedDownloader.retryCount) {
queuedDownloader.retryCount = 0;
}
if (queuedDownloader.retryCount >= this.maxRetries) {
this.emit('error', {
message: 'Max retries attempts.',
file: downloadInfos.fileName,
path: downloadInfos.filePath,
checksum: queuedDownloader.checksum,
fileChecksum: checksum,
});
return [2 /*return*/];
}
this.bytesToCheck += queuedDownloader.fileSize;
this.bytesToDownload += queuedDownloader.fileSize;
queuedDownloader.retryCount++;
return [4 /*yield*/, queuedDownloader.downloader.start()];
case 2:
_a.sent();
return [2 /*return*/];
case 3:
if (checksum) {
fs.writeFileSync("".concat(downloadInfos.filePath, ".").concat(this.checksumAlgo), checksum);
}
_a.label = 4;
case 4:
this.downloaderCompleted(queuedDownloader);
return [2 /*return*/];
}
});
}); });
queuedDownloader.downloader.on('stop', function () { return __awaiter(_this, void 0, void 0, function () {
return __generator(this, function (_a) {
this.downloaderStopped(queuedDownloader);
return [2 /*return*/];
});
}); });
queuedDownloader.downloader.on('error', function (error) { return __awaiter(_this, void 0, void 0, function () {
return __generator(this, function (_a) {
this.stop();
this.emit('error', error);
return [2 /*return*/];
});
}); });
_a = !this.forceDownload;
if (!_a) return [3 /*break*/, 2];
return [4 /*yield*/, this.isFileNeedUpdate(queuedDownloader)];
case 1:
_a = !(_b.sent());
_b.label = 2;
case 2:
if (_a) {
this.downloaderCompleted(queuedDownloader, true);
return [2 /*return*/];
}
if (!fs.existsSync(queuedDownloader.installPath)) {
fs.mkdirSync(queuedDownloader.installPath, {
recursive: true,
});
}
if (fs.existsSync(queuedDownloader.filePath)) {
fs.unlinkSync(queuedDownloader.filePath);
}
if (this.state === DownloaderState_1.DownloaderState.PAUSED) {
return [2 /*return*/];
}
if (!(this.state === DownloaderState_1.DownloaderState.STOPED)) return [3 /*break*/, 4];
return [4 /*yield*/, queuedDownloader.downloader.stop()];
case 3:
_b.sent();
return [2 /*return*/];
case 4: return [4 /*yield*/, queuedDownloader.downloader.start()];
case 5:
_b.sent();
return [2 /*return*/];
}
});
});
};
Downloader.prototype.downloaderStopped = function (queuedDownloader) {
this.removeDownloaderFromQueue(queuedDownloader);
if (this.downloadersQueue.length === 0 &&
this.downloadersInProgress.length === 0) {
this.clean();
this.emit('stop', {});
return;
}
this.startNextDownloader();
};
Downloader.prototype.downloaderCompleted = function (queuedDownloader, pass) {
if (pass === void 0) { pass = false; }
this.filesDownloaded++;
var stats = queuedDownloader.downloader.getStats();
if (pass) {
this.bytesDownloaded += queuedDownloader.fileSize;
}
this.dispatchProgress(stats);
if (this.progress >= 100) {
this.emit('end', {});
return;
}
this.removeDownloaderFromQueue(queuedDownloader);
this.startNextDownloader();
};
Downloader.prototype.startNextDownloader = function () {
if (this.downloadersInProgress.length >= this.simultaneusDownloads) {
return false;
}
var downloader = this.downloadersQueue.shift();
if (downloader) {
this.startDownloader(downloader);
return true;
}
return false;
};
Downloader.prototype.removeDownloaderFromQueue = function (queuedDownloader) {
var index = this.downloadersInProgress.indexOf(queuedDownloader);
if (index > -1) {
this.downloadersInProgress.splice(index, 1);
return true;
}
return false;
};
Downloader.prototype.addFile = function (fileUrl, installPath, fileName, checksum) {
if (fileName === void 0) { fileName = null; }
if (checksum === void 0) { checksum = null; }
if (this.state !== DownloaderState_1.DownloaderState.STAND_BY) {
throw new Error('Cannot add file while downloading.');
}
fileName = path.normalize(fileName || path.parse(fileUrl).base);
installPath = path.normalize(installPath);
var downloader = new node_downloader_helper_1.DownloaderHelper(fileUrl, installPath, __assign({
fileName: fileName,
forceResume: true,
override: true,
retry: { maxRetries: 3, delay: 3000 },
}, this.downloaderOptions));
var filePath = path.resolve(installPath, fileName);
var queuedDownloader = {
downloader: downloader,
checksum: checksum,
fileName: fileName,
filePath: filePath,
installPath: path.dirname(filePath),
fileSize: 0,
retryCount: 0
};
this.downloadersQueue.push(queuedDownloader);
this.filesToDownload++;
return this;
};
Downloader.prototype.start = function (forceDownload) {
if (forceDownload === void 0) { forceDownload = false; }
return __awaiter(this, void 0, void 0, function () {
var _i, _a, queuedDownloader, stats, fileSize;
var _this = this;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
if (this.state !== DownloaderState_1.DownloaderState.STAND_BY) {
this.emit('error', 'Download already in progress.');
throw new Error('Download already in progress.');
}
if (this.downloadersQueue.length === 0) {
this.emit('error', 'Any files in the queue list.');
throw new Error('Any files in the queue list.');
}
this.state = DownloaderState_1.DownloaderState.DOWNLOADING;
this.forceDownload = forceDownload;
this.filesToDownload = this.downloadersQueue.length;
this.lastProgressDispatchedTime = new Date().getTime();
_i = 0, _a = this.downloadersQueue;
_b.label = 1;
case 1:
if (!(_i < _a.length)) return [3 /*break*/, 4];
queuedDownloader = _a[_i];
return [4 /*yield*/, queuedDownloader.downloader.getTotalSize()];
case 2:
stats = _b.sent();
if (stats.total === null) {
throw new Error('Cannot get file size.');
}
fileSize = stats.total;
this.bytesToCheck += fileSize;
this.bytesToDownload += fileSize;
queuedDownloader.fileSize = fileSize;
_b.label = 3;
case 3:
_i++;
return [3 /*break*/, 1];
case 4:
this.downloadersQueue.forEach(function () {
if (_this.startNextDownloader() == false) {
return;
}
});
return [2 /*return*/];
}
});
});
};
Downloader.prototype.stop = function () {
this.state = DownloaderState_1.DownloaderState.STOPED;
this.downloadersInProgress.forEach(function (queuedDownloader) {
queuedDownloader.downloader.stop();
});
};
Downloader.prototype.pause = function () {
this.state = DownloaderState_1.DownloaderState.PAUSED;
this.downloadersInProgress.forEach(function (queuedDownloader) {
queuedDownloader.downloader.pause();
});
};
Downloader.prototype.resume = function () {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
this.state = DownloaderState_1.DownloaderState.DOWNLOADING;
if (!(this.downloadersInProgress.length > 0)) return [3 /*break*/, 1];
this.downloadersInProgress.forEach(function (queuedDownloader) {
queuedDownloader.downloader.resume();
});
return [3 /*break*/, 3];
case 1: return [4 /*yield*/, this.startNextDownloader()];
case 2:
_a.sent();
_a.label = 3;
case 3: return [2 /*return*/];
}
});
});
};
Downloader.prototype.stats = function () {
return {
files: this.filesToDownload,
fileDownloaded: this.filesDownloaded,
progress: this.progress,
progressDownload: this.downloadProgress,
progressCheck: this.checkProgress,
};
};
return Downloader;
}(events_1.EventEmitter));
exports.Downloader = Downloader;
function createDownloader(downloaderOptions) {
if (downloaderOptions === void 0) { downloaderOptions = {}; }
return new Downloader(downloaderOptions);
}
exports.createDownloader = createDownloader;
//# sourceMappingURL=Downloader.js.map