@devteks/downloader
Version:
Simple node.js file downloader
646 lines • 24.5 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Downloader = void 0;
const tslib_1 = require("tslib");
const promises_1 = require("fs/promises");
const fs_1 = require("fs");
const url_1 = require("url");
const path_1 = require("path");
const events_1 = require("events");
const Https = require("https");
const Http = require("http");
const types_1 = require("./types");
const utils_1 = require("./utils");
class Downloader extends events_1.EventEmitter {
/**
* Creates an instance of Downloader.
* @param {String} url
* @param {String} destFolder
* @param {Object} [options={}]
* @memberof Downloader
*/
constructor(url, destination, options) {
super();
this._state = types_1.DownloadState.IDLE;
this._opts = Object.assign({}, DEFAULT_OPTIONS);
this._pipes = [];
this._total = 0;
this._downloaded = 0;
this._progress = 0;
this._retryCount = 0;
this._request = null;
this._response = null;
this._isResumed = false;
this._isResumable = false;
this._isRedirected = false;
this._fileName = '';
this._filePath = '';
this._statsEstimate = { time: 0, bytes: 0, prevBytes: 0, throttleTime: 0 };
if (!(0, utils_1.validateParams)(url, destination)) {
return;
}
this._url = this._requestUrl = url;
this._destination = destination;
this.updateOptions(options);
}
/**
* request url
*
* @returns {String}
* @memberof Downloader
*/
get requestUrl() { return this._requestUrl; }
/**
* Where the download will be saved
*
* @returns {String}
* @memberof Downloader
*/
get downloadPath() { return this._filePath; }
/**
* Indicates if the download can be resumable (available after the start phase)
*
* @returns {Boolean}
* @memberof Downloader
*/
get isResumable() { return this._isResumable; }
/**
* Return the current download state
*
* @returns {DownloadState}
* @memberof Downloader
*/
get state() { return this._state; }
/**
* Current download progress stats
*
* @returns {DownloadStats}
* @memberof Downloader
*/
get stats() {
var _a;
return {
total: (_a = this._total) !== null && _a !== void 0 ? _a : 0,
name: this._fileName,
downloaded: this._downloaded,
progress: this._progress,
speed: this._statsEstimate.bytes,
};
}
on(event, listener) {
super.on(event, listener);
return this;
}
/**
* @returns {Promise<boolean>}
* @memberof Downloader
*/
start() {
return new Promise((resolve, reject) => {
this._resolve = resolve;
this._reject = reject;
this._start();
});
}
/**
* @returns {Promise<boolean>}
* @memberof Downloader
*/
pause() {
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
this._abort();
if (this._response) {
this._response.unpipe();
this._pipes.forEach((pipe) => pipe.stream.unpipe());
}
if (this._fileStream) {
this._fileStream.removeAllListeners();
yield (0, utils_1.closeStream)(this._fileStream, false);
}
this._setState(types_1.DownloadState.PAUSED);
this.emit(types_1.Events.pause);
return true;
});
}
/**
* @returns {void}
* @memberof Downloader
*/
resume() {
var _a;
var _b;
this._setState(types_1.DownloadState.RESUMED);
(_a = (_b = this._options).headers) !== null && _a !== void 0 ? _a : (_b.headers = {});
if (this._isResumable) {
this._isResumed = true;
this._options['headers']['range'] = 'bytes=' + this._downloaded + '-';
}
this.emit(types_1.Events.resume, this._isResumed);
this._start();
}
/**
* @returns {Promise<boolean>}
* @memberof Downloader
*/
stop() {
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
this._abort();
if (this._fileStream) {
yield (0, utils_1.closeStream)(this._fileStream, false);
}
if (this._opts.removeOnStop) {
if (yield (0, utils_1.canAccessFile)(this._filePath)) {
try {
yield (0, promises_1.unlink)(this._filePath);
}
catch (ex) {
this._setState(types_1.DownloadState.FAILED);
this.emit(types_1.Events.error, ex);
throw ex;
}
}
}
if (this._resolve) {
const resolve = this._resolve;
this._resolve = this._reject = undefined;
resolve(true);
}
this._setState(types_1.DownloadState.STOPPED);
this.emit(types_1.Events.stop);
return true;
});
}
/**
* Add pipes to the pipe list that will be applied later when the download starts
* @url https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options
* @param {stream.Readable} stream https://nodejs.org/api/stream.html#stream_class_stream_readable
* @param {Object} [options=null]
* @returns {stream.Readable}
* @memberof Downloader
*/
pipe(stream, options = null) {
this._pipes.push({ stream, options });
return stream;
}
/**
* Unpipe a stream
*
* @param {Stream} [stream=null]
* @returns
* @memberof Downloader
*/
unpipe(stream) {
// https://nodejs.org/api/stream.html#stream_readable_unpipe_destination
const pipe = this._pipes.find((p) => p.stream === stream);
if (pipe) {
const st = stream;
if (this._response) {
this._response.unpipe(st);
}
else {
st.unpipe();
}
this._pipes = this._pipes.filter((x) => x.stream !== stream);
}
return this;
}
/**
* Unpipe all streams
*
* @returns void
* @memberof Downloader
*/
unpipeAll() {
const _unpipe = (st) => {
if (this._response) {
this._response.unpipe(st);
}
else {
st.unpipe();
}
};
this._pipes.forEach((p) => _unpipe(p.stream));
this._pipes = [];
return this;
}
/**
* Updates the options, can be use on pause/resume events
*
* @param {Object} [options={}]
* @memberof Downloader
*/
updateOptions(options = {}) {
this._opts = Object.assign({}, this._opts, options !== null && options !== void 0 ? options : {});
this._headers = this._opts.headers;
// validate the progressThrottle, if invalid, use the default
if (typeof this._opts.progressThrottle !== 'number' || this._opts.progressThrottle < 0) {
this._opts.progressThrottle = DEFAULT_OPTIONS.progressThrottle;
}
this._options = (0, utils_1.getRequestOptions)(this._opts.method, this._url, this._opts.headers);
this._initProtocol(this._url);
}
/**
* Gets the total file size from the server
*
* @returns {Promise<{name:string, total:number|null}>}
* @memberof Downloader
*/
getTotalSize() {
const options = (0, utils_1.getRequestOptions)('HEAD', this._url, this._headers);
return new Promise((resolve, reject) => {
const request = this._protocol.request(options, (response) => {
var _a;
const redirectedURL = (0, utils_1.getRedirectUrl)(response, this._url);
if (redirectedURL) {
const options = (0, utils_1.getRequestOptions)('HEAD', redirectedURL, this._headers);
const request2 = this._protocol.request(options, (response2) => {
var _a;
if (response2.statusCode !== 200) {
return reject(new Error(`Response status was ${response2.statusCode}`));
}
resolve({
name: this._getFileNameFromHeaders(response2.headers, response2),
total: (_a = (0, utils_1.getContentLength)(response2)) !== null && _a !== void 0 ? _a : 0,
});
});
request2.end();
return;
}
if (response.statusCode !== 200) {
return reject(new Error(`Response status was ${response.statusCode}`));
}
resolve({
name: this._getFileNameFromHeaders(response.headers, response),
total: (_a = (0, utils_1.getContentLength)(response)) !== null && _a !== void 0 ? _a : 0,
});
});
request.end();
});
}
_start() {
if (!this._isRedirected && this._state !== types_1.DownloadState.RESUMED) {
this.emit(types_1.Events.start);
this._setState(types_1.DownloadState.STARTED);
}
if (!this._resolve)
return;
// Start the Download
this._response = null;
this._request = this._downloadRequest();
// Error Handling
this._request.on('error', (err) => this._onError(err));
this._request.on('timeout', () => this._onTimeout());
this._request.on('uncaughtException', (err) => this._onError(err, true));
if (this._opts.body) {
this._request.write(this._opts.body);
}
this._request.end();
}
_downloadRequest() {
return this._protocol.request(this._options, (response) => {
this._response = response;
//Stats
if (!this._isResumed) {
this._total = (0, utils_1.getContentLength)(response);
this._resetStats();
}
const redirectedURL = (0, utils_1.getRedirectUrl)(response, this._url);
if (redirectedURL) {
this._isRedirected = true;
this._initProtocol(redirectedURL);
return this._start();
}
// check if response wans't a success
if (response.statusCode !== 200 && response.statusCode !== 206) {
const error = new Error(`Response status was ${response.statusCode}`);
error.status = response.statusCode || 0;
error.body = response.body || '';
this._setState(types_1.DownloadState.FAILED);
this.emit(types_1.Events.error, error);
return this._reject(error);
}
if (this._opts.forceResume) {
this._isResumable = true;
}
else if (response.headers.hasOwnProperty('accept-ranges') &&
response.headers['accept-ranges'] !== 'none') {
this._isResumable = true;
}
this._startDownload(response);
});
}
_startDownload(response) {
var _a;
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
if (!this._isResumed) {
const _fileName = this._getFileNameFromHeaders(response.headers);
this._filePath = this._getFilePath(_fileName);
this._fileName = (_a = this._filePath.split(path_1.sep).pop()) !== null && _a !== void 0 ? _a : '';
if ((0, fs_1.existsSync)(this._filePath)) {
const downloadedSize = yield this._getFilesizeInBytes(this._filePath);
const totalSize = this._total ? this._total : 0;
const override = this._opts.override;
if (typeof override === 'object' &&
override.skip &&
(override.skipSmaller || downloadedSize >= totalSize)) {
this.emit(types_1.Events.skip, {
totalSize: this._total,
fileName: this._fileName,
filePath: this._filePath,
downloadedSize: downloadedSize,
});
this._setState(types_1.DownloadState.SKIPPED);
return this._resolve(true);
}
}
this._fileStream = (0, fs_1.createWriteStream)(this._filePath, {});
}
else {
this._fileStream = (0, fs_1.createWriteStream)(this._filePath, { flags: 'a' });
}
// Start Downloading
this.emit(types_1.Events.download, {
fileName: this._fileName,
filePath: this._filePath,
totalSize: this._total,
isResumed: this._isResumed,
downloadedSize: this._downloaded,
});
this._retryCount = 0;
this._isResumed = false;
this._isRedirected = false;
this._setState(types_1.DownloadState.DOWNLOADING);
this._statsEstimate.time = this._statsEstimate.throttleTime = Date.now();
// Add externals pipe
let readable = response;
readable.on('data', (chunk) => this._calculateStats(chunk.length));
this._pipes.forEach((pipe) => {
readable.pipe(pipe.stream, pipe.options);
readable = pipe.stream;
});
readable.pipe(this._fileStream);
readable.on('error', (err) => this._onError(err));
this._fileStream.on('finish', () => this._onFinished());
this._fileStream.on('error', (err) => this._onError(err));
});
}
_onFinished() {
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
try {
yield (0, utils_1.closeStream)(this._fileStream);
if ((0, utils_1.isFinishedState)(this._state)) {
this._setState(types_1.DownloadState.FINISHED);
this._pipes = [];
this.emit(types_1.Events.end, {
fileName: this._fileName,
filePath: this._filePath,
totalSize: this._total,
incomplete: !this._total ? false : this._downloaded !== this._total,
onDiskSize: yield this._getFilesizeInBytes(this._filePath),
downloadedSize: this._downloaded,
});
}
return this._resolve(this._downloaded === this._total);
}
catch (ex) {
this._reject(ex);
}
});
}
_onError(error, abortReq = false) {
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
this._pipes = [];
if (abortReq)
this._abort();
if (this._state === types_1.DownloadState.STOPPED || this._state === types_1.DownloadState.FAILED)
return;
const emitError = () => (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
yield this._removeFile();
this._setState(types_1.DownloadState.FAILED);
this.emit(types_1.Events.error, error);
this._reject(error);
});
if (!this._opts.retry) {
yield emitError();
}
try {
yield this._retry(error);
}
catch (ex) {
error = ex;
}
yield emitError();
});
}
_retry(err = null) {
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
const retry = this._opts.retry;
if (!retry) {
throw (err !== null && err !== void 0 ? err : UNKNOWN_ERROR);
}
if (typeof retry !== 'object' ||
!retry.hasOwnProperty('maxRetries') ||
!retry.hasOwnProperty('delay')) {
throw new Error('wrong retry options');
}
// reached the maximum retries
if (this._retryCount >= retry.maxRetries) {
throw err !== null && err !== void 0 ? err : new Error('reached the maximum retries');
}
this._retryCount++;
this._setState(types_1.DownloadState.RETRY);
this.emit(types_1.Events.retry, {
retryCount: this._retryCount,
maxRetries: retry.maxRetries,
delay: retry.delay,
error: err,
});
yield (0, utils_1.delay)(retry.delay);
if (this._downloaded > 0) {
this.resume();
}
else {
this._start();
}
});
}
_onTimeout() {
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
const reject = this._reject;
this._abort();
if (!this._opts.retry) {
yield this._removeFile();
this._setState(types_1.DownloadState.FAILED);
this.emit(types_1.Events.timeout);
reject(TIMEOUT_ERROR);
}
let err = null;
try {
yield this._retry(TIMEOUT_ERROR);
}
catch (ex) {
err = ex;
}
yield this._removeFile();
if (err) {
reject(err);
}
else {
this.emit(types_1.Events.timeout);
reject(TIMEOUT_ERROR);
}
});
}
_resetStats() {
this._retryCount = 0;
this._downloaded = 0;
this._progress = 0;
this._statsEstimate = {
time: 0,
bytes: 0,
prevBytes: 0,
throttleTime: 0,
};
}
_getFilePath(fileName) {
var _a, _b;
const currentPath = (0, path_1.join)(this._destination, fileName);
let filePath = currentPath;
if (!this._opts.override && this._state !== types_1.DownloadState.RESUMED) {
filePath = (0, utils_1.getUniqueFileName)(filePath);
if (currentPath !== filePath) {
const renamedData = {
path: filePath,
fileName: (_a = filePath.split(path_1.sep).pop()) !== null && _a !== void 0 ? _a : '',
prevPath: currentPath,
prevFileName: (_b = currentPath.split(path_1.sep).pop()) !== null && _b !== void 0 ? _b : '',
};
this.emit(types_1.Events.renamed, renamedData);
}
}
return filePath;
}
_getFileNameFromHeaders(headers, response) {
var _a;
let fileName = (0, utils_1.getFilenameFromContentDisposition)(headers);
if (!fileName) {
const baseName = (0, path_1.basename)(new url_1.URL(this._requestUrl).pathname);
if (baseName.length > 0) {
fileName = baseName;
}
else {
fileName = `${new url_1.URL(this._requestUrl).hostname}.html`;
}
}
const fileDef = this._opts.fileName;
if (fileDef) {
if (typeof fileDef === 'string')
return fileDef;
if (typeof fileDef === 'function') {
return fileDef(fileName, (0, path_1.join)(this._destination, fileName), (_a = (response ? response : this._response)) === null || _a === void 0 ? void 0 : _a.headers['content-type']);
}
fileName = (0, utils_1.getFileNameFromOptions)(fileName, fileDef);
}
// remove any trailing '.'
return fileName.replace(/\.*$/, '');
}
_calculateStats(receivedBytes) {
var _a;
if (!receivedBytes)
return;
const currentTime = Date.now();
const elaspsedTime = currentTime - this._statsEstimate.time;
const throttleElapseTime = currentTime - this._statsEstimate.throttleTime;
const total = this._total || 0;
this._downloaded += receivedBytes;
this._progress = total === 0 ? 0 : (this._downloaded / total) * 100;
// Calculate the speed every second or if finished
if (this._downloaded === total || elaspsedTime > 1000) {
this._statsEstimate.time = currentTime;
this._statsEstimate.bytes = this._downloaded - this._statsEstimate.prevBytes;
this._statsEstimate.prevBytes = this._downloaded;
}
const progressThrottle = (_a = this._opts.progressThrottle) !== null && _a !== void 0 ? _a : 0;
const stats = this.stats;
if (this._downloaded === total || throttleElapseTime > progressThrottle) {
this._statsEstimate.throttleTime = currentTime;
this.emit(types_1.Events.progressThrottled, stats);
}
// emit the progress
this.emit(types_1.Events.progress, stats);
}
_setState(state) {
this._state = state;
this.emit(types_1.Events.stateChanged, this._state);
}
_getFilesizeInBytes(filePath) {
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
try {
const s = yield (0, promises_1.stat)(filePath);
return s.size || 0;
}
catch (ex) { }
return 0;
});
}
_initProtocol(url) {
const defaultOpts = (0, utils_1.getRequestOptions)(this._opts.method, url, this._headers);
this._requestUrl = url;
if (url.indexOf('https://') > -1) {
this._protocol = Https;
this._options = Object.assign({}, defaultOpts, this._opts.httpsRequestOptions);
}
else {
this._protocol = Http;
this._options = Object.assign({}, defaultOpts, this._opts.httpRequestOptions);
}
}
_removeFile() {
return (0, tslib_1.__awaiter)(this, void 0, void 0, function* () {
if (!this._fileStream)
return;
yield (0, utils_1.closeStream)(this._fileStream, false);
if (this._opts.removeOnFail) {
try {
yield (0, promises_1.unlink)(this._filePath);
}
catch (ex) { }
}
});
}
_abort() {
try {
if (this._response) {
this._response.destroy();
}
if (this._request) {
if (this._request.destroy) {
// node => v13.14.*
this._request.destroy();
}
else {
this._request.abort();
}
}
}
catch (ex) { }
}
}
exports.Downloader = Downloader;
const DEFAULT_OPTIONS = {
body: null,
method: 'GET',
headers: {},
fileName: '',
retry: false,
forceResume: false,
removeOnStop: true,
removeOnFail: true,
override: false,
progressThrottle: 1000,
httpRequestOptions: {},
httpsRequestOptions: {},
};
const UNKNOWN_ERROR = new Error('Unknown Error');
const TIMEOUT_ERROR = new Error('timeout');
//# sourceMappingURL=downloader.js.map