ledown/dist/main.js

A library for nodejs to download http large file in pauseable and resumable way

'use strict';

function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'default' in ex) ? ex['default'] : ex; }

var fs = require('fs');
var path = _interopDefault(require('path'));
var EventEmitter = _interopDefault(require('events'));
var util = require('util');
var uuid = _interopDefault(require('uuid'));
var http = require('http');
var url = require('url');

var asyncGenerator = function () {
  function AwaitValue(value) {
    this.value = value;
  }

  function AsyncGenerator(gen) {
    var front, back;

    function send(key, arg) {
      return new Promise(function (resolve, reject) {
        var request$$1 = {
          key: key,
          arg: arg,
          resolve: resolve,
          reject: reject,
          next: null
        };

        if (back) {
          back = back.next = request$$1;
        } else {
          front = back = request$$1;
          resume(key, arg);
        }
      });
    }

    function resume(key, arg) {
      try {
        var result = gen[key](arg);
        var value = result.value;

        if (value instanceof AwaitValue) {
          Promise.resolve(value.value).then(function (arg) {
            resume("next", arg);
          }, function (arg) {
            resume("throw", arg);
          });
        } else {
          settle(result.done ? "return" : "normal", result.value);
        }
      } catch (err) {
        settle("throw", err);
      }
    }

    function settle(type, value) {
      switch (type) {
        case "return":
          front.resolve({
            value: value,
            done: true
          });
          break;

        case "throw":
          front.reject(value);
          break;

        default:
          front.resolve({
            value: value,
            done: false
          });
          break;
      }

      front = front.next;

      if (front) {
        resume(front.key, front.arg);
      } else {
        back = null;
      }
    }

    this._invoke = send;

    if (typeof gen.return !== "function") {
      this.return = undefined;
    }
  }

  if (typeof Symbol === "function" && Symbol.asyncIterator) {
    AsyncGenerator.prototype[Symbol.asyncIterator] = function () {
      return this;
    };
  }

  AsyncGenerator.prototype.next = function (arg) {
    return this._invoke("next", arg);
  };

  AsyncGenerator.prototype.throw = function (arg) {
    return this._invoke("throw", arg);
  };

  AsyncGenerator.prototype.return = function (arg) {
    return this._invoke("return", arg);
  };

  return {
    wrap: function (fn) {
      return function () {
        return new AsyncGenerator(fn.apply(this, arguments));
      };
    },
    await: function (value) {
      return new AwaitValue(value);
    }
  };
}();



var asyncToGenerator = function (fn) {
  return function () {
    var gen = fn.apply(this, arguments);
    return new Promise(function (resolve, reject) {
      function step(key, arg) {
        try {
          var info = gen[key](arg);
          var value = info.value;
        } catch (error) {
          reject(error);
          return;
        }

        if (info.done) {
          resolve(value);
        } else {
          return Promise.resolve(value).then(function (value) {
            step("next", value);
          }, function (err) {
            step("throw", err);
          });
        }
      }

      return step("next");
    });
  };
};











var _extends = Object.assign || function (target) {
  for (var i = 1; i < arguments.length; i++) {
    var source = arguments[i];

    for (var key in source) {
      if (Object.prototype.hasOwnProperty.call(source, key)) {
        target[key] = source[key];
      }
    }
  }

  return target;
};

function fetch(url$$1, options) {
  return new Promise((resolve, reject) => {
    const { path: path$$1, port, protocol, hostname } = url.parse(url$$1);
    const req = http.request(_extends({}, options, {
      port,
      hostname,
      protocol,
      path: path$$1
    }), resolve);
    req.on('error', reject);
    req.end();
  });
}

const cloneByteRange = data => {
  if (Array.isArray(data)) {
    return data.map(cloneByteRange);
  }
  return data;
};

class ByteSheet {

  /** z
   * @property indicates the range of bytes which are being downloading
   */
  constructor(arg = 0) {
    this.remaining = [];
    this.downloading = [];
    this.completed = [];

    if (typeof arg === 'number') {
      this.remaining.push([0, arg]);
    } else {
      const clone = cloneByteRange(arg);
      this.remaining = clone[0];
      this.downloading = [];
      this.completed = clone[1];
    }
  }

  /**
   * @property indicates the range of bytes which are finished downloading
   */

  /**
   * @property indicates the range of bytes which remain to be download
   */


  get completedByteCount() {
    return this.getByteCountOf(this.completed);
  }

  get downloadingByteCount() {
    return this.getByteCountOf(this.downloading);
  }

  get remainingByteCount() {
    return this.getByteCountOf(this.remaining);
  }

  getByteCountOf(section) {
    return section.reduce((bytes, item) => {
      return bytes + item[1] - item[0];
    }, 0);
  }

  toJSONObject() {
    const remaining = cloneByteRange(this.remaining);
    const downloading = cloneByteRange(this.downloading);
    const completed = cloneByteRange(this.completed);
    return [this.resolveRange([...remaining, ...downloading]), completed];
  }

  /**
   * resolve an array of byte range
   */
  resolveRange(ranges) {
    return ranges.filter(range => range.length > 0 && range[1] > range[0]).sort((prev, next) => prev[0] - next[0]).reduce((sum, range) => {
      const last = sum[sum.length - 1];
      if (last && last[1] >= range[0]) {
        last[1] = last[1] > range[1] ? last[1] : range[1];
      } else {
        sum.push(range);
      }
      return sum;
    }, []);
  }

  /**
   * take range of size from remaining byte range
   * @param {number} size
   */
  allocRangeToDownload(size) {
    const firstRange = this.remaining[0];
    if (!firstRange) return null;

    const start = firstRange[0];
    const end = Math.min(start + size, firstRange[1]);
    const range = [start, end];

    this.remaining = this.cutSection(range, this.remaining);
    this.downloading = this.resolveRange([...this.downloading, range]);
    return range;
  }

  setRangeAsCompleted(range) {
    this.downloading = this.cutSection(range, this.downloading);
    this.completed = this.resolveRange([...this.completed, range]);
  }

  cutSection(cut, section) {
    return this.resolveRange(section.reduce((sec, item) => {
      let range = [...item];
      sec.push(range);
      // Range: |--------|
      // Cut:     |---|
      if (cut[0] > range[0] && cut[1] < range[1]) {
        sec.push([cut[1], range[1]]);
        range[1] = cut[0];
      }

      // Range: |--------|
      // Cut: |----|
      // Cut: |------------|
      if (cut[0] <= range[0] && cut[1] > range[0]) {
        range[0] = cut[1];
      }

      // Range: |--------|
      // Cut:         |----|
      // Cut:  |-------------|
      if (cut[1] >= range[1] && cut[0] < range[1]) {
        range[1] = cut[0];
      }
      return sec;
    }, []));
  }

  resetDownloading() {
    this.remaining = this.resolveRange([...this.remaining, ...this.downloading]);
    this.downloading = [];
  }
}

class TaskMeta {

  constructor(param) {
    this.url = '';
    this.createDate = null;
    this.mimeType = '';
    this.downloadPath = '';
    this.totalBytes = 0;
    this.resumable = false;
    this.byteSheet = null;

    Object.assign(this, param);
    const { createDate, byteSheet } = param;
    if (createDate) {
      this.createDate = new Date(createDate);
    } else {
      this.createDate = new Date();
    }

    if (byteSheet && this.resumable) {
      this.byteSheet = new ByteSheet(byteSheet);
    } else {
      this.byteSheet = new ByteSheet(param.totalBytes);
    }
  }

  allocRange(size) {
    if (!this.resumable) {
      size = this.totalBytes;
    }
    return this.byteSheet.allocRangeToDownload(size);
  }

  completeRange(range) {
    return this.byteSheet.setRangeAsCompleted(range);
  }

  resetDownloadingRange() {
    return this.byteSheet.resetDownloading();
  }

  get progress() {
    return this.completedByteCount / this.totalBytes;
  }

  get remainingByteCount() {
    return this.byteSheet.remainingByteCount;
  }

  get completedByteCount() {
    return this.byteSheet.completedByteCount;
  }

  get downloadingByteCount() {
    return this.byteSheet.downloadingByteCount;
  }

  toJSONObject() {
    return {
      url: this.url,
      refererUrl: this.refererUrl,
      mimeType: this.mimeType,
      resumable: this.resumable,
      downloadPath: this.downloadPath,
      totalBytes: this.totalBytes,
      createDate: this.createDate.valueOf(),
      byteSheet: this.byteSheet.toJSONObject()
    };
  }
}

const writeAsync = util.promisify(fs.write);

class TaskThread extends EventEmitter {

  constructor(taskMeta, options) {
    var _this;

    _this = super();
    this.isRunning = false;
    this.pendingWrite = 0;
    this.executeRequest = asyncToGenerator(function* () {
      if (!_this.isRunning) return;

      const range = _this.taskMeta.allocRange(_this.options.windowSize);
      if (!range) {
        return _this.emitEnd();
      }
      try {
        _this.currentPosition = range[0];
        _this.currentRequest = yield fetch(_this.taskMeta.url, {
          method: 'GET',
          headers: {
            Range: `bytes=${range[0]}-${range[1] - 1}`
          }
        });
        if (_this.currentRequest.statusCode >= 400) {
          throw new Error(Task.ERROR.ERR_NETWORK);
        }
        _this.currentRequest.on('end', _this.handleSingleRequestEnd);
        _this.currentRequest.on('data', _this.handleData);
      } catch (e) {
        _this.emitError(e);
      }
    });
    this.taskMeta = taskMeta;
    this.options = options;

    this.handleData = this.handleData.bind(this);
    this.handleSingleRequestEnd = this.handleSingleRequestEnd.bind(this);
  }

  start() {
    if (this.isRunning) return;
    this.isRunning = true;
    this.executeRequest();
  }

  stop() {
    var _this2 = this;

    return asyncToGenerator(function* () {
      if (!_this2.isRunning) return;
      _this2.isRunning = false;
      if (_this2.currentRequest) {
        _this2.currentRequest.destroy();
        _this2.currentRequest = null;
      }
      if (_this2.pendingWrite > 0) {
        yield _this2.awaitPendingWrite();
      }
    })();
  }

  awaitPendingWrite() {
    var _this3 = this;

    return asyncToGenerator(function* () {
      return new Promise(function (resolve) {
        let awaitProgress = function () {
          if (_this3.pendingWrite === 0) {
            _this3.off('progress', awaitProgress);
            resolve();
          }
        };
        if (_this3.pendingWrite === 0) {
          resolve();
        } else {
          _this3.on('progress', awaitProgress);
        }
      });
    })();
  }

  handleData(data) {
    var _this4 = this;

    return asyncToGenerator(function* () {
      if (!_this4.isRunning) return;
      const start = _this4.currentPosition;
      const end = start + data.byteLength;
      _this4.currentPosition = end;
      _this4.pendingWrite++;
      try {
        yield writeAsync(_this4.options.fd, data, 0, data.byteLength, start);
      } catch (e) {
        _this4.pendingWrite--;
        return _this4.emitError(new Error(Task.ERROR.ERR_DISK_IO_WRITE));
      }
      _this4.pendingWrite--;
      _this4.taskMeta.completeRange([start, end]);
      _this4.emit('progress');
    })();
  }

  emitError(e) {
    var _this5 = this;

    return asyncToGenerator(function* () {
      yield _this5.stop();
      _this5.emit('error', e);
    })();
  }

  emitEnd() {
    var _this6 = this;

    return asyncToGenerator(function* () {
      yield _this6.stop();
      _this6.emit('end');
    })();
  }

  handleSingleRequestEnd() {
    var _this7 = this;

    return asyncToGenerator(function* () {
      if (!_this7.isRunning) return;
      _this7.currentRequest = null;
      yield _this7.awaitPendingWrite();
      _this7.executeRequest();
    })();
  }

  off(...arg) {
    return this.removeListener(...arg);
  }
}

const writeFileAsync = util.promisify(fs.writeFile);
const readFileAsync = util.promisify(fs.readFile);

const openAsync = util.promisify(fs.open);
const closeAsync = util.promisify(fs.close);

class Task extends EventEmitter {
  constructor(...args) {
    var _temp, _this;

    return _temp = _this = super(...args), this.metadata = null, this.threads = [], this.isRunning = false, this.emitError = (() => {
      var _ref = asyncToGenerator(function* (err) {
        yield _this.stop();
        _this.emit('error', err);
      });

      return function (_x) {
        return _ref.apply(this, arguments);
      };
    })(), _temp;
  }

  static create({ url: url$$1, referUrl, downloadDir, downloadPath }) {
    return asyncToGenerator(function* () {
      const task = new Task();
      const response = yield fetch(url$$1, { method: 'HEAD' });
      if (response.statusCode !== 200) {
        throw new Error(Task.ERROR.NETWORK_ERROR);
      }

      const acceptRanges = response.headers['accept-ranges'];
      const mimeType = response.headers['content-type'];
      const totalBytes = response.headers['content-length'];
      if (!downloadPath) {
        downloadPath = path.join(downloadDir, `${uuid()}.${mimeType.replace(/^.*\//, '')}`);
      }
      task.metadata = new TaskMeta({
        url: url$$1,
        referUrl,
        mimeType,
        totalBytes: Number(totalBytes),
        resumable: !!acceptRanges,
        downloadPath: downloadPath
      });
      return task;
    })();
  }

  static restoreFromFile(path$$1) {
    return asyncToGenerator(function* () {
      const content = yield readFileAsync(path$$1, 'utf-8');
      const task = new Task();
      task.metadata = new TaskMeta(JSON.parse(content));
      return task;
    })();
  }

  storeToFile(path$$1) {
    var _this2 = this;

    return asyncToGenerator(function* () {
      const content = JSON.stringify(_this2.metadata.toJSONObject());
      yield writeFileAsync(path$$1, content, 'utf-8');
    })();
  }

  start(numOfThread = 1, windowSize) {
    var _this3 = this;

    return asyncToGenerator(function* () {
      if (!_this3.metadata) {
        return _this3.emitError(new Error(Task.ERROR.ERR_NO_METADATA));
      }

      if (_this3.isRunning) return;
      _this3.isRunning = true;

      if (!windowSize) {
        windowSize = Math.ceil(_this3.metadata.remainingByteCount / numOfThread);
      }
      if (!_this3.metadata.resumable) {
        // non-resumable resource, use 1 thread to download all the bytes
        numOfThread = 1;
        windowSize = _this3.metadata.totalBytes;
      }

      yield _this3.openFd();
      if (_this3.fd) {
        let endHandler = _this3.getThreadEndHandler(numOfThread);
        const options = { windowSize, fd: _this3.fd };
        for (; numOfThread > 0; numOfThread--) {
          const taskThread = new TaskThread(_this3.metadata, options);
          taskThread.on('progress', function () {
            return _this3.emit('progress');
          });
          taskThread.on('end', endHandler);
          taskThread.on('error', _this3.emitError);
          taskThread.start();
          _this3.threads.push(taskThread);
        }
      }
    })();
  }

  getThreadEndHandler(num) {
    var _this4 = this;

    return asyncToGenerator(function* () {
      if (--num === 0) {
        _this4.isRunning = false;
        _this4.threads = [];
        yield _this4.closeFd();
        _this4.emit('end');
      }
    });
  }

  stop() {
    var _this5 = this;

    return asyncToGenerator(function* () {
      if (!_this5.isRunning) return;
      _this5.isRunning = false;
      yield Promise.all(_this5.threads.map(function (thread) {
        return thread.stop();
      }));
      _this5.threads = [];
      _this5.metadata.resetDownloadingRange();
      yield _this5.closeFd();
    })();
  }

  openFd(dir) {
    var _this6 = this;

    return asyncToGenerator(function* () {
      _this6.fd = yield openAsync(_this6.metadata.downloadPath, 'a');
    })();
  }

  closeFd() {
    var _this7 = this;

    return asyncToGenerator(function* () {
      yield closeAsync(_this7.fd);
    })();
  }

  off(...arg) {
    return this.removeListener(...arg);
  }

  get progress() {
    return this.metadata.progress;
  }
}
Task.ERROR = {
  ERR_NO_METADATA: 'ERR_NO_METADATA',
  ERR_DISK_IO_WRITE: 'ERR_DISK_IO_WRITE',
  ERR_NETWORK: 'ERR_NETWORK'
};

var index = { Task };

module.exports = index;