npm-count/lib/fetchTrending.js

a npm/download-counts wrapper

'use strict';

Object.defineProperty(exports, "__esModule", {
  value: true
});

var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; // dependencies


// self dependencies


var _moment = require('moment');

var _moment2 = _interopRequireDefault(_moment);

var _bluebird = require('bluebird');

var _bluebird2 = _interopRequireDefault(_bluebird);

var _request = require('request');

var _request2 = _interopRequireDefault(_request);

var _JSONStream = require('JSONStream');

var _JSONStream2 = _interopRequireDefault(_JSONStream);

var _immutable = require('immutable');

var _immutable2 = _interopRequireDefault(_immutable);

var _fetchBulkQuery = require('./fetchBulkQuery');

var _fetchBulkQuery2 = _interopRequireDefault(_fetchBulkQuery);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

// private
var api = 'https://skimdb.npmjs.com/-/all/since';

/**
* fetch the package stats base on `date`
*
* @module fetchTrending
* @param {string} date - fetch based on date(YYYY-MM-DD)
* @param {object} [options]
* @param {number} [options.step=10] - bulk query split size
* @returns {promise<collection>} info
* @see https://github.com/watilde/npm/issues/4#issuecomment-192640770
*/

exports.default = function (date) {
  var options = arguments.length <= 1 || arguments[1] === undefined ? {} : arguments[1];
  return new _bluebird2.default(function (resolve, reject) {
    var opts = _immutable2.default.fromJS({
      step: 10
    }).mergeDeep(options).toJS();

    // npmのdbから変更日を元にパッケージ情報を取得
    var startkey = _moment2.default.utc(date).startOf('day')._d.getTime();
    var endkey = _moment2.default.utc(date).endOf('day')._d.getTime();
    var url = api + '?stale=update_after&startkey=' + startkey + '&endkey=' + endkey;
    var stream = (0, _request2.default)(url, {
      headers: { host: 'registry.npmjs.org' },
      // see: https://github.com/bitinn/node-fetch/issues/15
      agentOptions: {
        rejectUnauthorized: false
      }
    });

    stream.on('error', reject).on('response', function (response) {
      if (response.statusCode >= 400) {
        return stream.on('data', function (data) {
          var _JSON$parse = JSON.parse(data.toString());

          var error = _JSON$parse.error;
          var reason = _JSON$parse.reason;

          reject(response.statusCode + ' ' + error + ': ' + reason);
        });
      }

      // パッケージの更新情報にダウンロード数を付与する
      var index = {};
      var summaries = [];
      var bulkQueries = [];
      var pendingNames = [];
      return stream.pipe(_JSONStream2.default.parse('*')).on('data', function (data) {
        if (data.name === undefined) {
          return;
        }

        // まれにdownloadsが取得できないパッケージがあるので、0で初期化する
        summaries.push(_extends({}, data, {
          downloads: 0
        }));
        index[data.name] = summaries.length - 1;

        // step件数溜まったらリクエストを発行
        pendingNames.push(data.name);
        if (pendingNames.length >= opts.step) {
          bulkQueries.push((0, _fetchBulkQuery2.default)(pendingNames));
          pendingNames = [];
        }
      }).on('end', function () {
        // 未解消のリクエストを開放
        if (pendingNames.length) {
          bulkQueries.push((0, _fetchBulkQuery2.default)(pendingNames));
          pendingNames = [];
        }

        // 全てのリクエストが終了したらパッケージ情報とマージ
        _bluebird2.default.all(bulkQueries).then(function (jsons) {
          jsons.forEach(function (json) {
            for (var key in json) {
              if (json.hasOwnProperty(key) === false) {
                continue;
              }

              var stat = json[key];
              var summary = summaries[index[key]];
              if (summary) {
                summary.downloads = stat.downloads;
              }
            }
          });

          resolve(summaries);
        });
      });
    });
  });
};

module.exports = exports['default'];