all-package-names/build/sync.js

Get all NPM package names

"use strict";

exports.__esModule = true;
exports.sync = sync;
exports.syncAction = syncAction;
exports["default"] = void 0;

var _https = require("https");

var _pLock = require("p-lock");

var _progress = _interopRequireDefault(require("progress"));

var _parseJsonObject = require("parse-json-object");

var _commanderVersion = require("commander-version");

var _load = require("./load");

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { "default": obj }; }

var getOptions = function getOptions(path) {
  if (path === void 0) {
    path = "";
  }

  return {
    hostname: "replicate.npmjs.com",
    port: 443,
    path: path,
    method: "GET"
  };
};

function getEnd() {
  return new Promise(function (resolve) {
    var data = "";
    var options = getOptions();
    (0, _https.request)(options, function (res) {
      res.on("data", function (chunk) {
        data += chunk.toString();
      });
      res.on("end", function () {
        var _ref = (0, _parseJsonObject.parseJSONObject)(data),
            update_seq = _ref.update_seq;

        resolve(update_seq);
      });
    }).end();
  });
}

var initialState = function initialState(_ref2, end) {
  var since = _ref2.since,
      packageNames = _ref2.packageNames;
  return {
    data: "",
    start: since,
    index: since,
    end: end,
    progress: 0,
    elapsed: 0,
    lastSave: new Date().getTime(),
    packageNames: new Set(packageNames)
  };
};

function isChange(item) {
  return item !== undefined && typeof item.seq === "number" && typeof item.id === "string";
}

function pump(state) {
  while (state.data.indexOf("\n") !== state.data.lastIndexOf("\n")) {
    var newline = state.data.indexOf("\n", 1);
    var line = state.data.slice(0, newline);

    if (line.endsWith(",")) {
      line = line.slice(0, line.length - 1);
    }

    var item = (0, _parseJsonObject.parseJSONObject)(line);

    if (isChange(item)) {
      state.index = item.seq;
      state.packageNames.add(item.id);
      state.progress = (state.index - state.start) / (state.end - state.start);
    }

    state.data = state.data.slice(newline);
  }
}

var lock = (0, _pLock.getLock)();

function sync(_temp) {
  var _ref3 = _temp === void 0 ? {} : _temp,
      onData = _ref3.onData,
      onStart = _ref3.onStart,
      onEnd = _ref3.onEnd,
      maxAge = _ref3.maxAge;

  var startTime = new Date().getTime();
  return new Promise(function (resolve) {
    lock().then(function (release) {
      (0, _load.load)({
        maxAge: maxAge
      }).then(function (data) {
        if ((0, _load.isFresh)(data, maxAge)) {
          release();
          resolve(data);
        } else {
          getEnd().then(function (end) {
            var state = initialState(data, end);

            if (onStart) {
              onStart(state);
            }

            (0, _https.request)(getOptions("/_changes?since=" + data.since), function (res) {
              res.on("data", function (chunk) {
                state.data += chunk.toString();
                pump(state);
                state.elapsed = new Date().getTime() - startTime;

                if (onData) {
                  onData(state);
                }

                if (new Date().getTime() - state.lastSave > 60000) {
                  state.lastSave = new Date().getTime();
                  var newSave = {
                    since: state.index,
                    timestamp: new Date().getTime(),
                    packageNames: Array.from(state.packageNames.values())
                  };
                  (0, _load.save)(newSave);
                }
              });
              res.on("end", function () {
                if (onEnd) {
                  onEnd(state);
                }

                var newSave = {
                  since: state.index,
                  timestamp: new Date().getTime(),
                  packageNames: Array.from(state.packageNames.values())
                };
                (0, _load.save)(newSave).then(function () {
                  release();
                  resolve(newSave);
                });
              });
            }).end();
          });
        }
      });
    });
  });
}

function syncAction(options) {
  var bar;
  sync({
    onStart: function onStart(state) {
      if (state.end > state.start) {
        bar = new _progress["default"]("syncing [:bar] :percent ", {
          total: state.end - state.start
        });
      }
    },
    onData: function onData(state) {
      var _bar;

      (_bar = bar) == null ? void 0 : _bar.update(state.progress);
    },
    onEnd: function onEnd(state) {
      console.info("New packages: " + (state.end - state.start));
      console.info("Total: " + Object.keys(state.packageNames).length);
      console.info("Time: " + state.elapsed / 1000 + "s");
    },
    maxAge: options == null ? void 0 : options.maxAge
  });
}

var _default = (0, _commanderVersion.createCommand)("sync").description("Sync latest packages from NPM").option("-m --max-age [milliseconds]", "Maximum milliseconds after a sync to avoid re-syncing", parseInt).action(syncAction);

exports["default"] = _default;