@ezs/basics/lib/url-pager.js

Basics statements for EZS

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.default = URLPager;

var _debug = _interopRequireDefault(require("debug"));

var _url = require("url");

var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));

var _lodash = _interopRequireDefault(require("lodash.get"));

var _parseHeaders = _interopRequireDefault(require("parse-headers"));

var _asyncRetry = _interopRequireDefault(require("async-retry"));

var _request = _interopRequireDefault(require("./request"));

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

/**
 * Take `Object` as parameters of URL, throw each chunk from the result
 *
 *
 * Input:
 *
 * ```json
 * [{"q": "a"}]
 * ```
 *
 * Script:
 *
 * ```ini
 * [URLPager]
 * url = https://api.search.net
 * path = total
 * ```
 *
 * Output:
 *
 * ```json
 * [
 *      {
 *          "q": "a",
 *          "total": 22
 *          "offset": 0,
 *          "pageNumber": 1,
 *          "totalPages", 3,
 *          "maxPages": 1000,
 *          "limit": 10
 *      },
 *      {
 *          "q": "a",
 *          "total": 22
 *          "offset": 10,
 *          "pageNumber": 2,
 *          "totalPages", 3,
 *          "maxPages": 1000,
 *          "limit": 10
 *      },
 *      {
 *          "q": "a",
 *          "total": 22
 *          "offset": 20,
 *          "pageNumber": 3,
 *          "totalPages", 3,
 *          "maxPages": 1000,
 *          "limit": 10
 *      }
 *  ]
 * ```
 *
 * #### Example with URLs
 *
 * Input:
 *
 * ```json
 * [
 *   "https://httpbin.org/get?a=a",
 *   "https://httpbin.org/get?a=b",
 *   "https://httpbin.org/get?a=c"
 * ]
 * ```
 *
 * Script:
 *
 * ```ini
 * [URLPager]
 * path = .args
 * ```
 *
 * Output:
 *
 * ```json
 * [{"a": "a"}, {"a": "b"}, {"a": "c" }]
 * ```
 *
 * @name URLPager
 * @param {String} [url] URL to fetch (by default input string is taken)
 * @param {String} [path=total] choose the path to find the number of result
 * @param {Number} [timeout=1000] Timeout in milliseconds
 * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
 * @param {Number} [retries=5] The maximum amount of times to retry the connection
 * @returns {Object}
 */
async function URLPager(data, feed) {
  if (this.isLast()) {
    return feed.close();
  }

  const url = this.getParam('url');
  const path = this.getParam('path', 'total');
  const limit = Number(this.getParam('limit', 10));
  const maxPages = Number(this.getParam('maxPages', 1000));
  const retries = Number(this.getParam('retries', 5));
  const noerror = Boolean(this.getParam('noerror', false));
  const timeout = Number(this.getParam('timeout')) || 1000;
  const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
  const cURL = new _url.URL(url || data);
  const controller = new _nodeAbortController.default();
  const parameters = {
    timeout,
    headers,
    signal: controller.signal
  };
  const options = {
    retries
  };
  cURL.search = new _url.URLSearchParams(data);

  const onError = e => {
    controller.abort();

    if (noerror) {
      (0, _debug.default)('ezs')(`Ignore item #${this.getIndex()} [URLPager] <${e}>`);
      return feed.send(data);
    }

    (0, _debug.default)('ezs')(`Break item #${this.getIndex()} [URLPager] <${e}>`);
    return feed.send(e);
  };

  try {
    const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
    const json = await response.json();
    const total = (0, _lodash.default)(json, path);

    if (total === 0) {
      return onError(new Error('No result.'));
    }

    if (total === undefined) {
      return onError(new Error('Unexpected response.'));
    }

    let totalPages = Math.ceil(json.total / limit);

    if (totalPages > maxPages) {
      totalPages = maxPages;
    }

    for (let pageNumber = 1; pageNumber <= totalPages; pageNumber += 1) {
      feed.write({ ...data,
        offset: (pageNumber - 1) * limit,
        pageNumber,
        totalPages,
        maxPages,
        limit
      });
    }

    feed.end();
  } catch (e) {
    onError(e);
  }
}