x-ray-scraper/src/crawler/node-http-driver.js

Scraper next gen based on x-ray (2.3.2)

const superagentProxyWrapper = require('superagent-proxy');
var superagent = superagentProxyWrapper(require('superagent'));

/**
 * Default HTTP driver
 *
 * @param {Object} opts
 * @return {Function}
 */

function driver(opts) {
  var agent = superagent.agent(opts || {});
  var proxy = process.env.HTTP_PROXY || process.env.http_proxy || null;

  return function http_driver(ctx, fn) {
    let get = agent.get(ctx.url);

    if (proxy) {
      get = get.proxy(proxy);
    }

    get.set(ctx.headers).end(function(err, res) {
      if (err && !err.status) return fn(err);

      ctx.status = res.status;
      ctx.set(res.headers);

      ctx.body = 'application/json' == ctx.type ? res.body : res.text;

      // update the URL if there were redirects
      ctx.url = res.redirects.length ? res.redirects.pop() : ctx.url;

      return fn(null, ctx);
    });
  };
}

module.exports = driver;