social-butterfly/dist/util/crawler.js

Incorporate federated social network protocols easily. Used with Hello, world federated blog.

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.fetchUrl = fetchUrl;
exports.fetchText = fetchText;
exports.fetchJSON = fetchJSON;
exports.createAbsoluteUrl = createAbsoluteUrl;
exports.isRobotViewing = isRobotViewing;
exports.sanitizeHTML = sanitizeHTML;

var _exceptions = require("./exceptions");

var _nodeFetch = _interopRequireDefault(require("node-fetch"));

var _sanitizeHtml = _interopRequireDefault(require("sanitize-html"));

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

async function fetchUrl(url, opt_headers) {
  try {
    const headers = Object.assign({
      'user-agent': 'hello, world bot.'
    }, opt_headers || {});
    const response = await (0, _nodeFetch.default)(url, {
      headers
    });

    if (response.status >= 400) {
      throw new _exceptions.HTTPError(response.status, url);
    }

    return response;
  } catch (ex) {
    throw ex;
  }
}

async function fetchText(url, opt_headers) {
  const response = await fetchUrl(url);
  return await response.text();
}

async function fetchJSON(url, opt_headers) {
  const response = await fetchUrl(url, opt_headers);
  return await response.json();
}

function createAbsoluteUrl(websiteUrl, url) {
  if (url?.startsWith('/')) {
    const parsedUrl = new URL(websiteUrl);
    url = `${parsedUrl.origin}${url}`;
  }

  return url;
}

function isRobotViewing(req) {
  const userAgent = req.headers['x-user-agent'] || req.headers['user-agent'];
  return !!userAgent?.match(/bot|spider|crawl|slurp|ia_archiver/i);
}

function sanitizeHTML(rawHTML) {
  return (0, _sanitizeHtml.default)(rawHTML, {
    allowedTags: _sanitizeHtml.default.defaults.allowedTags.concat(['img']),
    allowedAttributes: {
      a: ['href', 'name', 'target', 'title'],
      img: ['src', 'srcset', 'width', 'height', 'alt', 'title'],
      iframe: ['src', 'width', 'height', 'alt', 'title']
    }
  });
}