relatt-scraper/utils/getUrlHeaderData.js

Metascarper

const got = require("got");
const { generateLinkRequestLogsLine } = require("./generateLinkRequestLogLine");

const getUrlHeaderData = async ({
  link,
  operationType,
  fetchOptions = {},
  can_try_again = true,
}) => {
  let result = {};

  if (!fetchOptions.method) fetchOptions.method = "HEAD";

  try {
    const { headers } = await got(`${link}`, fetchOptions);

    result = {
      contentType: headers["content-type"],
      contentLength: headers["content-length"],
      logs: [generateLinkRequestLogsLine(`link`, operationType, 200)],
      ...headers,
    };
  } catch (error) {
    result.logs = [generateLinkRequestLogsLine(`link`, operationType, 404)];
    /**
     * if HEAD method return 403 Forbidden fetch with GET METHOD
     */
    if (can_try_again) {
      result = await getUrlHeaderData({
        link,
        operationType,
        fetchOptions: {
          ...fetchOptions,
          method: "GET",
          https: { rejectUnauthorized: false },
        },
        can_try_again: false,
      });
    } else {
      throw new Error("UNREACHABLE LINK");
    }
  }

  return result;
};

module.exports = { getUrlHeaderData };