@xapp/arachne-utils/lib/generateFollowThroughPattern.js

"use strict";
/*! Copyright (c) 2024, XAPP AI */
Object.defineProperty(exports, "__esModule", { value: true });
exports.generateFollowThroughPattern = generateFollowThroughPattern;
/**
 * Generates a URL pattern that will enable the crawler to
 * follow through sub paths for the provided URL.
 *
 * For example:
 *
 * www.example.com/faqs
 *
 * will become
 *
 * https://[(www.)?]example.com/faqs[.+]
 *
 * NOTE: This will throw an error if the URL was not a proper URL.
 *
 * @param urlString
 */
function generateFollowThroughPattern(urlString) {
    if (!urlString) {
        throw new Error(`Unabled to generate follow through patterns, URL not provided.`);
    }
    // Remove the trailing "/"
    if (urlString.endsWith("/")) {
        urlString = urlString.slice(0, -1);
    }
    let url;
    try {
        url = new URL(urlString);
    }
    catch (e) {
        throw new Error(`Unabled to generate follow through patterns, URL ${urlString} is invalid`);
    }
    const protocol = url.protocol;
    let host = url.host;
    // Remove the www. if it is there
    if (host.startsWith("www.")) {
        host = host.replace("www.", "");
    }
    let path = url.pathname;
    if (path.endsWith("/")) {
        path = path.slice(0, -1);
    }
    path = path.replace(/\/index\.html$/, '[(/index.html)?]'); // Make '/index.html' optional
    let pattern;
    const protocolPattern = protocol.startsWith("http") ? "http[(s)?]:" : protocol;
    // Either starts with www or is just domain.tld
    if (url.host.startsWith("www.") || url.host.split(".").length === 2) {
        // make www optional
        pattern = `${protocolPattern}//[(www.)?]${host}${path}[.+]`;
    }
    else {
        pattern = `${protocolPattern}//${host}${path}[.+]`;
    }
    return pattern;
}
//# sourceMappingURL=generateFollowThroughPattern.js.map