@xapp/arachne-utils/lib/cleanForCrawl.js

"use strict";
/*! Copyright (c) 2024, XAPP AI */
Object.defineProperty(exports, "__esModule", { value: true });
exports.cleanForCrawl = void 0;
/**
 * Prepare a URL for crawling.
 *
 * * Ensures there is no trailing slash
 * * Ensures there is no hash
 * * Ensures there is a scheme, defaults to https://
 *
 * @param url
 * @returns
 */
const cleanForCrawl = (url) => {
    // First make sure they have a scheme
    if (!url.startsWith("http")) {
        // make sure it doesn't have // in the beginning
        if (url.startsWith("//")) {
            url = `https:${url}`;
        }
        else {
            url = `https://${url}`;
        }
    }
    // Remove trailing slash
    if (url.endsWith("/")) {
        url = url.slice(0, -1);
    }
    // Remove hash
    const hashIndex = url.indexOf("#");
    if (hashIndex > -1) {
        url = url.slice(0, hashIndex);
    }
    return url;
};
exports.cleanForCrawl = cleanForCrawl;
//# sourceMappingURL=cleanForCrawl.js.map