@mintlify/link-rot/dist/static-checking/getBrokenExternalLinks.js

Static checking for broken internal links

var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
const CONCURRENCY = 5;
const TIMEOUT_MS = 10000;
const USER_AGENT = 'Mintlify-LinkChecker/1.0';
const isHttpUrl = (url) => {
    try {
        const parsed = new URL(url);
        return parsed.protocol === 'http:' || parsed.protocol === 'https:';
    }
    catch (_a) {
        return false;
    }
};
const checkExternalUrl = (url) => __awaiter(void 0, void 0, void 0, function* () {
    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), TIMEOUT_MS);
    try {
        const response = yield fetch(url, {
            method: 'HEAD',
            signal: controller.signal,
            headers: { 'User-Agent': USER_AGENT },
            redirect: 'follow',
        });
        clearTimeout(timeoutId);
        return {
            ok: response.status !== 404,
            status: response.status,
        };
    }
    catch (err) {
        clearTimeout(timeoutId);
        return {
            ok: false,
            status: null,
            error: err instanceof Error ? err.message : 'unknown error',
        };
    }
});
const pMap = (items, fn, concurrency) => __awaiter(void 0, void 0, void 0, function* () {
    const results = [];
    let index = 0;
    const run = () => __awaiter(void 0, void 0, void 0, function* () {
        while (index < items.length) {
            const currentIndex = index++;
            results[currentIndex] = yield fn(items[currentIndex]);
        }
    });
    const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => run());
    yield Promise.all(workers);
    return results;
});
export const getBrokenExternalLinks = (graph, options) => __awaiter(void 0, void 0, void 0, function* () {
    const urlSourceMap = new Map();
    for (const [nodeLabel, externalPaths] of graph.getExternalPathsByNode()) {
        if ((options === null || options === void 0 ? void 0 : options.sourceFiles) && !options.sourceFiles.has(nodeLabel))
            continue;
        for (const mdxPath of externalPaths) {
            if (isHttpUrl(mdxPath.originalPath)) {
                const url = mdxPath.originalPath;
                const existing = urlSourceMap.get(url);
                const source = { file: nodeLabel, originalPath: mdxPath.originalPath };
                if (existing) {
                    existing.push(source);
                }
                else {
                    urlSourceMap.set(url, [source]);
                }
            }
        }
    }
    const uniqueUrls = [...urlSourceMap.keys()];
    const results = yield pMap(uniqueUrls, (url) => __awaiter(void 0, void 0, void 0, function* () {
        const result = yield checkExternalUrl(url);
        if (result.ok)
            return null;
        const entry = {
            url,
            status: result.status,
            sources: urlSourceMap.get(url),
        };
        if (result.error)
            entry.error = result.error;
        return entry;
    }), CONCURRENCY);
    return results.filter((r) => r !== null);
});