@avenga/linkinator/build/src/links.js

Find broken links, missing images, etc in your HTML. Scurry around your site and find all those broken links.

import { Stream } from 'node:stream';
import { WritableStream } from 'htmlparser2/WritableStream';
import { parseSrcset } from 'srcset';
const tagConfigs = {
    a: { urlAttrs: ['href'], captureText: true },
    area: { urlAttrs: ['href'], attributeKeys: ['alt'] },
    audio: { urlAttrs: ['src'] },
    blockquote: { urlAttrs: ['cite'], captureText: true },
    body: { urlAttrs: ['background'] },
    command: { urlAttrs: ['icon'] },
    del: { urlAttrs: ['cite'], captureText: true },
    embed: {
        urlAttrs: ['href', 'pluginspage', 'pluginurl', 'src'],
        attributeKeys: ['type'],
    },
    frame: { urlAttrs: ['longdesc', 'src'], attributeKeys: ['name', 'title'] },
    html: { urlAttrs: ['manifest'] },
    iframe: { urlAttrs: ['longdesc', 'src'], attributeKeys: ['name', 'title'] },
    img: { urlAttrs: ['src', 'srcset'], attributeKeys: ['alt'] },
    input: { urlAttrs: ['src'], attributeKeys: ['alt'] },
    ins: { urlAttrs: ['cite'], captureText: true },
    link: { urlAttrs: ['href'], attributeKeys: ['rel'] },
    meta: { urlAttrs: ['content'], attributeKeys: ['name'] },
    object: { urlAttrs: ['data'], attributeKeys: ['type'] },
    q: { urlAttrs: ['cite'], captureText: true },
    script: { urlAttrs: ['src'], attributeKeys: ['type'] },
    source: { urlAttrs: ['src', 'srcset'], attributeKeys: ['type'] },
    track: { urlAttrs: ['src'], attributeKeys: ['kind'] },
    video: { urlAttrs: ['poster', 'src'] },
};
export async function getLinks(source, baseUrl) {
    let realBaseUrl = baseUrl;
    let baseSet = false;
    // Tracks all currently open tags that have text to be captured
    let activeTextCapture = [];
    const links = [];
    const parser = new WritableStream({
        onopentag(tag, attributes) {
            // Allow alternate base URL to be specified in tag:
            if (tag === 'base' && !baseSet && attributes.href) {
                realBaseUrl = getBaseUrl(attributes.href, baseUrl);
                baseSet = true;
            }
            // ignore href properties for link tags where rel is likely to fail
            const relValuesToIgnore = ['dns-prefetch', 'preconnect'];
            if (tag === 'link' && relValuesToIgnore.includes(attributes.rel)) {
                return;
            }
            // Only for <meta content=""> tags, only validate the url if
            // the content actually looks like a url
            if (tag === 'meta' && attributes.content) {
                try {
                    new URL(attributes.content);
                }
                catch {
                    return;
                }
            }
            const cfg = tagConfigs[tag];
            // Nothing to do for this tag
            if (!cfg) {
                return;
            }
            // Iterate over tag attributes that could contain URLs
            for (const attr of cfg.urlAttrs) {
                const raw = attributes[attr];
                if (!raw) {
                    continue;
                }
                for (const parsedAttribute of parseAttribute(attr, raw)) {
                    const parsedUrl = parseLink(parsedAttribute, realBaseUrl);
                    parsedUrl.metadata = {
                        tag,
                        ...(attributes.id ? { id: attributes.id } : {}),
                        ...(attributes.class ? { class: attributes.class } : {}),
                    };
                    // Use specified attributes if they exist to identify element
                    for (const attributeKey of cfg.attributeKeys ?? []) {
                        if (attributes[attributeKey]) {
                            parsedUrl.metadata[attributeKey] = attributes[attributeKey];
                        }
                    }
                    // Add element to array of currently open tags to capture following inner text
                    if (cfg.captureText) {
                        parsedUrl.metadata.text = '';
                        activeTextCapture.push({
                            tag,
                            parsed: parsedUrl,
                        });
                    }
                    links.push(parsedUrl);
                }
            }
        },
        ontext(data) {
            // Add text to all currently open tags
            for (const entry of activeTextCapture) {
                if (entry.parsed.metadata) {
                    entry.parsed.metadata.text += data.trim();
                }
            }
        },
        onclosetag(tag) {
            // Remove now closed tag from array of opened tags
            activeTextCapture = activeTextCapture.filter((e) => e.tag !== tag);
        },
    });
    await new Promise((resolve, reject) => {
        Stream.Readable.fromWeb(source)
            .pipe(parser)
            .on('finish', resolve)
            .on('error', reject);
    });
    return links;
}
function getBaseUrl(htmlBaseUrl, oldBaseUrl) {
    if (isAbsoluteUrl(htmlBaseUrl)) {
        return htmlBaseUrl;
    }
    const url = new URL(htmlBaseUrl, oldBaseUrl);
    url.hash = '';
    return url.href;
}
function isAbsoluteUrl(url) {
    // Don't match Windows paths
    if (/^[a-zA-Z]:\\/.test(url)) {
        return false;
    }
    // Scheme: https://tools.ietf.org/html/rfc3986#section-3.1
    // Absolute URL: https://tools.ietf.org/html/rfc3986#section-4.3
    return /^[a-zA-Z][a-zA-Z\d+\-.]*:/.test(url);
}
function parseAttribute(name, value) {
    switch (name) {
        case 'srcset': {
            return parseSrcset(value).map((p) => p.url);
        }
        default: {
            return [value];
        }
    }
}
function parseLink(link, baseUrl) {
    try {
        const url = new URL(link, baseUrl);
        url.hash = '';
        return { link, url };
    }
    catch (error) {
        return { link, error: error };
    }
}