pyseoa-ts/dist/crawler/crawlWebsite.js

SEO analysis tools for the web, ported from pyseoa (Python) to TypeScript

import { extractLinks } from "./extractLinks";
export async function crawlWebsite(startUrl, options = {}, onProgress) {
    const visited = new Set();
    const queue = [{ url: startUrl, depth: 0 }];
    const maxPages = options.maxPages ?? 50;
    const maxDepth = options.maxDepth ?? 3;
    const pages = [];
    while (queue.length > 0 && visited.size < maxPages) {
        if (options.delayMS) {
            await new Promise(resolve => setTimeout(resolve, options.delayMS));
        }
        const { url, depth } = queue.shift();
        if (visited.has(url) || depth > maxDepth) {
            onProgress?.({ type: "crawl", url, pageIndex: visited.size, stage: "skip" });
            continue;
        }
        try {
            console.log("📍 Processing URL:", url, typeof url);
            const res = await fetch(url, {
                headers: {
                    "User-Agent": options.userAgent ?? "pyseoa-ts analyzer bot",
                }
            });
            if (!res.ok)
                continue;
            const html = await res.text();
            onProgress?.({ type: "crawl", url, pageIndex: visited.size, stage: "fetch" });
            visited.add(url);
            pages.push({ url, html });
            const links = extractLinks(html, url);
            for (const link of links) {
                if (!visited.has(link)) {
                    if (typeof link !== "string") {
                        console.warn("🚨 Invalid link pushed to queue:", link);
                    }
                    queue.push({ url: link, depth: depth + 1 });
                }
            }
        }
        catch (err) {
            console.warn(`Failed to fetch ${url}`, err);
        }
    }
    return pages;
}