crawl-to-markdown/dist/index.d.cts

Crawl-to-markdown is a powerful TypeScript package designed to search search engines for a given keyword, crawl the resulting websites, and deliver the content in clean, readable Markdown format. Additionally, it can directly crawl specified websites for

import { Browser, Page } from 'puppeteer';

type SEARCH_ENGINE = "google";
declare class MarkdownCrawler {
    browser: Browser | undefined;
    headless: boolean;
    constructor(_headless: boolean);
    crawlFromUrl(url: string): Promise<{
        markdown: string;
        status: number;
    }>;
    crawlFromKeyword(keyword: string, search_engine: SEARCH_ENGINE): Promise<{
        markdown: string;
        url: string;
    }[]>;
    ensureBrowser(): Promise<boolean>;
    crawlSearchEngine(keyword: string): Promise<string[]>;
    crawllMultipleWebsite(urls: string[]): Promise<{
        url: string;
        markdown: string;
    }[]>;
    crawlSingleWebsite(url: string): Promise<{
        markdown: string;
        status: number;
    }>;
    extractLinks(page: Page, baseUrl: string): Promise<string[]>;
    getWebsiteMarkdown({ urls }: {
        urls: string[];
    }): Promise<{
        url: string;
        markdown: string;
    }[]>;
    fetchAndProcessPage(url: string): Promise<string>;
    isValidUrl(url: string): boolean;
}

export { MarkdownCrawler };