crawl-to-markdown/dist/index.cjs.map

Crawl-to-markdown is a powerful TypeScript package designed to search search engines for a given keyword, crawl the resulting websites, and deliver the content in clean, readable Markdown format. Additionally, it can directly crawl specified websites for

{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import puppeteer, { Page, Browser } from \"puppeteer\";\n\ntype SEARCH_ENGINE = \"google\";\n\nexport class MarkdownCrawler {\n    browser: Browser | undefined;\n    headless: boolean = false;\n    constructor(_headless: boolean) {\n        this.headless = _headless;\n    }\n\n    async crawlFromUrl(\n        url: string,\n    ): Promise<{ markdown: string; status: number }> {\n        if (!this.isValidUrl(url)) {\n            throw new Error(\n                \"Invalid URL provided, should be a full URL starting with http:// or https://\",\n            );\n        }\n\n        if (!(await this.ensureBrowser())) {\n            throw new Error(\"Could not start browser instance\");\n        }\n\n        return this.crawlSingleWebsite(url);\n    }\n    async crawlFromKeyword(\n        keyword: string,\n        search_engine: SEARCH_ENGINE,\n    ): Promise<{ markdown: string; url: string }[]> {\n        if (typeof keyword !== \"string\") {\n            throw new Error(\"Invalid Keyword provided, should be a string\");\n        }\n\n        if (search_engine !== \"google\") {\n            throw new Error(\n                \"only google search engine is supported at the moment\",\n            );\n        }\n        if (!(await this.ensureBrowser())) {\n            throw new Error(\"Could not start browser instance\");\n        }\n\n        const urls = await this.crawlSearchEngine(keyword);\n        if (urls.length > 0) {\n            return await this.crawllMultipleWebsite(urls);\n        } else {\n            throw new Error(\"Couldn't crawl for this keyword\");\n        }\n    }\n\n    async ensureBrowser():Promise<boolean> {\n        let retries = 3;\n        while (retries) {\n            if (!this.browser || !this.browser.connected) {\n                try {\n                    this.browser = await puppeteer.launch({ headless: this.headless });\n                    return true;\n                } catch (e) {\n                    console.error(\n                        \"Browser DO: Could not start browser instance. Error: ${e}\",\n                    );\n                    retries--;\n                    if (!retries) {\n                        return false;\n                    }\n\n                    console.log(\n                        \"Retrying to start browser instance. Retries left: ${retries}\",\n                    );\n                }\n            } else {\n                return true;\n            }\n        }\n        return false;\n    }\n    async crawlSearchEngine(keyword: string):Promise<string[]> {\n        const page = await this.browser!.newPage();\n        await page.goto(\"https://www.google.com\");\n        await page.type(\"textarea[name=q]\", keyword);\n        await Promise.all([\n            page.waitForNavigation(),\n            await page.keyboard.press(\"Enter\"), // Action that causes navigation\n        ]);\n\n        // Function to scroll the page\n        const scrollToBottom = async ():Promise<void> => {\n            await page.evaluate(async () => {\n                // Scroll down to the bottom of the page\n                window.scrollTo(0, document.body.scrollHeight);\n                // Wait for new content to load\n                await new Promise((resolve) => setTimeout(resolve, 2000));\n            });\n        };\n\n        let previousHeight;\n        let currentHeight = await page.evaluate(\n            () => document.body.scrollHeight,\n        );\n\n        while (true) {\n            previousHeight = currentHeight;\n            await scrollToBottom();\n            currentHeight = await page.evaluate(\n                () => document.body.scrollHeight,\n            );\n\n            // Break if we reached the bottom of the page\n            if (currentHeight === previousHeight) {\n                break;\n            }\n        }\n\n        const searchResults = await page.evaluate(() => {\n            const results = [];\n            const items = document.querySelectorAll(\"h3\");\n            for (let i = 0; i < items.length; i++) {\n                const titleElement = items[i];\n                if (titleElement) {\n                    const link = titleElement.closest(\"a\");\n                    results.push({\n                        title: titleElement.innerText,\n                        url: link ? link.href : \"\",\n                    });\n                }\n            }\n            return results;\n        });\n        const urls = searchResults.map((search) => search.url);\n        return urls;\n    }\n    async crawllMultipleWebsite(urls: string[]) : Promise<{url: string;markdown: string;}[]>{\n        const mds = [];\n        for (let index = 0; index < urls.length; index++) {\n            const url = urls[index];\n            try {\n                const markdown = await this.fetchAndProcessPage(url);\n                mds.push({ url, markdown });\n                console.log(`${index + 1}/${urls.length} done, url : ${url}`);\n            } catch (error) {\n                console.error(\n                    `${index + 1}/${urls.length} failed, url : ${url}`,\n                );\n            }\n        }\n        await this.browser?.close();\n        return mds;\n    }\n    async crawlSingleWebsite(url: string) : Promise<{markdown: string;status: number;}>{\n        const md = await this.getWebsiteMarkdown({\n            urls: [url],\n        });\n        await this.browser?.close();\n        return {\n            markdown: md[0]?.markdown,\n            status: 200,\n        };\n    }\n\n    async extractLinks(page: Page, baseUrl: string): Promise<string[]> {\n        return await page.evaluate((baseUrl) => {\n            return Array.from(document.querySelectorAll(\"a\"))\n                .map((link) => (link as { href: string }).href)\n                .filter((link) => link.startsWith(baseUrl));\n        }, baseUrl);\n    }\n\n    async getWebsiteMarkdown({ urls }: { urls: string[] }): Promise<{\n        url: string;\n        markdown: string;\n    }[]> {\n        const isBrowserActive = await this.ensureBrowser();\n\n        if (!isBrowserActive) {\n            throw new Error(\"Could not start browser instance\");\n        }\n\n        return await Promise.all(\n            urls.map(async (url) => {\n                try {\n                    const markdown = await this.fetchAndProcessPage(url);\n                    return { url, markdown };\n                } catch (error) {\n                    console.error(error);\n                    return { url, markdown: \"\" };\n                }\n            }),\n        );\n    }\n\n    async fetchAndProcessPage(url: string): Promise<string> {\n        const page = await this.browser!.newPage();\n        await page.goto(url, { waitUntil: \"networkidle0\" });\n        const md = await page.evaluate(() => {\n            function extractPageMarkdown():string {\n                const readabilityScript = document.createElement(\"script\");\n                readabilityScript.src =\n                    \"https://unpkg.com/@mozilla/readability/Readability.js\";\n                document.head.appendChild(readabilityScript);\n\n                const turndownScript = document.createElement(\"script\");\n                turndownScript.src =\n                    \"https://unpkg.com/turndown/dist/turndown.js\";\n                document.head.appendChild(turndownScript);\n\n                let md = \"no content\";\n\n                // Wait for the libraries to load\n                md = Promise.all([\n                    new Promise(\n                        (resolve) => (readabilityScript.onload = resolve),\n                    ),\n                    new Promise((resolve) => (turndownScript.onload = resolve)),\n                ]).then(() => {\n                    // Turndown instance to convert HTML to Markdown\n                    // @ts-ignore\n                    const turndownService = new TurndownService();\n\n                    const documentWithoutScripts = document.cloneNode(true);\n                    /*if (!(documentWithoutScripts instanceof Element)) {\n                        throw new Error('documentNode is not an Element');\n                    }*/\n                    // @ts-ignore\n                    documentWithoutScripts.querySelectorAll(\"script\")\n                        .forEach((browserItem: any) => browserItem.remove());\n                    // @ts-ignore\n                    documentWithoutScripts.querySelectorAll(\"style\").forEach((browserItem: any) => browserItem.remove());\n                    // @ts-ignore\n                    documentWithoutScripts.querySelectorAll(\"iframe\").forEach((browserItem: any) => browserItem.remove());\n                    // @ts-ignore\n                    documentWithoutScripts.querySelectorAll(\"noscript\").forEach((browserItem: any) => browserItem.remove());\n\n                    // article content to Markdown\n                    const markdown = turndownService.turndown(\n                        documentWithoutScripts,\n                    );\n\n                    return markdown;\n                }) as unknown as string;\n\n                return md;\n            }\n            return extractPageMarkdown();\n        });\n        await page.close();\n        return md;\n    }\n    isValidUrl(url: string): boolean {\n        return /^(http|https):\\/\\/[^ \"]+$/.test(url);\n    }\n}\n\nasync function test(){\n    const url = \"http://example.com/\"\n    const crawler = new MarkdownCrawler(true)\n    const result = await crawler.crawlFromUrl(url)\n    console.log(result);\n    \n}\ntest()"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBAAyC;AAIlC,IAAM,kBAAN,MAAsB;AAAA,EACzB;AAAA,EACA,WAAoB;AAAA,EACpB,YAAY,WAAoB;AAC5B,SAAK,WAAW;AAAA,EACpB;AAAA,EAEA,MAAM,aACF,KAC6C;AAC7C,QAAI,CAAC,KAAK,WAAW,GAAG,GAAG;AACvB,YAAM,IAAI;AAAA,QACN;AAAA,MACJ;AAAA,IACJ;AAEA,QAAI,CAAE,MAAM,KAAK,cAAc,GAAI;AAC/B,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACtD;AAEA,WAAO,KAAK,mBAAmB,GAAG;AAAA,EACtC;AAAA,EACA,MAAM,iBACF,SACA,eAC4C;AAC5C,QAAI,OAAO,YAAY,UAAU;AAC7B,YAAM,IAAI,MAAM,8CAA8C;AAAA,IAClE;AAEA,QAAI,kBAAkB,UAAU;AAC5B,YAAM,IAAI;AAAA,QACN;AAAA,MACJ;AAAA,IACJ;AACA,QAAI,CAAE,MAAM,KAAK,cAAc,GAAI;AAC/B,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACtD;AAEA,UAAM,OAAO,MAAM,KAAK,kBAAkB,OAAO;AACjD,QAAI,KAAK,SAAS,GAAG;AACjB,aAAO,MAAM,KAAK,sBAAsB,IAAI;AAAA,IAChD,OAAO;AACH,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACrD;AAAA,EACJ;AAAA,EAEA,MAAM,gBAAiC;AACnC,QAAI,UAAU;AACd,WAAO,SAAS;AACZ,UAAI,CAAC,KAAK,WAAW,CAAC,KAAK,QAAQ,WAAW;AAC1C,YAAI;AACA,eAAK,UAAU,MAAM,iBAAAA,QAAU,OAAO,EAAE,UAAU,KAAK,SAAS,CAAC;AACjE,iBAAO;AAAA,QACX,SAAS,GAAG;AACR,kBAAQ;AAAA,YACJ;AAAA,UACJ;AACA;AACA,cAAI,CAAC,SAAS;AACV,mBAAO;AAAA,UACX;AAEA,kBAAQ;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ,OAAO;AACH,eAAO;AAAA,MACX;AAAA,IACJ;AACA,WAAO;AAAA,EACX;AAAA,EACA,MAAM,kBAAkB,SAAmC;AACvD,UAAM,OAAO,MAAM,KAAK,QAAS,QAAQ;AACzC,UAAM,KAAK,KAAK,wBAAwB;AACxC,UAAM,KAAK,KAAK,oBAAoB,OAAO;AAC3C,UAAM,QAAQ,IAAI;AAAA,MACd,KAAK,kBAAkB;AAAA,MACvB,MAAM,KAAK,SAAS,MAAM,OAAO;AAAA;AAAA,IACrC,CAAC;AAGD,UAAM,iBAAiB,YAA0B;AAC7C,YAAM,KAAK,SAAS,YAAY;AAE5B,eAAO,SAAS,GAAG,SAAS,KAAK,YAAY;AAE7C,cAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,GAAI,CAAC;AAAA,MAC5D,CAAC;AAAA,IACL;AAEA,QAAI;AACJ,QAAI,gBAAgB,MAAM,KAAK;AAAA,MAC3B,MAAM,SAAS,KAAK;AAAA,IACxB;AAEA,WAAO,MAAM;AACT,uBAAiB;AACjB,YAAM,eAAe;AACrB,sBAAgB,MAAM,KAAK;AAAA,QACvB,MAAM,SAAS,KAAK;AAAA,MACxB;AAGA,UAAI,kBAAkB,gBAAgB;AAClC;AAAA,MACJ;AAAA,IACJ;AAEA,UAAM,gBAAgB,MAAM,KAAK,SAAS,MAAM;AAC5C,YAAM,UAAU,CAAC;AACjB,YAAM,QAAQ,SAAS,iBAAiB,IAAI;AAC5C,eAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACnC,cAAM,eAAe,MAAM,CAAC;AAC5B,YAAI,cAAc;AACd,gBAAM,OAAO,aAAa,QAAQ,GAAG;AACrC,kBAAQ,KAAK;AAAA,YACT,OAAO,aAAa;AAAA,YACpB,KAAK,OAAO,KAAK,OAAO;AAAA,UAC5B,CAAC;AAAA,QACL;AAAA,MACJ;AACA,aAAO;AAAA,IACX,CAAC;AACD,UAAM,OAAO,cAAc,IAAI,CAAC,WAAW,OAAO,GAAG;AACrD,WAAO;AAAA,EACX;AAAA,EACA,MAAM,sBAAsB,MAA4D;AACpF,UAAM,MAAM,CAAC;AACb,aAAS,QAAQ,GAAG,QAAQ,KAAK,QAAQ,SAAS;AAC9C,YAAM,MAAM,KAAK,KAAK;AACtB,UAAI;AACA,cAAM,WAAW,MAAM,KAAK,oBAAoB,GAAG;AACnD,YAAI,KAAK,EAAE,KAAK,SAAS,CAAC;AAC1B,gBAAQ,IAAI,GAAG,QAAQ,CAAC,IAAI,KAAK,MAAM,gBAAgB,GAAG,EAAE;AAAA,MAChE,SAAS,OAAO;AACZ,gBAAQ;AAAA,UACJ,GAAG,QAAQ,CAAC,IAAI,KAAK,MAAM,kBAAkB,GAAG;AAAA,QACpD;AAAA,MACJ;AAAA,IACJ;AACA,UAAM,KAAK,SAAS,MAAM;AAC1B,WAAO;AAAA,EACX;AAAA,EACA,MAAM,mBAAmB,KAA0D;AAC/E,UAAM,KAAK,MAAM,KAAK,mBAAmB;AAAA,MACrC,MAAM,CAAC,GAAG;AAAA,IACd,CAAC;AACD,UAAM,KAAK,SAAS,MAAM;AAC1B,WAAO;AAAA,MACH,UAAU,GAAG,CAAC,GAAG;AAAA,MACjB,QAAQ;AAAA,IACZ;AAAA,EACJ;AAAA,EAEA,MAAM,aAAa,MAAY,SAAoC;AAC/D,WAAO,MAAM,KAAK,SAAS,CAACC,aAAY;AACpC,aAAO,MAAM,KAAK,SAAS,iBAAiB,GAAG,CAAC,EAC3C,IAAI,CAAC,SAAU,KAA0B,IAAI,EAC7C,OAAO,CAAC,SAAS,KAAK,WAAWA,QAAO,CAAC;AAAA,IAClD,GAAG,OAAO;AAAA,EACd;AAAA,EAEA,MAAM,mBAAmB,EAAE,KAAK,GAG3B;AACD,UAAM,kBAAkB,MAAM,KAAK,cAAc;AAEjD,QAAI,CAAC,iBAAiB;AAClB,YAAM,IAAI,MAAM,kCAAkC;AAAA,IACtD;AAEA,WAAO,MAAM,QAAQ;AAAA,MACjB,KAAK,IAAI,OAAO,QAAQ;AACpB,YAAI;AACA,gBAAM,WAAW,MAAM,KAAK,oBAAoB,GAAG;AACnD,iBAAO,EAAE,KAAK,SAAS;AAAA,QAC3B,SAAS,OAAO;AACZ,kBAAQ,MAAM,KAAK;AACnB,iBAAO,EAAE,KAAK,UAAU,GAAG;AAAA,QAC/B;AAAA,MACJ,CAAC;AAAA,IACL;AAAA,EACJ;AAAA,EAEA,MAAM,oBAAoB,KAA8B;AACpD,UAAM,OAAO,MAAM,KAAK,QAAS,QAAQ;AACzC,UAAM,KAAK,KAAK,KAAK,EAAE,WAAW,eAAe,CAAC;AAClD,UAAM,KAAK,MAAM,KAAK,SAAS,MAAM;AACjC,eAAS,sBAA6B;AAClC,cAAM,oBAAoB,SAAS,cAAc,QAAQ;AACzD,0BAAkB,MACd;AACJ,iBAAS,KAAK,YAAY,iBAAiB;AAE3C,cAAM,iBAAiB,SAAS,cAAc,QAAQ;AACtD,uBAAe,MACX;AACJ,iBAAS,KAAK,YAAY,cAAc;AAExC,YAAIC,MAAK;AAGT,QAAAA,MAAK,QAAQ,IAAI;AAAA,UACb,IAAI;AAAA,YACA,CAAC,YAAa,kBAAkB,SAAS;AAAA,UAC7C;AAAA,UACA,IAAI,QAAQ,CAAC,YAAa,eAAe,SAAS,OAAQ;AAAA,QAC9D,CAAC,EAAE,KAAK,MAAM;AAGV,gBAAM,kBAAkB,IAAI,gBAAgB;AAE5C,gBAAM,yBAAyB,SAAS,UAAU,IAAI;AAKtD,iCAAuB,iBAAiB,QAAQ,EAC3C,QAAQ,CAAC,gBAAqB,YAAY,OAAO,CAAC;AAEvD,iCAAuB,iBAAiB,OAAO,EAAE,QAAQ,CAAC,gBAAqB,YAAY,OAAO,CAAC;AAEnG,iCAAuB,iBAAiB,QAAQ,EAAE,QAAQ,CAAC,gBAAqB,YAAY,OAAO,CAAC;AAEpG,iCAAuB,iBAAiB,UAAU,EAAE,QAAQ,CAAC,gBAAqB,YAAY,OAAO,CAAC;AAGtG,gBAAM,WAAW,gBAAgB;AAAA,YAC7B;AAAA,UACJ;AAEA,iBAAO;AAAA,QACX,CAAC;AAED,eAAOA;AAAA,MACX;AACA,aAAO,oBAAoB;AAAA,IAC/B,CAAC;AACD,UAAM,KAAK,MAAM;AACjB,WAAO;AAAA,EACX;AAAA,EACA,WAAW,KAAsB;AAC7B,WAAO,4BAA4B,KAAK,GAAG;AAAA,EAC/C;AACJ;AAEA,eAAe,OAAM;AACjB,QAAM,MAAM;AACZ,QAAM,UAAU,IAAI,gBAAgB,IAAI;AACxC,QAAM,SAAS,MAAM,QAAQ,aAAa,GAAG;AAC7C,UAAQ,IAAI,MAAM;AAEtB;AACA,KAAK;","names":["puppeteer","baseUrl","md"]}