crawl-to-markdown
Version:
Crawl-to-markdown is a powerful TypeScript package designed to search search engines for a given keyword, crawl the resulting websites, and deliver the content in clean, readable Markdown format. Additionally, it can directly crawl specified websites for
210 lines • 6.71 kB
JavaScript
// src/index.ts
import puppeteer from "puppeteer";
var MarkdownCrawler = class {
browser;
headless = false;
constructor(_headless) {
this.headless = _headless;
}
async crawlFromUrl(url) {
if (!this.isValidUrl(url)) {
throw new Error(
"Invalid URL provided, should be a full URL starting with http:// or https://"
);
}
if (!await this.ensureBrowser()) {
throw new Error("Could not start browser instance");
}
return this.crawlSingleWebsite(url);
}
async crawlFromKeyword(keyword, search_engine) {
if (typeof keyword !== "string") {
throw new Error("Invalid Keyword provided, should be a string");
}
if (search_engine !== "google") {
throw new Error(
"only google search engine is supported at the moment"
);
}
if (!await this.ensureBrowser()) {
throw new Error("Could not start browser instance");
}
const urls = await this.crawlSearchEngine(keyword);
if (urls.length > 0) {
return await this.crawllMultipleWebsite(urls);
} else {
throw new Error("Couldn't crawl for this keyword");
}
}
async ensureBrowser() {
let retries = 3;
while (retries) {
if (!this.browser || !this.browser.connected) {
try {
this.browser = await puppeteer.launch({ headless: this.headless });
return true;
} catch (e) {
console.error(
"Browser DO: Could not start browser instance. Error: ${e}"
);
retries--;
if (!retries) {
return false;
}
console.log(
"Retrying to start browser instance. Retries left: ${retries}"
);
}
} else {
return true;
}
}
return false;
}
async crawlSearchEngine(keyword) {
const page = await this.browser.newPage();
await page.goto("https://www.google.com");
await page.type("textarea[name=q]", keyword);
await Promise.all([
page.waitForNavigation(),
await page.keyboard.press("Enter")
// Action that causes navigation
]);
const scrollToBottom = async () => {
await page.evaluate(async () => {
window.scrollTo(0, document.body.scrollHeight);
await new Promise((resolve) => setTimeout(resolve, 2e3));
});
};
let previousHeight;
let currentHeight = await page.evaluate(
() => document.body.scrollHeight
);
while (true) {
previousHeight = currentHeight;
await scrollToBottom();
currentHeight = await page.evaluate(
() => document.body.scrollHeight
);
if (currentHeight === previousHeight) {
break;
}
}
const searchResults = await page.evaluate(() => {
const results = [];
const items = document.querySelectorAll("h3");
for (let i = 0; i < items.length; i++) {
const titleElement = items[i];
if (titleElement) {
const link = titleElement.closest("a");
results.push({
title: titleElement.innerText,
url: link ? link.href : ""
});
}
}
return results;
});
const urls = searchResults.map((search) => search.url);
return urls;
}
async crawllMultipleWebsite(urls) {
const mds = [];
for (let index = 0; index < urls.length; index++) {
const url = urls[index];
try {
const markdown = await this.fetchAndProcessPage(url);
mds.push({ url, markdown });
console.log(`${index + 1}/${urls.length} done, url : ${url}`);
} catch (error) {
console.error(
`${index + 1}/${urls.length} failed, url : ${url}`
);
}
}
await this.browser?.close();
return mds;
}
async crawlSingleWebsite(url) {
const md = await this.getWebsiteMarkdown({
urls: [url]
});
await this.browser?.close();
return {
markdown: md[0]?.markdown,
status: 200
};
}
async extractLinks(page, baseUrl) {
return await page.evaluate((baseUrl2) => {
return Array.from(document.querySelectorAll("a")).map((link) => link.href).filter((link) => link.startsWith(baseUrl2));
}, baseUrl);
}
async getWebsiteMarkdown({ urls }) {
const isBrowserActive = await this.ensureBrowser();
if (!isBrowserActive) {
throw new Error("Could not start browser instance");
}
return await Promise.all(
urls.map(async (url) => {
try {
const markdown = await this.fetchAndProcessPage(url);
return { url, markdown };
} catch (error) {
console.error(error);
return { url, markdown: "" };
}
})
);
}
async fetchAndProcessPage(url) {
const page = await this.browser.newPage();
await page.goto(url, { waitUntil: "networkidle0" });
const md = await page.evaluate(() => {
function extractPageMarkdown() {
const readabilityScript = document.createElement("script");
readabilityScript.src = "https://unpkg.com/@mozilla/readability/Readability.js";
document.head.appendChild(readabilityScript);
const turndownScript = document.createElement("script");
turndownScript.src = "https://unpkg.com/turndown/dist/turndown.js";
document.head.appendChild(turndownScript);
let md2 = "no content";
md2 = Promise.all([
new Promise(
(resolve) => readabilityScript.onload = resolve
),
new Promise((resolve) => turndownScript.onload = resolve)
]).then(() => {
const turndownService = new TurndownService();
const documentWithoutScripts = document.cloneNode(true);
documentWithoutScripts.querySelectorAll("script").forEach((browserItem) => browserItem.remove());
documentWithoutScripts.querySelectorAll("style").forEach((browserItem) => browserItem.remove());
documentWithoutScripts.querySelectorAll("iframe").forEach((browserItem) => browserItem.remove());
documentWithoutScripts.querySelectorAll("noscript").forEach((browserItem) => browserItem.remove());
const markdown = turndownService.turndown(
documentWithoutScripts
);
return markdown;
});
return md2;
}
return extractPageMarkdown();
});
await page.close();
return md;
}
isValidUrl(url) {
return /^(http|https):\/\/[^ "]+$/.test(url);
}
};
async function test() {
const url = "http://example.com/";
const crawler = new MarkdownCrawler(true);
const result = await crawler.crawlFromUrl(url);
console.log(result);
}
test();
export {
MarkdownCrawler
};
//# sourceMappingURL=index.js.map