crawl-to-markdown
Version:
Crawl-to-markdown is a powerful TypeScript package designed to search search engines for a given keyword, crawl the resulting websites, and deliver the content in clean, readable Markdown format. Additionally, it can directly crawl specified websites for
245 lines (244 loc) • 8.36 kB
JavaScript
;
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// src/index.ts
var src_exports = {};
__export(src_exports, {
MarkdownCrawler: () => MarkdownCrawler
});
module.exports = __toCommonJS(src_exports);
var import_puppeteer = __toESM(require("puppeteer"), 1);
var MarkdownCrawler = class {
browser;
headless = false;
constructor(_headless) {
this.headless = _headless;
}
async crawlFromUrl(url) {
if (!this.isValidUrl(url)) {
throw new Error(
"Invalid URL provided, should be a full URL starting with http:// or https://"
);
}
if (!await this.ensureBrowser()) {
throw new Error("Could not start browser instance");
}
return this.crawlSingleWebsite(url);
}
async crawlFromKeyword(keyword, search_engine) {
if (typeof keyword !== "string") {
throw new Error("Invalid Keyword provided, should be a string");
}
if (search_engine !== "google") {
throw new Error(
"only google search engine is supported at the moment"
);
}
if (!await this.ensureBrowser()) {
throw new Error("Could not start browser instance");
}
const urls = await this.crawlSearchEngine(keyword);
if (urls.length > 0) {
return await this.crawllMultipleWebsite(urls);
} else {
throw new Error("Couldn't crawl for this keyword");
}
}
async ensureBrowser() {
let retries = 3;
while (retries) {
if (!this.browser || !this.browser.connected) {
try {
this.browser = await import_puppeteer.default.launch({ headless: this.headless });
return true;
} catch (e) {
console.error(
"Browser DO: Could not start browser instance. Error: ${e}"
);
retries--;
if (!retries) {
return false;
}
console.log(
"Retrying to start browser instance. Retries left: ${retries}"
);
}
} else {
return true;
}
}
return false;
}
async crawlSearchEngine(keyword) {
const page = await this.browser.newPage();
await page.goto("https://www.google.com");
await page.type("textarea[name=q]", keyword);
await Promise.all([
page.waitForNavigation(),
await page.keyboard.press("Enter")
// Action that causes navigation
]);
const scrollToBottom = async () => {
await page.evaluate(async () => {
window.scrollTo(0, document.body.scrollHeight);
await new Promise((resolve) => setTimeout(resolve, 2e3));
});
};
let previousHeight;
let currentHeight = await page.evaluate(
() => document.body.scrollHeight
);
while (true) {
previousHeight = currentHeight;
await scrollToBottom();
currentHeight = await page.evaluate(
() => document.body.scrollHeight
);
if (currentHeight === previousHeight) {
break;
}
}
const searchResults = await page.evaluate(() => {
const results = [];
const items = document.querySelectorAll("h3");
for (let i = 0; i < items.length; i++) {
const titleElement = items[i];
if (titleElement) {
const link = titleElement.closest("a");
results.push({
title: titleElement.innerText,
url: link ? link.href : ""
});
}
}
return results;
});
const urls = searchResults.map((search) => search.url);
return urls;
}
async crawllMultipleWebsite(urls) {
const mds = [];
for (let index = 0; index < urls.length; index++) {
const url = urls[index];
try {
const markdown = await this.fetchAndProcessPage(url);
mds.push({ url, markdown });
console.log(`${index + 1}/${urls.length} done, url : ${url}`);
} catch (error) {
console.error(
`${index + 1}/${urls.length} failed, url : ${url}`
);
}
}
await this.browser?.close();
return mds;
}
async crawlSingleWebsite(url) {
const md = await this.getWebsiteMarkdown({
urls: [url]
});
await this.browser?.close();
return {
markdown: md[0]?.markdown,
status: 200
};
}
async extractLinks(page, baseUrl) {
return await page.evaluate((baseUrl2) => {
return Array.from(document.querySelectorAll("a")).map((link) => link.href).filter((link) => link.startsWith(baseUrl2));
}, baseUrl);
}
async getWebsiteMarkdown({ urls }) {
const isBrowserActive = await this.ensureBrowser();
if (!isBrowserActive) {
throw new Error("Could not start browser instance");
}
return await Promise.all(
urls.map(async (url) => {
try {
const markdown = await this.fetchAndProcessPage(url);
return { url, markdown };
} catch (error) {
console.error(error);
return { url, markdown: "" };
}
})
);
}
async fetchAndProcessPage(url) {
const page = await this.browser.newPage();
await page.goto(url, { waitUntil: "networkidle0" });
const md = await page.evaluate(() => {
function extractPageMarkdown() {
const readabilityScript = document.createElement("script");
readabilityScript.src = "https://unpkg.com/@mozilla/readability/Readability.js";
document.head.appendChild(readabilityScript);
const turndownScript = document.createElement("script");
turndownScript.src = "https://unpkg.com/turndown/dist/turndown.js";
document.head.appendChild(turndownScript);
let md2 = "no content";
md2 = Promise.all([
new Promise(
(resolve) => readabilityScript.onload = resolve
),
new Promise((resolve) => turndownScript.onload = resolve)
]).then(() => {
const turndownService = new TurndownService();
const documentWithoutScripts = document.cloneNode(true);
documentWithoutScripts.querySelectorAll("script").forEach((browserItem) => browserItem.remove());
documentWithoutScripts.querySelectorAll("style").forEach((browserItem) => browserItem.remove());
documentWithoutScripts.querySelectorAll("iframe").forEach((browserItem) => browserItem.remove());
documentWithoutScripts.querySelectorAll("noscript").forEach((browserItem) => browserItem.remove());
const markdown = turndownService.turndown(
documentWithoutScripts
);
return markdown;
});
return md2;
}
return extractPageMarkdown();
});
await page.close();
return md;
}
isValidUrl(url) {
return /^(http|https):\/\/[^ "]+$/.test(url);
}
};
async function test() {
const url = "http://example.com/";
const crawler = new MarkdownCrawler(true);
const result = await crawler.crawlFromUrl(url);
console.log(result);
}
test();
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
MarkdownCrawler
});
//# sourceMappingURL=index.cjs.map