omnichron/dist/commoncrawl.mjs

Unified interface for web archive providers

import { createErrorResponse, createFetchOptions, createSuccessResponse, mergeOptions, normalizeDomain, waybackTimestampToISO } from "./_utils.mjs";
import { cleanDoubleSlashes } from "ufo";
import { $fetch } from "ofetch";

//#region src/providers/commoncrawl.ts
/**
* Create a Common Crawl archive provider.
*
* @param initOptions - Initial Common Crawl options (e.g., collection, limit, cache settings).
* @returns ArchiveProvider instance for fetching snapshots from Common Crawl.
*/
function commonCrawl(initOptions = {}) {
	return {
		name: "Common Crawl",
		slug: "commoncrawl",
		async getSnapshots(domain, reqOptions = {}) {
			const options = await mergeOptions(initOptions, reqOptions);
			const baseURL = "https://index.commoncrawl.org";
			const dataBaseURL = "https://data.commoncrawl.org";
			let collectionName = options.collection;
			let indexName;
			if (!collectionName || collectionName === "CC-MAIN-latest") {
				let apiPath;
				try {
					const collinfoOpts = await createFetchOptions(baseURL, {}, { timeout: options.timeout ?? 6e4 });
					const collinfo = await $fetch("/collinfo.json", collinfoOpts);
					if (Array.isArray(collinfo) && collinfo.length > 0) {
						const first = collinfo[0];
						const cdxApiProp = first["cdx-api"] || first.cdxApi;
						if (typeof cdxApiProp === "string") {
							let raw = cdxApiProp.startsWith("http") ? new URL(cdxApiProp).pathname : cdxApiProp;
							raw = raw.startsWith("/") ? raw.slice(1) : raw;
							apiPath = raw;
							collectionName = raw.endsWith("-index") ? raw.slice(0, -6) : raw;
						} else if (typeof first.name === "string") {
							collectionName = first.name;
							apiPath = collectionName.endsWith("-index") ? collectionName : `${collectionName}-index`;
						}
					}
				} catch {}
				if (!collectionName) collectionName = "CC-MAIN-latest";
				if (!apiPath) apiPath = collectionName.endsWith("-index") ? collectionName : `${collectionName}-index`;
				indexName = apiPath;
			} else indexName = collectionName.endsWith("-index") ? collectionName : `${collectionName}-index`;
			const urlPattern = normalizeDomain(domain);
			const params = {
				url: urlPattern,
				output: "json",
				fl: "url,timestamp,status,mime,length,offset,filename,digest",
				collapse: "digest",
				limit: String(options.limit ?? 1e3)
			};
			const fetchOptions = await createFetchOptions(baseURL, params, {
				timeout: options.timeout ?? 6e4,
				responseType: "text"
			});
			try {
				const raw = await $fetch(`/${indexName}`, fetchOptions);
				const text = typeof raw === "string" ? raw : String(raw);
				const lines = text.split("\n").filter((line) => line.trim());
				if (lines.length === 0) return createSuccessResponse([], "commoncrawl", {
					collection: collectionName,
					queryParams: fetchOptions.params
				});
				const records = lines.map((line) => JSON.parse(line));
				const pages = records.map((record) => {
					const isoTimestamp = waybackTimestampToISO(record.timestamp || "");
					const cleanedUrl = cleanDoubleSlashes(record.url || "");
					const snapUrl = `${dataBaseURL}/${record.filename}`;
					return {
						url: cleanedUrl,
						timestamp: isoTimestamp,
						snapshot: snapUrl,
						_meta: {
							timestamp: record.timestamp,
							status: Number.parseInt(record.status || "0", 10),
							digest: record.digest,
							mime: record.mime,
							length: record.length,
							offset: record.offset,
							filename: record.filename,
							collection: collectionName,
							provider: "commoncrawl"
						}
					};
				});
				return createSuccessResponse(pages, "commoncrawl", {
					collection: collectionName,
					count: pages.length,
					queryParams: fetchOptions.params
				});
			} catch (error) {
				return createErrorResponse(error, "commoncrawl", { collection: collectionName });
			}
		}
	};
}

//#endregion
export { commonCrawl as default };