@crawlee/utils/index.mjs

A set of shared utilities that can be used by crawlers

import mod from "./index.js";

export default mod;
export const CLOUDFLARE_RETRY_CSS_SELECTORS = mod.CLOUDFLARE_RETRY_CSS_SELECTORS;
export const RETRY_CSS_SELECTORS = mod.RETRY_CSS_SELECTORS;
export const ROTATE_PROXY_ERRORS = mod.ROTATE_PROXY_ERRORS;
export const RobotsFile = mod.RobotsFile;
export const RobotsTxtFile = mod.RobotsTxtFile;
export const Sitemap = mod.Sitemap;
export const URL_NO_COMMAS_REGEX = mod.URL_NO_COMMAS_REGEX;
export const URL_WITH_COMMAS_REGEX = mod.URL_WITH_COMMAS_REGEX;
export const applySearchParams = mod.applySearchParams;
export const asyncifyIterable = mod.asyncifyIterable;
export const chunk = mod.chunk;
export const chunkedAsyncIterable = mod.chunkedAsyncIterable;
export const createRequestDebugInfo = mod.createRequestDebugInfo;
export const downloadListOfUrls = mod.downloadListOfUrls;
export const entries = mod.entries;
export const expandShadowRoots = mod.expandShadowRoots;
export const extractUrls = mod.extractUrls;
export const extractUrlsFromCheerio = mod.extractUrlsFromCheerio;
export const getCgroupsVersion = mod.getCgroupsVersion;
export const getCurrentCpuTicksV2 = mod.getCurrentCpuTicksV2;
export const getMemoryInfo = mod.getMemoryInfo;
export const getMemoryInfoV2 = mod.getMemoryInfoV2;
export const getObjectType = mod.getObjectType;
export const gotScraping = mod.gotScraping;
export const htmlToText = mod.htmlToText;
export const isAsyncIterable = mod.isAsyncIterable;
export const isContainerized = mod.isContainerized;
export const isDocker = mod.isDocker;
export const isIterable = mod.isIterable;
export const isLambda = mod.isLambda;
export const keys = mod.keys;
export const parseOpenGraph = mod.parseOpenGraph;
export const parseSitemap = mod.parseSitemap;
export const peekableAsyncIterable = mod.peekableAsyncIterable;
export const sleep = mod.sleep;
export const snakeCaseToCamelCase = mod.snakeCaseToCamelCase;
export const social = mod.social;
export const tryAbsoluteURL = mod.tryAbsoluteURL;
export const weightedAvg = mod.weightedAvg;