@blocklet/crawler/lib/es/middlewares/utils.d.ts

blocklet crawler lib

import puppeteer from '@blocklet/puppeteer';
export * from '@blocklet/sdk/lib/config';
export declare const api: import("axios").AxiosInstance;
export declare const sleep: (ms: number) => Promise<unknown>;
export declare const clearCheckBrowserTimer: () => void;
export declare const closeBrowser: ({ trimCache }?: {
    trimCache?: boolean;
}) => Promise<void>;
export declare const getBrowser: () => Promise<any>;
export declare const CRAWLER_FLAG = "x-crawler";
export declare const isSelfCrawler: (req: any) => boolean;
export declare const initPage: ({ abortResourceTypes, }?: {
    abortResourceTypes?: string[] | undefined;
}) => Promise<any>;
export declare const getDefaultRobotsUrl: (req: any) => string;
export declare const getDefaultSitemapUrl: (req: any) => string;
export declare const isAcceptCrawler: (req: any) => Promise<boolean>;
export declare const getSitemapList: (req: any) => Promise<any>;
export declare const isBotUserAgent: (req: any) => boolean;
export declare const getBlockletPathname: (req: any) => any;
export declare const getComponentInfo: () => {};
export declare const getFullUrl: (req: any) => string;
export declare const getRelativePath: (url: string) => string;
export declare const cachePool: import("generic-pool").Pool<any>;
export declare const withCache: (cb: Function) => Promise<any>;
export declare const formatKey: (key: string) => string;
export declare const useCache: {
    get: (key: string) => Promise<any>;
    set: (key: string, value: any, options?: any) => Promise<any>;
    remove: (key: string) => Promise<any>;
    list: (key?: string) => Promise<any>;
};
export declare const checkBrowserDownloaded: () => Promise<void>;
export declare const getPuppeteerrc: () => {
    cacheDirectory?: string;
    temporaryDirectory?: string;
};
export { puppeteer };