@blocklet/crawler/lib/es/middlewares/crawler.d.ts

blocklet crawler lib

import { getRelativePath } from './utils';
import { PageOptions } from './types';
export { getRelativePath };
export declare const getPageContent: ({ url, formatPageContent }: PageOptions) => Promise<string>;
export declare const getUrlInfoFromCache: (url: string) => Promise<any>;
export declare const setUrlInfoToCache: ({ url, content, lastmod, nextDate, }: {
    url: string;
    content: string;
    lastmod?: string;
    nextDate?: string;
}) => Promise<any>;
export declare const crawlUrl: ({ urls, lastmodMap, formatPageContent, autoCloseBrowserCount, }: {
    urls: string[] | string;
    lastmodMap?: Map<string, string>;
    formatPageContent?: Function | null | undefined;
    autoCloseBrowserCount?: number;
}) => Promise<void>;
export declare const crawlBlocklet: () => Promise<void>;
export declare const initCronCrawlBlocklet: ({ time, options, }?: {
    time: string;
    options: any;
}) => any;
export declare const cancelCronCrawlBlocklet: () => void;