semantic-analysis/lib/tools/batch/CrawlAnalyzer.d.ts

Extensible semantic Structured Data analysis library for Microdata and JSON-LD

import { IBatchAnalyzer, IOptions, ITaskDone } from "./IBatchAnalyzer";
import { IVisitLog } from "./VisitLog";
export interface ICrawlOptions extends IOptions {
    visitLog?: IVisitLog;
    visitLogPath?: string;
    perSiteLimit?: number;
}
/**
 * Analyzer crawling through on-site links
 */
export declare class CrawlAnalyzer implements IBatchAnalyzer {
    private readonly options?;
    private readonly batchAnalyzer;
    private readonly visitLog;
    private readonly logReady;
    constructor(options?: Partial<ICrawlOptions> | undefined);
    get queued(): import("./IBatchAnalyzer").ITask[];
    get running(): import("./IBatchAnalyzer").ITaskRunning[];
    get done(): import("./IBatchAnalyzer").ITaskArchived[];
    get failed(): import("./IBatchAnalyzer").ITaskArchived[];
    readonly addTask: (url: string, source?: string | undefined) => Promise<ITaskDone>;
    readonly state: () => {
        queued: import("./IBatchAnalyzer").ITaskArchived[];
        running: import("./IBatchAnalyzer").ITaskArchived[];
        done: import("./IBatchAnalyzer").ITaskArchived[];
        failed: import("./IBatchAnalyzer").ITaskArchived[];
    };
    readonly dispose: () => void;
}