mongodb-simplecrawler-queue/lib/src/utils/operations.d.ts

MongoDB FetchQueue Implementation for Simplecrawler

import { Collection } from 'mongodb';
import { MongoQueueItem } from '../typings';
/**
 * Aggregation result object used for monitoring tasks
 * Collects all statistic data about state of the queue at the moment
 */
export interface AggregationResult {
    actualDataSizeMax: number;
    contentLengthMax: number;
    downloadTimeMax: number;
    requestLatencyMax: number;
    requestTimeMax: number;
    actualDataSizeMin: number;
    contentLengthMin: number;
    downloadTimeMin: number;
    requestLatencyMin: number;
    requestTimeMin: number;
    actualDataSizeAvg: number;
    contentLengthAvg: number;
    downloadTimeAvg: number;
    requestLatencyAvg: number;
    requestTimeAvg: number;
    queued: number;
    downloaded: number;
    headers: number;
    spooled: number;
    downloadprevented: number;
    timeout: number;
    created: number;
    failed: number;
    notfound: number;
    redirected: number;
    pulled: number;
    totalCount: number;
    fetchedCount: number;
    timestamp: number;
    timestampFinish: number;
    crawlers: [];
}
/**
 * Operations class - single tasks operation
 */
export declare class Operations {
    /**
     * Single Monitoring task operation - collect all statistic data about state of the queue
     * and put it to statistic collection as new Item
     *
     * @param queueCollection source QueueCollection of {@link QueueItem}
     * @param statisticCollection destination Statistic Collection of {@link AggregationResult} items
     */
    static monitorTask(queueCollection: Collection<MongoQueueItem>, statisticCollection: Collection<MongoQueueItem>): Promise<AggregationResult>;
    /**
     * Single garbage collector task. Roll back all spooled, but not fetched items with "old" modification timestamp
     *
     * @param queueCollection source QueueCollection of {@link QueueItem}
     * @param invalidPeriodMs invalid period in milliseconds, after this amount of ms item is considered as "old"
     */
    static gcTask(queueCollection: Collection<MongoQueueItem>, invalidPeriodMs: number): Promise<import("mongodb").UpdateWriteOpResult>;
}
//# sourceMappingURL=operations.d.ts.map