microfrontier
Version:
A web crawler frontier implementation in TypeScript backed by Redis. MicroFrontier is a scalable and distributed frontier implemented through Redis Queues.
22 lines (21 loc) • 621 B
TypeScript
import IORedis from "ioredis";
import { URLFrontierConfig } from "./URLFrontierConfig";
export declare class FrontendWorker {
redisClient: IORedis.Redis;
config: URLFrontierConfig;
stopped: boolean;
orderedPriority: {
probability: number;
name: string;
}[];
constructor(configuration: {
config: URLFrontierConfig;
redisClient?: IORedis.Redis;
redisOptions?: IORedis.RedisOptions;
});
start(): void;
stop(): void;
private pickPriorityQueueByPrioritizationStrategy;
workOnce(forcedPriority?: string): Promise<boolean>;
private work;
}