crawler
Version:
Crawler is a ready-to-use web spider that works with proxies, asynchrony, rate limit, configurable request pools, jQuery, and HTTP/2 support.
39 lines • 1.1 kB
TypeScript
import Cluster from "./cluster.js";
export type Task = {
(done: () => void, limiter?: number): void;
};
export type TaskWrapper = {
next: Task;
rateLimiterId?: number;
};
export type RateLimiterOptions = {
maxConnections: number;
rateLimit: number;
priorityLevels: number;
defaultPriority: number;
cluster?: Cluster;
};
declare class RateLimiter {
private _waitingTasks;
private _cluster?;
id?: number;
maxConnections: number;
nextRequestTime: number;
rateLimit: number;
runningSize: number;
priorityLevels: number;
defaultPriority: number;
constructor({ maxConnections, rateLimit, priorityLevels, defaultPriority, cluster }: RateLimiterOptions);
get waitingSize(): number;
hasWaitingTasks(): boolean;
setId(id: number): void;
setRateLimit(rateLimit: number): void;
submit(options: {
priority: number;
} | number, task: Task): void;
private _schedule;
directDequeue(): Task;
dequeue(): TaskWrapper | undefined;
}
export default RateLimiter;
//# sourceMappingURL=rateLimiter.d.ts.map