@crawlee/core
Version:
The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.
100 lines (99 loc) • 3.93 kB
TypeScript
import type { EventManager, EventTypeName } from '../events/event_manager';
import type { ClientInfo } from './system_status';
/**
* A snapshot of a resource's overload state at a point in time.
*/
export interface LoadSnapshot {
createdAt: Date;
isOverloaded: boolean;
}
/**
* A signal that reports whether a particular resource is overloaded.
*
* `SystemStatus` aggregates multiple `LoadSignal` instances to determine
* overall system health. The built-in signals cover memory, CPU, event loop,
* and API client rate limits. You can implement this interface to add
* custom overload signals (e.g. navigation timeouts, proxy health).
*/
export interface LoadSignal {
/** Human-readable name used in logging and `SystemInfo` keys. */
readonly name: string;
/**
* Maximum ratio of overloaded snapshots in a sample before the signal
* is considered overloaded. For example, `0.2` means the signal fires
* when more than 20% of the sample window is overloaded.
*/
readonly overloadedRatio: number;
/** Start collecting snapshots. Called when the pool starts. */
start(): Promise<void>;
/** Stop collecting snapshots. Called when the pool shuts down. */
stop(): Promise<void>;
/**
* Return snapshots for a recent time window (used for "current" status).
* @param sampleDurationMillis How far back to look, in milliseconds.
*/
getSample(sampleDurationMillis?: number): LoadSnapshot[];
}
/**
* A time-pruning, time-windowed store for `LoadSnapshot` values.
* Signals compose with this instead of inheriting from a base class.
*/
export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
private snapshots;
private readonly historyMillis;
constructor(historyMillis?: number);
/**
* Add a snapshot and prune entries older than the history window.
*/
push(snapshot: T, now?: Date): void;
/**
* Return all snapshots, or only those within the given time window.
*/
getSample(sampleDurationMillis?: number): T[];
/**
* Direct access to the underlying array (for backward-compat getters).
*/
getAll(): T[];
/**
* Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
*
* The `handler` receives the store (to read previous snapshots) and the
* interval callback (which it **must** call when done). It should call
* `store.push()` to record a snapshot.
*/
static fromInterval<T extends LoadSnapshot>(options: {
name: string;
overloadedRatio: number;
intervalMillis: number;
snapshotHistoryMillis?: number;
handler: (store: SnapshotStore<T>, intervalCallback: () => unknown) => void;
}): Omit<LoadSignal, 'getSample'> & {
store: SnapshotStore<T>;
handle: (cb: () => unknown) => void;
getSample(sampleDurationMillis?: number): T[];
};
/**
* Create a `LoadSignal` that snapshots in response to an `EventManager` event.
*
* The `handler` receives the event payload and the store. It should call
* `store.push()` to record a snapshot.
*/
static fromEvent<T extends LoadSnapshot, E>(options: {
name: string;
overloadedRatio: number;
events: EventManager;
event: EventTypeName;
snapshotHistoryMillis?: number;
handler: (store: SnapshotStore<T>, payload: E) => void;
}): Omit<LoadSignal, 'getSample'> & {
store: SnapshotStore<T>;
handle: (payload: E) => void;
getSample(sampleDurationMillis?: number): T[];
};
}
/**
* Evaluate whether a sample of `LoadSnapshot` values exceeds the given
* overloaded ratio, using a time-weighted average. This is the shared
* evaluation logic used by `SystemStatus` for all signal types.
*/
export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;