@crawlee/core/autoscaling/client_load_signal.js

The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.createClientLoadSignal = createClientLoadSignal;
const load_signal_1 = require("./load_signal");
const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
/**
 * Periodically checks the storage client for rate-limit errors (HTTP 429)
 * and reports overload when the error delta exceeds a threshold.
 */
function createClientLoadSignal(options) {
    const maxClientErrors = options.maxClientErrors ?? 3;
    const signal = load_signal_1.SnapshotStore.fromInterval({
        name: 'clientInfo',
        overloadedRatio: options.overloadedRatio ?? 0.3,
        intervalMillis: (options.clientSnapshotIntervalSecs ?? 1) * 1000,
        snapshotHistoryMillis: options.snapshotHistoryMillis,
        handler(store, intervalCallback) {
            const now = new Date();
            const allErrorCounts = options.client.stats?.rateLimitErrors ?? [];
            const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
            const snapshot = {
                createdAt: now,
                isOverloaded: false,
                rateLimitErrorCount: currentErrCount,
            };
            const all = store.getAll();
            const previousSnapshot = all[all.length - 1];
            if (previousSnapshot) {
                const { rateLimitErrorCount } = previousSnapshot;
                const delta = currentErrCount - rateLimitErrorCount;
                if (delta > maxClientErrors)
                    snapshot.isOverloaded = true;
            }
            store.push(snapshot, now);
            intervalCallback();
        },
    });
    return signal;
}