@crawlee/core/events/local_event_manager.js

The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.LocalEventManager = void 0;
const tslib_1 = require("tslib");
const node_os_1 = tslib_1.__importDefault(require("node:os"));
const utils_1 = require("@crawlee/utils");
const log_1 = tslib_1.__importDefault(require("@apify/log"));
const utilities_1 = require("@apify/utilities");
const event_manager_1 = require("./event_manager");
class LocalEventManager extends event_manager_1.EventManager {
    constructor() {
        super(...arguments);
        Object.defineProperty(this, "previousTicks", {
            enumerable: true,
            configurable: true,
            writable: true,
            value: { idle: 0, total: 0 }
        });
    }
    /**
     * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
     * This is automatically called at the beginning of `crawler.run()`.
     */
    async init() {
        if (this.initialized) {
            return;
        }
        await super.init();
        const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
        this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
        this.intervals.systemInfo = (0, utilities_1.betterSetInterval)(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
    }
    /**
     * @inheritDoc
     */
    async close() {
        if (!this.initialized) {
            return;
        }
        await super.close();
        (0, utilities_1.betterClearInterval)(this.intervals.systemInfo);
    }
    /**
     * @internal
     */
    async emitSystemInfoEvent(intervalCallback) {
        const info = await this.createSystemInfo({
            maxUsedCpuRatio: this.config.get('maxUsedCpuRatio'),
        });
        this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
        intervalCallback();
    }
    /**
     * @internal
     */
    async isContainerizedWrapper() {
        return this.config.get('containerized', await (0, utils_1.isContainerized)());
    }
    getCurrentCpuTicks() {
        const cpus = node_os_1.default.cpus();
        return cpus.reduce((acc, cpu) => {
            const cpuTimes = Object.values(cpu.times);
            return {
                idle: acc.idle + cpu.times.idle,
                total: acc.total + cpuTimes.reduce((sum, num) => sum + num),
            };
        }, { idle: 0, total: 0 });
    }
    /**
     * Creates a SystemInfo object based on local metrics.
     */
    async createSystemInfo(options) {
        return {
            createdAt: new Date(),
            ...(await this.createCpuInfo(options)),
            ...(await this.createMemoryInfo()),
        };
    }
    async createCpuInfo(options) {
        if (this.config.get('systemInfoV2')) {
            const usedCpuRatio = await (0, utils_1.getCurrentCpuTicksV2)(await this.isContainerizedWrapper());
            return {
                cpuCurrentUsage: usedCpuRatio * 100,
                isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
            };
        }
        const ticks = this.getCurrentCpuTicks();
        const idleTicksDelta = ticks.idle - this.previousTicks.idle;
        const totalTicksDelta = ticks.total - this.previousTicks.total;
        const usedCpuRatio = totalTicksDelta ? 1 - idleTicksDelta / totalTicksDelta : 0;
        Object.assign(this.previousTicks, ticks);
        return {
            cpuCurrentUsage: usedCpuRatio * 100,
            isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
        };
    }
    async createMemoryInfo() {
        try {
            if (this.config.get('systemInfoV2')) {
                const memInfo = await (0, utils_1.getMemoryInfoV2)(await this.isContainerizedWrapper());
                return {
                    memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
                };
            }
            const memInfo = await (0, utils_1.getMemoryInfo)();
            return {
                memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
            };
        }
        catch (err) {
            log_1.default.exception(err, 'Memory snapshot failed.');
            return {};
        }
    }
}
exports.LocalEventManager = LocalEventManager;
//# sourceMappingURL=local_event_manager.js.map