@crawlee/core
Version:
The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.
158 lines (157 loc) • 6.41 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SystemStatus = void 0;
const tslib_1 = require("tslib");
const ow_1 = tslib_1.__importDefault(require("ow"));
const load_signal_1 = require("./load_signal");
const snapshotter_1 = require("./snapshotter");
/** The four built-in signal names that map to typed `SystemInfo` fields. */
const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
/**
* Provides a simple interface to reading system status from a {@link Snapshotter} instance.
* It only exposes two functions {@link SystemStatus.getCurrentStatus}
* and {@link SystemStatus.getHistoricalStatus}.
* The system status is calculated using a weighted average of overloaded
* messages in the snapshots, with the weights being the time intervals
* between the snapshots. Each resource is calculated separately
* and the system is overloaded whenever at least one resource is overloaded.
* The class is used by the {@link AutoscaledPool} class.
*
* {@link SystemStatus.getCurrentStatus}
* returns a boolean that represents the current status of the system.
* The length of the current timeframe in seconds is configurable
* by the `currentHistorySecs` option and represents the max age
* of snapshots to be considered for the calculation.
*
* {@link SystemStatus.getHistoricalStatus}
* returns a boolean that represents the long-term status
* of the system. It considers the full snapshot history available
* in the {@link Snapshotter} instance.
* @category Scaling
*/
class SystemStatus {
constructor(options = {}) {
Object.defineProperty(this, "currentHistoryMillis", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "snapshotter", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "signals", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
/**
* Per-signal ratio overrides. The built-in four get their overrides from
* the legacy `max*OverloadedRatio` options; custom signals use their own
* `overloadedRatio`.
*/
Object.defineProperty(this, "ratioOverrides", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
(0, ow_1.default)(options, ow_1.default.object.exactShape({
currentHistorySecs: ow_1.default.optional.number,
maxMemoryOverloadedRatio: ow_1.default.optional.number,
maxEventLoopOverloadedRatio: ow_1.default.optional.number,
maxCpuOverloadedRatio: ow_1.default.optional.number,
maxClientOverloadedRatio: ow_1.default.optional.number,
snapshotter: ow_1.default.optional.object,
loadSignals: ow_1.default.optional.array,
config: ow_1.default.optional.object,
}));
const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], config, } = options;
this.currentHistoryMillis = currentHistorySecs * 1000;
this.snapshotter = snapshotter || new snapshotter_1.Snapshotter({ config });
// Built-in signals from the snapshotter + any custom signals
this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
// Allow legacy options to override the built-in signal ratios
this.ratioOverrides = {
memInfo: maxMemoryOverloadedRatio,
eventLoopInfo: maxEventLoopOverloadedRatio,
cpuInfo: maxCpuOverloadedRatio,
clientInfo: maxClientOverloadedRatio,
};
}
/**
* Returns an {@link SystemInfo} object with the following structure:
*
* ```javascript
* {
* isSystemIdle: Boolean,
* memInfo: Object,
* eventLoopInfo: Object,
* cpuInfo: Object
* }
* ```
*
* Where the `isSystemIdle` property is set to `false` if the system
* has been overloaded in the last `options.currentHistorySecs` seconds,
* and `true` otherwise.
*/
getCurrentStatus() {
return this._isSystemIdle(this.currentHistoryMillis);
}
/**
* Returns an {@link SystemInfo} object with the following structure:
*
* ```javascript
* {
* isSystemIdle: Boolean,
* memInfo: Object,
* eventLoopInfo: Object,
* cpuInfo: Object
* }
* ```
*
* Where the `isSystemIdle` property is set to `false` if the system
* has been overloaded in the full history of the {@link Snapshotter}
* (which is configurable in the {@link Snapshotter}) and `true` otherwise.
*/
getHistoricalStatus() {
return this._isSystemIdle();
}
/**
* Returns a system status object.
*/
_isSystemIdle(sampleDurationMillis) {
const result = {
isSystemIdle: true,
memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
};
let loadSignalInfo;
for (const signal of this.signals) {
const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
const sample = signal.getSample(sampleDurationMillis);
const info = (0, load_signal_1.evaluateLoadSignalSample)(sample, ratio);
if (info.isOverloaded) {
result.isSystemIdle = false;
}
if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
result[signal.name] = info;
}
else {
loadSignalInfo ?? (loadSignalInfo = {});
loadSignalInfo[signal.name] = info;
}
}
if (loadSignalInfo) {
result.loadSignalInfo = loadSignalInfo;
}
return result;
}
}
exports.SystemStatus = SystemStatus;