@crawlee/core
Version:
The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.
159 lines (158 loc) • 6.55 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.MemoryLoadSignal = void 0;
const utils_1 = require("@crawlee/utils");
const log_1 = require("../log");
const load_signal_1 = require("./load_signal");
const RESERVE_MEMORY_RATIO = 0.5;
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
/**
* Tracks memory usage via `SYSTEM_INFO` events and reports overload when
* the used-to-available memory ratio exceeds a threshold.
*/
class MemoryLoadSignal {
constructor(options) {
Object.defineProperty(this, "name", {
enumerable: true,
configurable: true,
writable: true,
value: 'memInfo'
});
Object.defineProperty(this, "overloadedRatio", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "store", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "config", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "events", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "log", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "maxUsedMemoryRatio", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "maxMemoryRatio", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "maxMemoryBytes", {
enumerable: true,
configurable: true,
writable: true,
value: void 0
});
Object.defineProperty(this, "lastLoggedCriticalMemoryOverloadAt", {
enumerable: true,
configurable: true,
writable: true,
value: null
});
this.store = new load_signal_1.SnapshotStore(options.snapshotHistoryMillis);
this.config = options.config;
this.events = this.config.getEventManager();
this.log = options.log ?? log_1.log.child({ prefix: 'MemoryLoadSignal' });
this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
this.overloadedRatio = options.overloadedRatio ?? 0.2;
this._onSystemInfo = this._onSystemInfo.bind(this);
}
async start() {
const memoryMbytes = this.config.get('memoryMbytes', 0);
if (memoryMbytes > 0) {
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
}
else {
this.maxMemoryRatio = this.config.get('availableMemoryRatio');
if (!this.maxMemoryRatio) {
throw new Error('availableMemoryRatio is not set in configuration.');
}
else {
this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
}
// Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
this.maxMemoryBytes = await this._getTotalMemoryBytes();
}
this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
}
async stop() {
this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
}
getSample(sampleDurationMillis) {
return this.store.getSample(sampleDurationMillis);
}
/**
* Returns typed memory snapshots for backward compatibility with `Snapshotter`.
*/
getMemorySnapshots() {
return this.store.getAll();
}
/** @internal */
_onSystemInfo(systemInfo) {
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
const { memCurrentBytes, memTotalBytes } = systemInfo;
let maxMemoryBytes = this.maxMemoryBytes;
if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
}
const snapshot = {
createdAt,
isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
usedBytes: memCurrentBytes,
};
this.store.push(snapshot, createdAt);
this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
}
/** @internal */
_memoryOverloadWarning(systemInfo, maxMemoryBytes) {
const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
const { memCurrentBytes } = systemInfo;
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
if (this.lastLoggedCriticalMemoryOverloadAt &&
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
return;
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * effectiveMax;
const reserveMemory = effectiveMax * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
if (isCriticalOverload) {
const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
this.log.warning('Memory is critically overloaded. ' +
`Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
}
}
async _getTotalMemoryBytes() {
if (this.config.get('systemInfoV2')) {
const containerized = this.config.get('containerized', await (0, utils_1.isContainerized)());
return (await (0, utils_1.getMemoryInfoV2)(containerized)).totalBytes;
}
return (await (0, utils_1.getMemoryInfo)()).totalBytes;
}
}
exports.MemoryLoadSignal = MemoryLoadSignal;