@crawlee/utils
Version:
A set of shared utilities that can be used by crawlers
122 lines • 5.5 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.getMemoryInfoV2 = getMemoryInfoV2;
const tslib_1 = require("tslib");
const node_child_process_1 = require("node:child_process");
const promises_1 = require("node:fs/promises");
const node_os_1 = require("node:os");
const log_1 = tslib_1.__importDefault(require("@apify/log"));
const general_1 = require("../general");
const ps_tree_1 = require("./ps-tree");
const MEMORY_FILE_PATHS = {
TOTAL: {
V1: '/sys/fs/cgroup/memory/memory.limit_in_bytes',
V2: '/sys/fs/cgroup/memory.max',
},
USED: {
V1: '/sys/fs/cgroup/memory/memory.usage_in_bytes',
V2: '/sys/fs/cgroup/memory.current',
},
};
/**
* Returns memory statistics of the process and the system, see {@link MemoryInfo}.
*
* If the process runs inside of a container, the `getMemoryInfo` gets container memory limits,
* otherwise it gets system memory limits.
*
* Beware that the function is quite inefficient because it spawns a new process.
* Therefore you shouldn't call it too often, like more than once per second.
* @returns An object containing the free and used memory metrics.
* @internal
*/
async function getMemoryInfoV2(containerized = false) {
let mainProcessBytes = -1;
let childProcessesBytes = 0;
// lambda does *not* have `ps` and other command line tools
// required to extract memory usage.
if ((0, general_1.isLambda)()) {
// reported in bytes
mainProcessBytes = process.memoryUsage().rss;
// https://stackoverflow.com/a/55914335/129415
const memInfo = (0, node_child_process_1.execSync)('cat /proc/meminfo').toString();
const values = memInfo.split(/[\n: ]/).filter((val) => val.trim());
// /proc/meminfo reports in kb, not bytes, the total used memory is reported by meminfo
// subtract memory used by the main node process in order to infer memory used by any child processes
childProcessesBytes = +values[19] * 1000 - mainProcessBytes;
}
else {
// Query both root and child processes
const processes = await (0, ps_tree_1.psTree)(process.pid, true);
processes.forEach((rec) => {
// Obtain main process' memory separately
if (rec.PID === `${process.pid}`) {
mainProcessBytes = rec.RSS;
return;
}
childProcessesBytes += rec.RSS;
});
}
let totalBytes;
let usedBytes;
let freeBytes;
if ((0, general_1.isLambda)()) {
// memory size is defined in megabytes
totalBytes = parseInt(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE, 10) * 1000000;
usedBytes = mainProcessBytes + childProcessesBytes;
freeBytes = totalBytes - usedBytes;
log_1.default.debug(`lambda size of ${totalBytes} with ${freeBytes} free bytes`);
}
else if (containerized) {
// When running inside a container, use container memory limits
const cgroupsVersion = await (0, general_1.getCgroupsVersion)();
try {
if (cgroupsVersion === null) {
throw new Error('cgroup not available');
}
let [totalBytesStr, usedBytesStr] = await Promise.all([
(0, promises_1.readFile)(MEMORY_FILE_PATHS.TOTAL[cgroupsVersion], 'utf8'),
(0, promises_1.readFile)(MEMORY_FILE_PATHS.USED[cgroupsVersion], 'utf8'),
]);
// Cgroups V2 files contains newline character. Getting rid of it for better handling in later part of the code.
totalBytesStr = totalBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
usedBytesStr = usedBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
// Cgroups V2 contains 'max' string if memory is not limited
// See https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git/tree/Documentation/admin-guide/cgroup-v2.rst (see "memory.max")
if (totalBytesStr === 'max') {
totalBytes = (0, node_os_1.totalmem)();
// Cgroups V1 is set to number related to platform and page size if memory is not limited
// See https://unix.stackexchange.com/q/420906
}
else {
totalBytes = parseInt(totalBytesStr, 10);
const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
if (containerRunsWithUnlimitedMemory)
totalBytes = (0, node_os_1.totalmem)();
}
usedBytes = parseInt(usedBytesStr, 10);
freeBytes = totalBytes - usedBytes;
}
catch (err) {
// log.deprecated logs a warning only once
log_1.default.deprecated('Your environment is containerized, but your system does not support memory cgroups. ' +
"If you're running containers with limited memory, memory auto-scaling will not work properly.\n\n" +
`Cause: ${err.message}`);
totalBytes = (0, node_os_1.totalmem)();
freeBytes = (0, node_os_1.freemem)();
usedBytes = totalBytes - freeBytes;
}
}
else {
totalBytes = (0, node_os_1.totalmem)();
freeBytes = (0, node_os_1.freemem)();
usedBytes = totalBytes - freeBytes;
}
return {
totalBytes,
freeBytes,
usedBytes,
mainProcessBytes,
childProcessesBytes,
};
}
//# sourceMappingURL=memory-info.js.map