pm2-autoscale
Version:
PM2 module to help dynamically scale applications based on utilization demand
250 lines (249 loc) • 11.9 kB
JavaScript
;
/// <reference path="../@types/global.d.ts" />
/// <reference path="../@types/pm2.d.ts" />
Object.defineProperty(exports, "__esModule", { value: true });
exports.startPm2Connect = void 0;
const tslib_1 = require("tslib");
const pm2_1 = tslib_1.__importDefault(require("pm2"));
const node_os_1 = tslib_1.__importDefault(require("node:os"));
const pidusage_1 = tslib_1.__importDefault(require("pidusage"));
const app_1 = require("./app");
const utils_1 = require("../utils");
const logger_1 = require("../utils/logger");
const cpu_1 = require("../utils/cpu");
const WORKER_CHECK_INTERVAL = 1000;
const SHOW_STAT_INTERVAL = 10000;
const MEMORY_MB = 1048576;
const TOTAL_CPUS = (0, cpu_1.getCpuCount)();
const DEFAULT_MIN_SECONDS_TO_ADD_WORKER = 10;
const DEFAULT_MIN_SECONDS_TO_RELEASE_WORKER = 30;
const DEFAULT_MAX_AVAILABLE_WORKERS_COUNT = TOTAL_CPUS - 1;
const APPS = {};
const isMonitoringApp = (app) => {
const pm2_env = app.pm2_env;
if (pm2_env.axm_options.isModule ||
!app.name ||
!app.pid ||
app.pm_id === undefined || // pm_id might be zero
pm2_env.status !== 'online') {
return false;
}
return true;
};
const updateAppPidsData = (workingApp, pidData) => {
workingApp.updatePid({
id: pidData.id,
memory: Math.round((pidData.memory || 0) / MEMORY_MB),
cpu: pidData.cpu || 0,
pmId: pidData.pmId,
});
};
const getPm2AutoscaleConfig = (app) => {
const pm2_env = app.pm2_env;
const logger = (0, logger_1.getLogger)();
let config = {};
if (pm2_env.env.pm2_autoscale) {
try {
config = JSON.parse(pm2_env.env.pm2_autoscale);
}
catch (error) {
logger.debug(`Error: Can not parse "pm2_autoscale" env config for app ${app.name}`);
}
}
return config;
};
const detectActiveApps = (conf) => {
const logger = (0, logger_1.getLogger)();
pm2_1.default.list((err, apps) => {
if (err)
return console.error(err.stack || err);
const pidsMonit = {};
const mapAppData = {};
const appsToIgnore = (conf.ignore_apps ? conf.ignore_apps.split(',') : []).map((entry) => entry.trim());
// Fill all available apps pids
apps.forEach((app) => {
const appName = app.name;
if (!isMonitoringApp(app) || !appName || !app.pid || app.pm_id === undefined) {
return;
}
if (appsToIgnore.includes(appName)) {
logger.debug(`App "${appName}" is in ignore list`);
delete APPS[appName];
return;
}
const appConfig = getPm2AutoscaleConfig(app);
if (appConfig.is_enabled === false) {
logger.debug(`Autoscale module is disabled for the app "${appName}"`);
delete APPS[appName];
return;
}
const pm2Env = app.pm2_env;
if (!mapAppData[appName]) {
mapAppData[appName] = {
pids: [],
pm2Env,
appConfig,
};
}
mapAppData[appName].pids.push(app.pid);
// Fill monitoring data
pidsMonit[app.pid] = { cpu: 0, memory: 0, pmId: app.pm_id, id: app.pid };
});
// Filters existed apps which do not have active pids
Object.keys(APPS).forEach((appName) => {
const processingApp = mapAppData[appName];
if (!processingApp) {
logger.debug(`Delete ${appName} because it not longer exists`);
delete APPS[appName];
}
else {
const workingApp = APPS[appName];
if (workingApp) {
const activePids = processingApp.pids;
workingApp.removeNotActivePids(activePids);
// Set new config data
workingApp.setAppConfig(mapAppData[appName].appConfig);
}
}
});
// Create new apps if not exist
for (const [appName, entry] of Object.entries(mapAppData)) {
if (entry.pids.length && !APPS[appName]) {
APPS[appName] = new app_1.App(appName, entry.pm2Env.instances);
}
}
// Get all pids to monit
const pids = Object.keys(pidsMonit);
if (pids.length) {
// Get real pids data.
// !ATTENTION! Can not use PM2 app.monit because of incorrect values of CPU usage
(0, pidusage_1.default)(pids, (err, stats) => {
if (err)
return console.error(err.stack || err);
// Fill data for all pids
if (stats && Object.keys(stats).length) {
for (const [pid, stat] of Object.entries(stats)) {
const pidId = Number(pid);
if (pidId && pidsMonit[pidId]) {
pidsMonit[pidId].cpu = Math.round(stat.cpu * 10) / 10;
pidsMonit[pidId].memory = stat.memory;
}
}
}
for (const [appName, entry] of Object.entries(mapAppData)) {
const workingApp = APPS[appName];
if (workingApp) {
entry.pids.forEach((pidId) => {
const monit = pidsMonit[pidId];
if (monit) {
updateAppPidsData(workingApp, monit);
}
});
// Processing...
processWorkingApp(conf, workingApp);
}
}
});
}
});
};
const startPm2Connect = (conf) => {
pm2_1.default.connect((err) => {
if (err)
return console.error(err.stack || err);
setInterval(() => {
detectActiveApps(conf);
}, WORKER_CHECK_INTERVAL);
if (conf.debug) {
setInterval(() => {
(0, logger_1.getLogger)().debug(`System: Free memory ${(0, utils_1.handleUnit)(node_os_1.default.freemem())}, Total memory ${(0, utils_1.handleUnit)(node_os_1.default.totalmem())}, CPU available: ${TOTAL_CPUS}`);
if (Object.keys(APPS).length) {
for (const [, app] of Object.entries(APPS)) {
(0, logger_1.getLogger)().debug(`App "${app.getName()}" has ${app.getActiveWorkersCount()} worker(s). CPU: ${app.getCpuThreshold()}, Memory: ${app.getTotalUsedMemory()}MB`);
}
}
else {
(0, logger_1.getLogger)().debug(`No apps available`);
}
}, SHOW_STAT_INTERVAL);
}
});
};
exports.startPm2Connect = startPm2Connect;
function processWorkingApp(conf, workingApp) {
var _a, _b, _c, _d, _e, _f, _g;
if (workingApp.isProcessing) {
(0, logger_1.getLogger)().debug(`App "${workingApp.getName()}" is busy`);
return;
}
const cpuValues = [...workingApp.getCpuThreshold()];
const cpuValuesString = cpuValues.join(',');
const maxCpuValue = Math.max(...workingApp.getCpuThreshold());
const averageCpuValue = Math.round(cpuValues.reduce((sum, value) => sum + value) / cpuValues.length);
const scaleCpuThreshold = (_a = workingApp.getAppConfig().scale_cpu_threshold) !== null && _a !== void 0 ? _a : conf.scale_cpu_threshold;
const releaseCpuThreshold = (_b = workingApp.getAppConfig().release_cpu_threshold) !== null && _b !== void 0 ? _b : conf.release_cpu_threshold;
const configWorkers = (_c = workingApp.getAppConfig().max_workers) !== null && _c !== void 0 ? _c : conf.max_workers;
const parsedConfigWorkers = parseInt(String(configWorkers), 10);
let maxWorkers = DEFAULT_MAX_AVAILABLE_WORKERS_COUNT;
if (configWorkers === 'max' || parsedConfigWorkers === 0) {
maxWorkers = TOTAL_CPUS;
}
else if (!isNaN(parsedConfigWorkers) && parsedConfigWorkers > 0) {
maxWorkers = parsedConfigWorkers;
}
if (workingApp.getActiveWorkersCount() >= maxWorkers) {
(0, logger_1.getLogger)().debug(`App "${workingApp.getName()}" is reached max workers "${maxWorkers}. CPUs: ${cpuValuesString}"`);
}
const needIncreaseWorkers =
// Increase workers if any of CPUs loaded more then "scaleCpuThreshold"
maxCpuValue >= scaleCpuThreshold &&
// Increase workers only if we have available CPUs for that
workingApp.getActiveWorkersCount() < maxWorkers;
const minSecondsToAddWorker = (_e = (_d = workingApp.getAppConfig().min_seconds_to_add_worker) !== null && _d !== void 0 ? _d : conf.min_seconds_to_add_worker) !== null && _e !== void 0 ? _e : DEFAULT_MIN_SECONDS_TO_ADD_WORKER;
const minSecondsToReleaseWorker = (_g = (_f = workingApp.getAppConfig().min_seconds_to_release_worker) !== null && _f !== void 0 ? _f : conf.min_seconds_to_release_worker) !== null && _g !== void 0 ? _g : DEFAULT_MIN_SECONDS_TO_RELEASE_WORKER;
if (needIncreaseWorkers) {
(0, logger_1.getLogger)().info(`App "${workingApp.getName()}" needs increase workers because ${maxCpuValue}>${scaleCpuThreshold}. CPUs: ${cpuValuesString}`);
const freeMem = Math.round(node_os_1.default.freemem() / MEMORY_MB);
const avgAppUseMemory = workingApp.getAverageUsedMemory();
const memoryAfterNewWorker = freeMem - avgAppUseMemory;
if (memoryAfterNewWorker <= 0) {
// Increase workers only if we have anought free memory
(0, logger_1.getLogger)().debug(`Not enough memory to increase worker for app "${workingApp.getName()}". Free memory ${freeMem}MB, App average memeory ${avgAppUseMemory}MB `);
return;
}
const now = Number(new Date());
const secondsDiff = Math.round((now - workingApp.getLastIncreaseWorkersTime()) / 1000);
if (secondsDiff > minSecondsToAddWorker) {
// Add small delay between increasing workers to detect load
(0, logger_1.getLogger)().debug(`Increase workers for app "${workingApp.getName()}"`);
workingApp.isProcessing = true;
pm2_1.default.scale(workingApp.getName(), '+1', () => {
workingApp.updateLastIncreaseWorkersTime();
workingApp.isProcessing = false;
(0, logger_1.getLogger)().info(`App "${workingApp.getName()}" scaled with +1 worker`);
});
}
}
else {
if (
// Decrease workers if average CPUs load less then "releaseCpuThreshold"
averageCpuValue < releaseCpuThreshold &&
// Process only if we have more workers than default value
workingApp.getActiveWorkersCount() > workingApp.getDefaultWorkersCount()) {
const now = Number(new Date());
const secondsDiff = Math.round((now - workingApp.getLastDecreaseWorkersTime()) / 1000);
if (secondsDiff > minSecondsToReleaseWorker) {
(0, logger_1.getLogger)().debug(`Decrease workers for app "${workingApp.getName()}". Average CPU ${averageCpuValue} < Release CPU ${releaseCpuThreshold}`);
const newWorkers = workingApp.getActiveWorkersCount() - 1;
if (newWorkers >= workingApp.getDefaultWorkersCount()) {
workingApp.isProcessing = true;
pm2_1.default.scale(workingApp.getName(), newWorkers, () => {
workingApp.updateLastDecreaseWorkersTime();
workingApp.isProcessing = false;
(0, logger_1.getLogger)().info(`App "${workingApp.getName()}" decreased one worker`);
});
}
}
}
}
}