@5minds/processcube_engine
Version:
The ProcessCube Engine. Stores and executes BPMNs.
521 lines • 24.5 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
var MonitoringManager_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.FunctionReporter = exports.MonitoringManager = exports.FUNCTION_REPORTS_CHANNEL_NAME = exports.API_REQUESTS_CHANNEL_NAME = void 0;
const diagnostics_channel = __importStar(require("diagnostics_channel"));
const inversify_1 = require("inversify");
const os = __importStar(require("os"));
const pidusage_1 = __importDefault(require("pidusage"));
const stringHash = __importStar(require("string-hash"));
const uuid = __importStar(require("uuid"));
const v8 = __importStar(require("v8"));
const processcube_engine_sdk_1 = require("@5minds/processcube_engine_sdk");
exports.API_REQUESTS_CHANNEL_NAME = 'api-requests';
exports.FUNCTION_REPORTS_CHANNEL_NAME = 'function-reports';
const functionReportChannel = diagnostics_channel.channel(exports.FUNCTION_REPORTS_CHANNEL_NAME);
const functionReportLogger = new processcube_engine_sdk_1.Logger('function-reporter');
let MonitoringManager = MonitoringManager_1 = class MonitoringManager {
queryWorker;
fetchAndLockWorker;
runtimeWorker;
logger = new processcube_engine_sdk_1.Logger('monitoring-manager');
iterations = {};
intervals = [];
apiRequestsChannelSubscription = (apiRequestsReport) => {
this.reportApiRequest(apiRequestsReport);
};
functionReportsChannelSubscription = (functionReport) => {
this.reportFunctionReport(functionReport);
};
static getNewFunctionReporter(name, parameters, values) {
if (!MonitoringManager_1.metricsEnabled()) {
return new FunctionReporter(null);
}
const partialFunctionReport = {
id: uuid.v4(),
name,
totalDuration: 0,
startedAt: new Date(),
currentTimestamp: new Date(),
finished: false,
workerId: process.title,
};
try {
if (parameters) {
partialFunctionReport.parameters = JSON.stringify(parameters);
}
if (values) {
partialFunctionReport.values = JSON.stringify(values);
}
const obj = {};
Error.captureStackTrace(obj, this.getNewFunctionReporter);
let callstack = obj.stack;
const stackLines = callstack.split('\n');
stackLines.shift();
callstack = stackLines.join('\n');
partialFunctionReport.callstack = callstack;
}
catch (error) {
functionReportLogger.warn('Error capturing stack trace', { err: error });
}
return new FunctionReporter(partialFunctionReport);
}
static amountOfIterations() {
const defaultValue = 10;
const valueFromEnv = MonitoringManager_1.getNumberFromEnv('METRICS_AMOUNT_OF_ITERATIONS');
if (valueFromEnv !== undefined) {
return valueFromEnv;
}
return defaultValue;
}
static timeBetweenIterations() {
const defaultValue = 60000;
const valueFromEnv = MonitoringManager_1.getNumberFromEnv('METRICS_TIME_BETWEEN_ITERATIONS');
if (valueFromEnv !== undefined) {
return valueFromEnv;
}
return defaultValue;
}
static timeBetweenStatsCollection() {
const defaultValue = 5000;
const valueFromEnv = MonitoringManager_1.getNumberFromEnv('METRICS_TIME_BETWEEN_STATS_COLLECTION');
if (valueFromEnv !== undefined) {
return valueFromEnv;
}
return defaultValue;
}
static timeBetweenReportPrint() {
const defaultValue = 30000;
const valueFromEnv = MonitoringManager_1.getNumberFromEnv('METRICS_TIME_BETWEEN_REPORT_PRINT');
if (valueFromEnv !== undefined) {
return valueFromEnv;
}
return defaultValue;
}
static enableReportPrint() {
const defaultValue = false;
const valueFromEnv = MonitoringManager_1.getBooleanFromEnv('METRICS_ENABLE_REPORT_PRINT');
if (valueFromEnv !== undefined) {
return valueFromEnv;
}
return defaultValue;
}
static disableAuth() {
const defaultValue = false;
const valueFromEnv = MonitoringManager_1.getBooleanFromEnv('METRICS_DISABLE_AUTH');
if (valueFromEnv !== undefined) {
return valueFromEnv;
}
return defaultValue;
}
static metricsEnabled() {
const defaultValue = false;
const valueFromEnv = MonitoringManager_1.getBooleanFromEnv('METRICS_ENABLED');
if (valueFromEnv !== undefined) {
return valueFromEnv;
}
return defaultValue;
}
init(runtimeWorker, queryWorker, fetchAndLockWorker) {
if (!MonitoringManager_1.metricsEnabled()) {
this.logger.info('Metrics are disabled');
return;
}
this.logger.info('Metrics are enabled');
diagnostics_channel.subscribe(exports.API_REQUESTS_CHANNEL_NAME, this.apiRequestsChannelSubscription);
diagnostics_channel.subscribe(exports.FUNCTION_REPORTS_CHANNEL_NAME, this.functionReportsChannelSubscription);
this.queryWorker = queryWorker;
this.fetchAndLockWorker = fetchAndLockWorker;
this.runtimeWorker = runtimeWorker;
this.iterations = {};
for (let i = 0; i < MonitoringManager_1.amountOfIterations(); i++) {
this.iterations[i] = this.getNewIteration();
}
this.intervals.push(setInterval(() => {
this.rotateIteration();
}, MonitoringManager_1.timeBetweenIterations()));
this.intervals.push(setInterval(() => {
this.collectStats();
}, MonitoringManager_1.timeBetweenStatsCollection()));
if (MonitoringManager_1.enableReportPrint()) {
this.intervals.push(setInterval(() => {
this.printReport();
}, MonitoringManager_1.timeBetweenReportPrint()));
}
}
printReport() {
if (!MonitoringManager_1.metricsEnabled()) {
return;
}
try {
const report = this.getReport();
const delta = new Date().getTime() - report.oldestIterationTimestamp.getTime();
const humanReadableDelta = this.getHumanReadableTime(delta);
this.logger.debug(`Reporting resource usage for the last ${humanReadableDelta}.`, report);
}
catch (err) {
this.logger.error('Error printing report', { err });
}
}
reportApiRequest(report) {
if (!MonitoringManager_1.metricsEnabled()) {
return;
}
this.getLatestIteration().apiRequests[report.path] = this.getLatestIteration().apiRequests[report.path] || [];
this.getLatestIteration().apiRequests[report.path].push(report);
}
reportFunctionReport(report) {
if (!MonitoringManager_1.metricsEnabled()) {
return;
}
this.getLatestIteration().functionReports[report.name] = this.getLatestIteration().functionReports[report.name] || {};
this.getLatestIteration().functionReports[report.name][report.id] = report;
}
getReport() {
if (!MonitoringManager_1.metricsEnabled()) {
return {
status: 'disabled',
};
}
const mergedIteration = this.getMergedIterations();
const latestIteration = this.getLatestIteration();
const cpuUsage = this.getCPUUsageReport(mergedIteration, latestIteration);
const memoryUsage = this.getMemoryUsageReport(mergedIteration, latestIteration);
const apiRequests = this.getApiRequestsReport(mergedIteration);
const systemStats = this.getSystemStatsReport();
const functionsReport = this.getFunctionsReport(mergedIteration);
const oldestIterationTimestamp = this.iterations[MonitoringManager_1.amountOfIterations() - 1].timestamp;
return { cpuUsage, memoryUsage, systemStats, apiRequests, functionsReport, oldestIterationTimestamp };
}
dispose() {
if (!MonitoringManager_1.metricsEnabled()) {
return;
}
this.intervals.forEach((interval) => clearInterval(interval));
diagnostics_channel.unsubscribe(exports.API_REQUESTS_CHANNEL_NAME, this.apiRequestsChannelSubscription);
diagnostics_channel.unsubscribe(exports.FUNCTION_REPORTS_CHANNEL_NAME, this.functionReportsChannelSubscription);
}
getFunctionsReport(mergedIteration) {
const mergedFunctionReports = mergedIteration.functionReports;
const functionsReport = {};
for (const functionName of Object.keys(mergedFunctionReports)) {
const reportsForFunctionAsArray = Object.values(mergedFunctionReports[functionName]);
const partionedByWorkerId = reportsForFunctionAsArray.reduce((acc, curr) => {
acc[curr.workerId] = acc[curr.workerId] || [];
acc[curr.workerId].push(curr);
return acc;
}, {});
for (const workerId of Object.keys(partionedByWorkerId)) {
functionsReport[workerId] = functionsReport[workerId] || {};
functionsReport[workerId][functionName] = functionsReport[workerId][functionName] || [];
const partialReportsForWorkerId = partionedByWorkerId[workerId];
const partionedByParamsAndCallStack = partialReportsForWorkerId.reduce((acc, curr) => {
const key = stringHash(`${curr.parameters}:${curr.callstack}`);
acc[key] = acc[key] || [];
acc[key].push(curr);
return acc;
}, {});
for (const key of Object.keys(partionedByParamsAndCallStack)) {
const partialReports = partionedByParamsAndCallStack[key];
const totalDuration = partialReports.reduce((acc, curr) => acc + curr.totalDuration, 0);
const averageDuration = totalDuration / partialReports.length;
const maxDuration = Math.max(...partialReports.map((report) => report.totalDuration));
const minDuration = Math.min(...partialReports.map((report) => report.totalDuration));
const parameters = partialReports[0].parameters;
const values = partialReports.map((report) => report.values).filter((value) => value);
const timestamps = partialReports.map((report) => report.currentTimestamp).filter((timestamp) => timestamp);
const callstack = partialReports[0].callstack;
functionsReport[workerId][functionName].push({
totalDuration: this.getHumanReadableTime(totalDuration),
averageDuration: this.getHumanReadableTime(averageDuration),
maxDuration: this.getHumanReadableTime(maxDuration),
minDuration: this.getHumanReadableTime(minDuration),
count: partialReports.length,
parameters,
callstack,
values,
timestamps,
});
}
}
}
return functionsReport;
}
getMemoryUsageReport(mergedIteration, latestIteration) {
const statsFromAllIterations = mergedIteration.stats;
const latestStats = latestIteration.stats[latestIteration.stats.length - 1];
const memoryUsage = {
main: {
current: this.getHumanReadableMemory(latestStats?.main?.memory ?? 0),
average: this.getHumanReadableMemory(statsFromAllIterations.reduce((acc, curr) => acc + curr.main?.memory, 0) / statsFromAllIterations.length),
},
query: {
current: this.getHumanReadableMemory(latestStats?.query?.memory ?? 0),
average: this.getHumanReadableMemory(statsFromAllIterations.reduce((acc, curr) => acc + curr.query?.memory, 0) / statsFromAllIterations.length),
},
fetchAndLock: {
current: this.getHumanReadableMemory(latestStats?.fetchAndLock?.memory ?? 0),
average: this.getHumanReadableMemory(statsFromAllIterations.reduce((acc, curr) => acc + curr.fetchAndLock?.memory, 0) / statsFromAllIterations.length),
},
runtime: {
current: this.getHumanReadableMemory(latestStats?.runtime?.memory ?? 0),
average: this.getHumanReadableMemory(statsFromAllIterations.reduce((acc, curr) => acc + curr.runtime?.memory, 0) / statsFromAllIterations.length),
},
};
return memoryUsage;
}
getCPUUsageReport(mergedIteration, latestIteration) {
const statsFromAllIterations = mergedIteration.stats;
const latestStats = latestIteration.stats[latestIteration.stats.length - 1];
const cpuUsage = {
main: {
current: this.getHumanReadableCpu(latestStats?.main?.cpu ?? 0),
average: this.getHumanReadableCpu(statsFromAllIterations.reduce((acc, curr) => acc + curr.main?.cpu, 0) / statsFromAllIterations.length),
},
query: {
current: this.getHumanReadableCpu(latestStats?.query?.cpu ?? 0),
average: this.getHumanReadableCpu(statsFromAllIterations.reduce((acc, curr) => acc + curr.query?.cpu, 0) / statsFromAllIterations.length),
},
fetchAndLock: {
current: this.getHumanReadableCpu(latestStats?.fetchAndLock?.cpu ?? 0),
average: this.getHumanReadableCpu(statsFromAllIterations.reduce((acc, curr) => acc + curr.fetchAndLock?.cpu, 0) / statsFromAllIterations.length),
},
runtime: {
current: this.getHumanReadableCpu(latestStats?.runtime?.cpu ?? 0),
average: this.getHumanReadableCpu(statsFromAllIterations.reduce((acc, curr) => acc + curr.runtime?.cpu, 0) / statsFromAllIterations.length),
},
};
return cpuUsage;
}
getApiRequestsReport(mergedIteration) {
const mergedApiRequests = mergedIteration.apiRequests;
const apiRequestsReport = {};
for (const key of Object.keys(mergedApiRequests)) {
const apiRequests = mergedApiRequests[key];
const requestPayloadSizes = apiRequests.map((request) => request.requestPayloadSize);
const responsePayloadSizes = apiRequests.map((request) => request.responsePayloadSize);
const requestDurations = apiRequests.map((request) => request.requestDuration);
const partitionedByUserAgent = apiRequests.reduce((acc, curr) => {
const agentToUse = curr.source.productversion || curr.source.userAgent || 'unknown';
acc[agentToUse] = acc[agentToUse] || [];
acc[agentToUse].push(curr);
return acc;
}, {});
apiRequestsReport[key] = {
totalCount: apiRequests.length,
countPerUserAgent: Object.keys(partitionedByUserAgent).reduce((acc, curr) => {
acc[curr] = partitionedByUserAgent[curr].length;
return acc;
}, {}),
averageDuration: this.getHumanReadableTime(requestDurations.reduce((acc, curr) => acc + curr, 0) / requestDurations.length),
maxDuration: this.getHumanReadableTime(Math.max(...requestDurations)),
minDuration: this.getHumanReadableTime(Math.min(...requestDurations)),
averageRequestPayloadSize: this.getHumanReadableMemory(requestPayloadSizes.reduce((acc, curr) => acc + curr, 0) / requestPayloadSizes.length),
averageResponsePayloadSize: this.getHumanReadableMemory(responsePayloadSizes.reduce((acc, curr) => acc + curr, 0) / responsePayloadSizes.length),
maxRequestPayloadSize: this.getHumanReadableMemory(Math.max(...requestPayloadSizes)),
minRequestPayloadSize: this.getHumanReadableMemory(Math.min(...requestPayloadSizes)),
maxResponsePayloadSize: this.getHumanReadableMemory(Math.max(...responsePayloadSizes)),
minResponsePayloadSize: this.getHumanReadableMemory(Math.min(...responsePayloadSizes)),
};
}
return apiRequestsReport;
}
getSystemStatsReport() {
const systemStats = this.getSystemStats();
const systemStatsHumanReadable = {
totalMemory: this.getHumanReadableMemory(systemStats.totalMemory),
freeMemory: this.getHumanReadableMemory(systemStats.freeMemory),
heapSizeLimit: this.getHumanReadableMemory(systemStats.heapSizeLimit),
};
return systemStatsHumanReadable;
}
getSystemStats() {
return {
totalMemory: os.totalmem(),
freeMemory: os.freemem(),
heapSizeLimit: v8.getHeapStatistics().heap_size_limit,
};
}
async rotateIteration() {
try {
for (let i = MonitoringManager_1.amountOfIterations() - 1; i > 0; i--) {
this.iterations[i] = this.iterations[i - 1];
}
this.iterations[0] = this.getNewIteration();
}
catch (error) {
this.logger.error('Error rotating iteration', error);
}
}
async collectStats() {
try {
const stats = {};
const runtimePromise = (0, pidusage_1.default)(this.runtimeWorker.pid);
const queryPromise = this.queryWorker ? (0, pidusage_1.default)(this.queryWorker.pid) : Promise.resolve(undefined);
const fetchAndLockPromise = this.fetchAndLockWorker ? (0, pidusage_1.default)(this.fetchAndLockWorker.pid) : Promise.resolve(undefined);
const mainPromise = (0, pidusage_1.default)(process.pid);
stats.runtime = await runtimePromise;
stats.query = await queryPromise;
stats.fetchAndLock = await fetchAndLockPromise;
stats.main = await mainPromise;
this.getLatestIteration().stats.push(stats);
}
catch (error) {
this.logger.error('Error collecting stats', error);
}
}
getHumanReadableCpu(cpu) {
return Math.round(cpu * 100) / 100 + ' %';
}
getHumanReadableMemory(memory) {
if (memory < 1024) {
return Math.round(memory * 100) / 100 + ' B';
}
else if (memory < 1024 * 1024) {
return Math.round((memory / 1024) * 100) / 100 + ' KB';
}
else if (memory < 1024 * 1024 * 1024) {
return Math.round((memory / (1024 * 1024)) * 100) / 100 + ' MB';
}
else {
return Math.round((memory / (1024 * 1024 * 1024)) * 100) / 100 + ' GB';
}
}
getHumanReadableTime(time) {
if (time < 1000) {
return Math.round(time * 100) / 100 + ' ms';
}
else if (time < 1000 * 60) {
return Math.round((time / 1000) * 100) / 100 + ' s';
}
else if (time < 1000 * 60 * 60) {
return Math.round((time / (1000 * 60)) * 100) / 100 + ' m';
}
else {
return Math.round((time / (1000 * 60 * 60)) * 100) / 100 + ' h';
}
}
getNewIteration() {
return {
timestamp: new Date(),
stats: [],
apiRequests: {},
functionReports: {},
};
}
getLatestIteration() {
return this.iterations[0];
}
getMergedIterations() {
const mergedIteration = {
timestamp: new Date(),
stats: [],
apiRequests: {},
functionReports: {},
};
for (let i = 0; i < MonitoringManager_1.amountOfIterations(); i++) {
mergedIteration.stats = mergedIteration.stats.concat(this.iterations[i].stats);
for (const key of Object.keys(this.iterations[i].apiRequests)) {
mergedIteration.apiRequests[key] = (mergedIteration.apiRequests[key] ?? []).concat(this.iterations[i].apiRequests[key] ?? []);
}
for (const functionName of Object.keys(this.iterations[i].functionReports)) {
mergedIteration.functionReports[functionName] = mergedIteration.functionReports[functionName] || {};
for (const functionId of Object.keys(this.iterations[i].functionReports[functionName])) {
mergedIteration.functionReports[functionName][functionId] = this.iterations[i].functionReports[functionName][functionId];
}
}
}
return mergedIteration;
}
static getNumberFromEnv(env) {
const envValue = process.env[env];
if (envValue == undefined) {
return undefined;
}
try {
return parseInt(env, 10);
}
catch (error) {
return undefined;
}
}
static getBooleanFromEnv(env) {
const envValue = process.env[env];
if (envValue == undefined) {
return undefined;
}
return envValue === 'true';
}
};
exports.MonitoringManager = MonitoringManager;
exports.MonitoringManager = MonitoringManager = MonitoringManager_1 = __decorate([
(0, inversify_1.injectable)()
], MonitoringManager);
class FunctionReporter {
partialFunctionReport;
constructor(partialFunctionReport) {
this.partialFunctionReport = partialFunctionReport;
this.updateAndSend();
}
updateAndSend(finished = false) {
if (!MonitoringManager.metricsEnabled()) {
return;
}
try {
this.partialFunctionReport.currentTimestamp = new Date();
this.partialFunctionReport.totalDuration = this.partialFunctionReport.currentTimestamp.getTime() - this.partialFunctionReport.startedAt.getTime();
this.partialFunctionReport.finished = finished;
functionReportChannel.publish(this.partialFunctionReport);
}
catch (error) {
functionReportLogger.warn('Error updating and sending function report', { err: error });
}
}
finish() {
this.updateAndSend(true);
}
}
exports.FunctionReporter = FunctionReporter;
//# sourceMappingURL=MonitoringManager.js.map