apminsight/lib/communication/connadapter.js

monitor nodejs applications

var metricstore = require("./../metrics/metricstore");
var constants = require("./../constants");
var logger = require("./../util/logger");
var reqhandler = require("./reqhandler");
var os = require("os");
var dns = require("dns");
var fs = require("fs");
var path = require("path");
var process = require("process");
var rescodehandler = require("./responsecode");
var utils = require("./../util/utils");
var cloudchecker = require("./cloudchecker");
var dataformatter = require("./dataformatter");
var keyTxnTh = require("./../config/keyTxnThreshold");

var connInfo,
    connPayload,
    isConnSuccess = false,
    connectInterval;

function init() {
    checkAndSendConnect();
    const config = apmInsightAgentInstance.getConfig();
    if (!utils.isEmpty(config) && !config.isDataExporterEnabled()) {
        scheduleDataService();
    } else {
        scheduleUnifiedDataService();
    }

}

function scheduleDataService() {
    setInterval(function () {
        try {
            logger.info("data processing starts");
            if (
                apmInsightAgentInstance.getInsInfo().getStatus() ===
                constants.shutdown
            ) {
                logger.critical("agent is shutdown state");
                return;
            }
            processCollectedData();
            logger.info("data processing ends");

        } catch (e) {
            logger.error("Exception occured in data processing ::", e);
        } finally {
            metricstore.clearMetric();
        }
    }, 60 * 1000).unref();

    setInterval(function () {
        metricstore.clearTraceHistory();
    }, 15 * 60 * 1000).unref();
}



// Unified Data Service with Cycle Counter Logic
function scheduleUnifiedDataService() {
    let cycleCounter = 0;
    
    // Configuration constants
    const APP_PARAMS_CYCLE = 1;        // Every 1 cycle (30 seconds)
    const NVM_HEARTBEAT_CYCLE = 2;     // Every 2 cycles (60 seconds)
    const BASE_INTERVAL = 30 * 1000;   // 30 seconds base interval
    
    // For cycles 1 and 2, reset after 2 cycles to maintain pattern
    const RESET_CYCLE = 2;
    
    logger.info(`Starting unified data service scheduler - Base interval: ${BASE_INTERVAL/1000}s, Reset cycle: ${RESET_CYCLE}`);
    
    setInterval(function () {
        try {
            cycleCounter++;
            logger.debug(`Unified scheduler cycle: ${cycleCounter}`);
            
            // App Parameters Service (every 30 seconds - every cycle)
            if (cycleCounter % APP_PARAMS_CYCLE === 0) {
                logger.debug("Executing app parameters service");
                sendAppParametersData();
            }
            
            // NVM and Heartbeat Service (every 60 seconds - every 2 cycles)  
            if (cycleCounter % NVM_HEARTBEAT_CYCLE === 0) {
                logger.debug("Executing NVM and heartbeat service");
                sendNvmDataAndHeartbeat();
            }
            
            // Reset counter after completing full cycle to prevent overflow and maintain pattern
            if (cycleCounter >= RESET_CYCLE) {
                logger.debug(`Resetting cycle counter after ${RESET_CYCLE} cycles`);
                cycleCounter = 0;
            }
            
        } catch (e) {
            logger.error("Exception occurred in unified data processing:", e);
        }
    }, BASE_INTERVAL).unref();
}

// Extract app parameters logic into separate function
function sendAppParametersData() {
    try {
        const appParametersData = dataformatter.getAppParametersMetrics();
        if (appParametersData) {
            logger.info("Sending app parameters data to data exporter");
            const appParamsData = {
                apm: {
                    app_parameters: appParametersData
                }
            };
            apmInsightAgentInstance.getDataExporter().sendTxnData(appParamsData);

        }
    } catch (e) {
        logger.error("Exception occured in app parameters data processing ::", e);
    }
    finally {
        metricstore.clearMetric();
    }
}

// Extract NVM and heartbeat logic into separate function
function sendNvmDataAndHeartbeat() {
    try {
        let dataExporter = apmInsightAgentInstance.getDataExporter();
        const nvmMetrics = dataformatter.getNodeVMMetrics();
        if (nvmMetrics && Object.keys(nvmMetrics).length > 0) {
            logger.info("Sending NVM data to data exporter");
            const nvmData = {
                apm: {
                    nvm: {
                        data: true,
                        value: nvmMetrics
                    }
                }
            };
            dataExporter && dataExporter.sendTxnData(nvmData);
        }

        // Send heartbeat signal along with NVM data
        dataExporter && dataExporter.sendHeartBeat();

    } catch (e) {
        logger.error("Exception occured in nvm data processing ::", e);
    }
}

function checkAndSendConnect() {
    if (!isConnSuccess) {
        sendConnect(); // Initial call

        if (!connectInterval) {
            connectInterval = setInterval(function () {
                if (isConnSuccess) {
                    clearInterval(connectInterval);
                    connectInterval = null;
                    return;
                }
                sendConnect();
            }, 60 * 1000).unref();
        }
    }
}

function sendConnect() {
    var config = apmInsightAgentInstance.getConfig();
    fetchAndIncludeDependencies();
    if (!config.isLkeyConfiguredProperly()) {
        logger.critical(
            "Agent not configured properly. Please check the apminsightnode.json file for the configured licenseKey. Note: Copy-paste the entire alphanumeric device key as the license key."
        );
        return;
    }
    if (!config.isAppNameConfiguredProperly()) {
        logger.critical(
            "Agent not configured properly. Please check the apminsightnode.json file for the configured appName. The appName must not be an empty string. Trying to find package.json file to read the application name."
        );
        return;
    }
    if (!config.isPortConfiguredProperly()) {
        logger.critical(
            "Agent not configured properly. Please check the apminsightnode.json file for the configured port number. The port must be a positive number. Trying to auto-detect the application's port number."
        );
        return;
    }
    var insInfo = apmInsightAgentInstance.getInsInfo();
    if (
        !insInfo.getStatus() ||
        rescodehandler.isAllowedToSendRequest(
            insInfo.getStatus(),
            insInfo.getRetryCounter()
        )
    ) {
        if (!connPayload) {
            getConnectPayload(function (payload) {
                connPayload = payload;
                sendConnectReq();
            });
        } else {
            sendConnectReq();
        }
    }
}

function sendConnectReq() {
    const config = apmInsightAgentInstance.getConfig();
    if (config.isDataExporterEnabled()) {
        if (apmInsightAgentInstance.getConfig().isPrintPayloadEnabled()) {
            logger.debug(`Initiating connect request to data exporter with payload: ${connPayload}`);
        }
        apmInsightAgentInstance.getDataExporter().makeConnectRequest(connPayload, handleExporterConnectResponse);
    } else {
        reqhandler.sendToCollector(
            constants.connectUrl,
            connPayload,
            handleConnectResponse
        );
    }
}

function handleExporterConnectResponse(response) {
    if (!response) {
        isConnSuccess = false;
        logger.error('Data exporter connect failed, will retry in next cycle');
        return;
    }
    logger.info(`response received for arh/connect from data-exporter`);
    //checking for instance id and status
    if (!utils.isEmpty(response['instance.id']) && (response['instance.status'] === constants.agentConfigUpdated || response['instance.status'] === constants.manageAgent)) {
        var instanceId = response['instance.id'];
        var instanceStatus = response['instance.status'];
        let agentSpecificInfo = {}, customConfigInfo = {};
        if (response[constants.logLevel]) {
            agentSpecificInfo[constants.logLevel] = response[constants.logLevel];
        }
        if (response[constants.sqlStracktraceKey]) {
            customConfigInfo[constants.sqlStracktraceKey] = response[constants.sqlStracktraceKey];
        }
        logger.info('Data exporter connect response received - Instance ID: ' + instanceId + ', Status: ' + instanceStatus);

        apmInsightAgentInstance
            .getInsInfo()
            .updateInstanceInfo(instanceId, instanceStatus);

        //agentInfoPath won't be available for one agent
        if (apmInsightAgentInstance.getConfig() && apmInsightAgentInstance.getConfig().getAgentInfoPath()) {
            doPermissionChanges(
                apmInsightAgentInstance.getConfig()
            );
        }
        isConnSuccess = true;

        // Clear the connect interval when successful
        if (connectInterval) {
            clearInterval(connectInterval);
            connectInterval = null;
            logger.info("Connection successful, cleared connect retry interval");
        }

        // Update thresholds and configurations
        apmInsightAgentInstance.getThreshold().update(customConfigInfo, agentSpecificInfo);
        logger.info('Data exporter connect response processed successfully');
    } else {
        isConnSuccess = false;
        logger.error('Data exporter connect failed - missing instance.id or instance.status, retrying in next cycle');
    }
}

function handleConnectResponse(err, uri, responseData) {
    if (!isValidResponse(err, uri, responseData)) {
        return;
    }
    var data = responseData.data;
    var resCode = data[constants.responseCode];
    var instanceInfo = data[constants.instanceInfo];

    if (resCode) {
        logger.critical(
            "received response code :",
            rescodehandler.getRescodeMessage(resCode)
        );
    }

    if (!instanceInfo || utils.isEmpty(instanceInfo[constants.instanceid])) {
        apmInsightAgentInstance.getInsInfo().updateStatus(resCode);
        return;
    }

    var instanceId = instanceInfo[constants.instanceid];
    apmInsightAgentInstance
        .getInsInfo()
        .updateInstanceInfo(instanceId, resCode);
    apmInsightAgentInstance
        .getThreshold()
        .update(
            data[constants.customConfigInfo],
            data[constants.agentSpecifInfo]
        );
    if (data[constants.txnSpecificInfo]) {
        keyTxnTh.updateKeyTxnInfo(data[constants.txnSpecificInfo]);
    }
    //agentInfoPath won't be available for one agent
    if (apmInsightAgentInstance.getConfig() && apmInsightAgentInstance.getConfig().getAgentInfoPath()) {
        doPermissionChanges(
            apmInsightAgentInstance.getConfig()
        );
    }
    isConnSuccess = true;

    // Clear the connect interval when successful
    if (connectInterval) {
        clearInterval(connectInterval);
        logger.info("Connection successful, cleared connect retry interval");
        connectInterval = null;
    }
}

function handleDataResponse(err, uri, responseData) {
    if (!isValidResponse(err, uri, responseData)) {
        return;
    }
    var data = responseData.data;
    var resCode = data[constants.responseCode];
    var insInfo = apmInsightAgentInstance.getInsInfo();
    if (resCode && resCode !== insInfo.getStatus()) {
        logger.critical(
            "received response code :",
            rescodehandler.getRescodeMessage(resCode)
        );
    }
    insInfo.updateStatus(resCode);
    if (data[constants.customConfigInfo] || data[constants.agentSpecifInfo]) {
        apmInsightAgentInstance
            .getThreshold()
            .update(
                data[constants.customConfigInfo],
                data[constants.agentSpecifInfo]
            );
    }
    if (data[constants.txnSpecificInfo]) {
        keyTxnTh.updateKeyTxnInfo(data[constants.txnSpecificInfo]);
    }
}

function isValidResponse(err, uri, responseData) {
    if (err) {
        logger.error("Error occured for in " + uri + " request", err);
        return false;
    }
    if (!responseData || !responseData.data) {
        logger.critical("received empty response data for " + uri + " request");
        return false;
    }
    logger.info("Response data for " + uri, responseData);
    return true;
}

function handleConfigUpdateResponse(err, uri, responseData) {
    if (!isValidResponse(err, uri, responseData)) {
        return;
    }
    apmInsightAgentInstance.getConfig()._newRumAppKeyAdded = false;
}

function delayedSendToCollector(url, payload, callback, delay) {
    setTimeout(function () {
        reqhandler.sendToCollector(url, payload, callback);
    }, delay); // delay is dynamically set by the function parameter
}

function processCollectedData() {
    var insInfo = apmInsightAgentInstance.getInsInfo();
    var payload;
    switch (insInfo.getStatus()) {
        case constants.manageAgent:
            payload = JSON.stringify(
                getDataWithTime(insInfo, dataformatter.getTxnMetric())
            );
            reqhandler.sendToCollector(
                constants.dataUrl,
                payload,
                handleDataResponse
            );

            var nvmPayload = JSON.stringify(dataformatter.getNodeVMMetrics());
            delayedSendToCollector(
                constants.nodeVmUrl,
                nvmPayload,
                handleDataResponse,
                constants.nodeVmDelay
            );

            var traceData = dataformatter.getFormattedTraceData();
            if (traceData && traceData.length > 0) {
                var traceDataStr = JSON.stringify(
                    getDataWithTime(insInfo, traceData)
                );
                delayedSendToCollector(
                    constants.traceUrl,
                    traceDataStr,
                    handleDataResponse,
                    constants.traceDataDelay
                );
            }
            if (apmInsightAgentInstance.getConfig().isNewRumAppKeyAdded()) {
                var rumIntegAppKeyArray = apmInsightAgentInstance
                    .getConfig()
                    .getRumIntegAppKey();
                var appKeyObj = {
                    "rum.app.keys": rumIntegAppKeyArray
                };
                delayedSendToCollector(
                    constants.configUpdateUrl,
                    JSON.stringify(appKeyObj),
                    handleConfigUpdateResponse,
                    constants.configUpdateDelay
                );
            }
            return;

        case constants.unmanageAgent:
            payload = JSON.stringify(getDataWithTime(insInfo, []));
            reqhandler.sendToCollector(
                constants.dataUrl,
                payload,
                handleDataResponse
            );
            return;

        default:
            if (
                rescodehandler.isAllowedToSendRequest(
                    insInfo.getStatus(),
                    insInfo.getRetryCounter()
                )
            ) {
                payload = JSON.stringify(getDataWithTime(insInfo, []));
                reqhandler.sendToCollector(
                    constants.dataUrl,
                    payload,
                    handleDataResponse
                );
            } else {
                logger.critical("request will be send in specific interval");
            }
            return;
    }
}

// TODO : Last reported time
function getDataWithTime(insInfo, data) {
    var millis = new Date().getTime();
    var lastReported = insInfo.getLastReported()
        ? insInfo.getLastReported()
        : millis;
    return [millis, lastReported, data];
}

function getConnectPayload(cb) {
    cloudchecker.doCloudCheck(function (instanceName, cloudProvider, isAutoScaleEnabled) {
        var config = apmInsightAgentInstance.getConfig();
        var payload = {
            agent_info: {
                [constants.applicationTypeKey]: constants.applicationType,
                [constants.agentVersion]: config.getAgentVersion(),
                [constants.agentVersionInfo]: config.getAgentFullVersion(),
                [constants.appNameKey]: config.getApplicationName(),
                [constants.port]: config.getApplicationPort(),
                [constants.hostType]: utils.isEmpty(cloudProvider)
                    ? os.platform()
                    : cloudProvider,
                [constants.hostname]: utils.isEmpty(instanceName)
                    ? os.hostname()
                    : instanceName,
                [constants.hostLicenseApply]: config.isHostBasedLicense(),
                [constants.isAutoScaleEnabled]: !!isAutoScaleEnabled,
            },
            environment: {
                [constants.userName]: process.env.USER,
                [constants.osVersion]: os.release(),
                [constants.machineName]: os.hostname(),
                [constants.agentInstallPath]: config.getAgentInstalledPath(),
                [constants.nodejsVersion]: process.version,
                [constants.osArch]: os.arch(),
                [constants.os]: os.platform(),
                [constants.ipAddress]: getIP()
            }
        };
        setAgentSpecificInfo(payload, config);
        checkCgroup(payload);
        if (!utils.isEmpty(instanceName)) {
            payload.agent_info[constants.cloudInstanceId] = instanceName;
        }
        if (!utils.isEmpty(config.getServerMonitorKey())) {
            payload.agent_info[constants.serverMonitorKey] = config.getServerMonitorKey();
        }
        addFQDN(function (data) {
            payload.agent_info[constants.fqdn] = data;
            //for data-exporter purpose
            if (apmInsightAgentInstance.getConfig().isDataExporterEnabled()) {
                payload = {
                    connect_info: payload,
                    misc_info: {
                        [constants.licenseKey]: config.getLicenseKey()
                    }
                };
            }
            cb(JSON.stringify(payload));
        });
    });
}

function addFQDN(cb) {
    dns.lookup(os.hostname(), { hints: dns.ADDRCONFIG }, function (err, ip) {
        if (err) {
            logger.error("Error while getting IP from hostname : " + err);
            cb(os.hostname());
            return;
        }
        dns.lookupService(ip, 0, function (err, fqdn) {
            if (err) {
                logger.error("Error while getting FQDN from IP : " + err);
                cb(os.hostname());
                return;
            }
            cb(fqdn);
        });
    });
}

function getIP() {
    var nets = os.networkInterfaces();
    var IPs = [];
    try {
        for (var name of Object.keys(nets)) {
            for (var net of nets[name]) {
                var familyV4Value = typeof net.family === "string" ? "IPv4" : 4;
                if (net.family === familyV4Value && !net.internal) {
                    IPs.push(net.address);
                }
            }
        }
    } catch (err) {
        logger.error(
            "Error while finding the IP address of the machine :: ",
            err
        );
    }
    return IPs;
}

function setAgentSpecificInfo(payload, config) {
    if (utils.isEmpty(payload) || utils.isEmpty(config)) {
        logger.error("Invalid arguments passed to setAgentSpecificInfo");
        return;
    }

    const appGroupName = config.getAppGroupName();
    const tags = config.getTags();

    if (!utils.isEmpty(appGroupName) || !utils.isEmpty(tags)) {
        payload.agent_specific_info = payload.agent_specific_info || {};

        if (!utils.isEmpty(appGroupName)) {
            payload.agent_specific_info[constants.appGroupName] = appGroupName;
        }

        if (!utils.isEmpty(tags)) {
            payload.agent_specific_info[constants.tags] = tags;
        }
    }
}

function fetchAndIncludeDependencies() {
    try {
        var filePath = path.join(process.cwd(), "package.json");
        var packageJson = JSON.parse(fs.readFileSync(filePath));
        if (packageJson && packageJson.dependencies) {
            apmInsightAgentInstance.setDependencyList(packageJson.dependencies);
            logger.info(
                "Dependencies : " + JSON.stringify(packageJson.dependencies)
            );
        }
    } catch (e) {
        logger.info("package.json file not in process started path.");
    }
}

function checkCgroup(payload) {
    try {
        var cgroupInfo = fs.readFileSync("/proc/self/cgroup", "utf8");
        var mountInfoFile = fs.readFileSync("/proc/self/mountinfo", "utf8");
        var regExp = /^cpu(.+|$)/; //Line which starts with cpu
        var regExpForDocker =
            /^\/(docker\/|lxc\/|.+?\/docker-|ecs\/.+?\/|kubepods\/.+?\/)([a-f0-9]+)($|\.scope|\/kubepods.*)/gm; //Docker container check which included ECS, EKS, Kubepods and mostly all container setup.
        var dockerId = "";
        if (!utils.isEmpty(cgroupInfo)) {
            let lines = cgroupInfo.split("\n");
            for (
                var cgroupIndex = 0;
                cgroupIndex < lines.length;
                cgroupIndex++
            ) {
                var cgroupParts = lines[cgroupIndex].split(":");
                if (cgroupParts.length == 3 && regExp.exec(cgroupParts[1])) {
                    var groups = regExpForDocker.exec(cgroupParts[2]);
                    if (groups && groups[2]) {
                        dockerId = groups[2];
                    }
                }
            }
        }
        if (utils.isEmpty(dockerId) && !utils.isEmpty(mountInfoFile)) {
            let lines = mountInfoFile.split("\n");
            for (var mountIndex = 0; mountIndex < lines.length; mountIndex++) {
                var mountParts = lines[mountIndex].split("docker/containers/");
                if (mountParts.length == 2) {
                    dockerId = mountParts[1].split("/", 2)[0];
                }
            }
        }
        if (!utils.isEmpty(dockerId)) {
            payload.agent_info["hostname"] = dockerId;
            payload.agent_info["host.type"] = "DOCKER";
            logger.info("Application hosted in Docker.");
            return;
        }
    } catch (e) {
        logger.info("Application not hosted in docker.");
    }
}

function doPermissionChanges(configDetails) {
    if (utils.isEmpty(configDetails.getBaseDir())) {
        logger.info(
            "Agent basedir is undefined. Might be an error occured while creating apminsightdata folder."
        );
        return;
    }
    const insInfo = apmInsightAgentInstance.getInsInfo();
    const apmInfoFilePath = insInfo.getInfoFilePath();
    if (utils.isEmpty(apmInfoFilePath)) {
        logger.info("Unable to find apminsight.json file");
    } else {
        fs.chmod(apmInfoFilePath, 0o600, function (error) {
            if (error) {
                logger.error(
                    "Error while changing the permission of apminsight.json file :: ",
                    error
                );
                return;
            }
            logger.info(
                "Made permission changes in apminsight.json file successfully"
            );
        });
    }
    var agentInfo = configDetails.getAgentInfoPath();
    if (utils.isEmpty(agentInfo)) {
        logger.info("Unable to find apminsightnode.json file");
    } else {
        fs.chmod(agentInfo, 0o600, function (error) {
            if (error) {
                logger.error(
                    "Error while changing the permission of apminsightnode.json file :: ",
                    error
                );
                return;
            }
            logger.info(
                "Made permission changes in apminsightnode.json file successfully"
            );
        });
    }
}

module.exports = {
    init: init,
    checkAndSendConnect: checkAndSendConnect
};