ca-apm-probe/lib/arf-client.js

CA APM Node.js Agent monitors real-time health and performance of Node.js applications

/**
 * Copyright (c) 2015 CA. All rights reserved.
 *
 * This software and all information contained therein is confidential and proprietary and
 * shall not be duplicated, used, disclosed or disseminated in any way except as authorized
 * by the applicable license agreement, without the express written permission of CA. All
 * authorized reproductions must be marked with this language.
 *
 * EXCEPT AS SET FORTH IN THE APPLICABLE LICENSE AGREEMENT, TO THE EXTENT
 * PERMITTED BY APPLICABLE LAW, CA PROVIDES THIS SOFTWARE WITHOUT WARRANTY
 * OF ANY KIND, INCLUDING WITHOUT LIMITATION, ANY IMPLIED WARRANTIES OF
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT WILL CA BE
 * LIABLE TO THE END USER OR ANY THIRD PARTY FOR ANY LOSS OR DAMAGE, DIRECT OR
 * INDIRECT, FROM THE USE OF THIS SOFTWARE, INCLUDING WITHOUT LIMITATION, LOST
 * PROFITS, BUSINESS INTERRUPTION, GOODWILL, OR LOST DATA, EVEN IF CA IS
 * EXPRESSLY ADVISED OF SUCH LOSS OR DAMAGE.
 */


var net = require('net');
var os = require('os');
var util = require('util');
var semver = require('semver');
var logger = require("./logger.js");
var configData = require('./configdata')
var config = configData.getConfigData();
var probeUtil = require('./utils/common-utils');

var virtualstack = require('./virtualstack');
var collectorMsg = require("./collectormsg");
var probeClientUtil = require("./probe-client-util");

var pid = process.pid;
var instanceId = util.format('%s-%s', os.hostname(), pid);
var maxPingDelay = config.maxPingDelay || 15000;
var lastPingResponseTime = new Date().getTime();
var reconnectAttempt = 1;
var tryReconnect = config.tryReconnect || false;

var buff = [];
var currBuffSize = 0;
var maxAllowedBuffSize = 50000;
var buffFull = false;

var currentHost;
var currentPort;
var currentProbeName;

var commandClient = null;
var dataClient = null;
var isCommandConnected = false;
var isDataConnected = false;

var speakPingProbeTimer = null;

var ArfMessageParser = require('./arf-parser').ArfMessageParser;
var arfMessageParser = null;


var logTraceEventStartFlag = true;
var logTraceEventEndFlag = true;
var logMetricsSendFlag = true;

// latest arf protocol version this probe complies to
var ARFP_VERSION = '1.1.0';

// minimum compatible collector version for metric reporting purpose
var METRICS_COMPATIBILITY = { minCollectorVersion: '10.5.2', minArfpVersion: '1.1.0' };
var metricsEnableDelay = 5000;
var metricsReportingEnabled = false;
var checkMetricsCompat = config.metrics.enabled && config.metrics.checkCompatibility;
var reportedAttributes = {};

var dataConnMsg = function () {
	var object = {
		op: 'dataConn',
		probe: 'nodejs',
		ver: ARFP_VERSION,
		pid: pid,
		instid: instanceId
	};
	return object;
};


var commandConnMsg = function () {
	var object = {
		op: 'commandConn',
		probe: 'nodejs',
		ver: ARFP_VERSION,
		instid: instanceId,
		pgm: currentProbeName,
		prms: { appName: resolveAppName() }
	};

	var hostName = resolveHostName();
	if (hostName)
		object.prms.hostName = hostName;

	var containerId = require('./find-container-id')();
	if (containerId)
		object.prms.containerId = containerId;

	if(!containerId){
		require('./pod-details')(object);
		
	}

	return object;
};

var arfMsg = function () {
	var object = {
		op: 'arf'
	};
	return object;
};

var resolveHostName = function () {
	return config.hostName;
};

var resolveAppName = function () {
	return config.appName;
};

module.exports.start = start;

function connectCommandClient(host, port) {
	var client = net.connect(port, host, function () {
		logger.info('Command connection established with collector agent listening on host: %s, port: %d', host, port);
		client.write(generateMsg(commandConnMsg()));
	});
	client.on('end', function () {
		logger.info('Command connection disconnected');
		isCommandConnected = false;
	});

	arfMessageParser = new ArfMessageParser();
	// pass data to arf parser
	client.pipe(arfMessageParser);

	arfMessageParser.on('arf-message', function (message) {
		message = message.trim();

		if (message && message.length > 0) {
			try {
				processSingleMessage(message);
			}
			catch (e) {
				logger.debug('got exception: %s while processing message: %s', e.message, message);
			}
		}
	});

	return client;
}

function closeCommandClient() {
	/*Clear the speakPingProbeTimer since the Command Connection has been closed.
	No need to check now if the Collector is sending Speak messages or not.
	*/
	isCommandConnected = false;
	clearInterval(speakPingProbeTimer);
	if (commandClient) {
		commandClient.unpipe();
		commandClient.removeAllListeners();
		commandClient.destroy();
		delete commandClient;
		commandClient = null;
	}
}

function reconnectCommandClient() {
    setTimeout(function () {
    		logger.debug('Command reconnection Started');
    		commandClient = connectCommandClient(currentHost, currentPort);
    		logger.debug('Command reconnection Completed');
    		commandClient.on('error', function (error) {
    			logger.error('[Command Conn Error]' + error);
    			reconnectAttempt = 1;
    			closeCommandClient();
    			reconnectCommandClient();
    		});
    		commandClient.on('close', function () {
    			logger.info('Command connection Closed');
    			reconnectAttempt = 1;
    			closeCommandClient();
    			reconnectCommandClient();
    		});
    	}, 2000);
}

function start(opts) {

	currentHost = config.infrastructureAgent.host;
	currentPort = config.infrastructureAgent.port;
	currentProbeName = config.probeName;
	
	opts.currentHost = currentHost;
    opts.currentPort = currentPort;

	logger.debug('Trying to connect with collector agent listening on host: %s, port: %d', currentHost, currentPort);

	commandClient = connectCommandClient(currentHost, currentPort);
	// on connection announce custom attributes
	setTimeout(reportConfigAttributes,metricsEnableDelay+2000);
	// register reporting of custom attributes at reporting interval
	setInterval(reportConfigAttributes,reportingInterval());
	commandClient.on('error', function (error) {
		logger.info('[Command Conn Error]' + error);
		closeCommandClient();
		reconnectCommandClient();
	});
	commandClient.on('close', function () {

		closeCommandClient();
		reconnectCommandClient();
	});
}

function reportingInterval() {
	if (config.hasOwnProperty("attribute")) {
		return config.attribute.decoration.reportingInterval || 900000;
	} else if (config.hasOwnProperty("attribute.decoration")) {
		//It is for backward compatible
		return config["attribute.decoration"]["reporting.interval"] || 900000;
	} else {
		return 900000;
	}
}

function reportConfigAttributes() {
	try {
		config = configData.getConfigData();
		
		commMsg = commandConnMsg();
		var customAttributes = {};
		// check attribute decoration is enabled
		if (!config.attribute.decoration) {
			return;
		}

		if (config.attribute.decoration.enableExtension != true) {
			logger.debug("custom attribute decoration is not enabled");
			return;
		}
		// load env attributes
		var envParams = config.attribute.decoration.environment.properties;
		if (envParams) {
			for (var i = 0; i < envParams.length; i++) {
				var envname = envParams[i]
				customAttributes["attribute\.decoration\.env\." + envname] = process.env[envname] || "undefined"
			}
		}
		var staticParams = config.attribute.decoration.static.attributes;
		if (staticParams) {
			Object.keys(staticParams).forEach(function (key) {
				customAttributes["attribute\.decoration\.user\." + key] = staticParams[key];
			});
		}
		var extParams = config.attribute.decoration.externalfile;
		if (extParams && extParams.fileName) {
			var extAttributes = readExternalAttributes(extParams.fileName);
			if (extAttributes) {
				Object.keys(extAttributes).forEach(function (key) {
					customAttributes[key] = extAttributes[key];
				});
			}
		}
		var deletedAttr = findDeletedAttributes(reportedAttributes, customAttributes);

		commMsg.prms = probeUtil.merge(commMsg.prms,customAttributes);
		commMsg.prms = probeUtil.merge(commMsg.prms,deletedAttr);

		// report custom attributes
		if (commandClient != null && commandClient.readyState !== 'close') {
			commandClient.write(generateMsg(commMsg));
			reportedAttributes = customAttributes;
		} else {
			logger.warn("Unable to report custom attributes , commandClient unavailable");
		}

	} catch (e) {
		logger.debug("got exception: %s , unable to process custom attributes", e.message);
	}

}
module.exports.reportConfigAttributes = reportConfigAttributes;

function processSingleMessage(message) {
	if (message.indexOf('speak') != -1) {
		commandClient.write(generateMsg(arfMsg()));
		lastPingResponseTime = new Date().getTime();
		logger.debug("Sent reply for 'speak' message received. lastPingResponseTime updated to: %d", lastPingResponseTime);

	} else {
		processConfigMessage(message);
	}
}

// processes configuration message received from collector agent
function processConfigMessage(message) {
	var obj = JSON.parse(message);

	if (obj != null && obj.op == 'config') {
		if (obj.cmd == 'require') {
			logger.debug("received pbd configuration message for module: %s", obj.mod);

			var mod = obj.mod;
			var methodMap = obj.prms;
			logger.debug("module method map: %s", methodMap[0]);
			probeMod = CAAPMPROBE.getFromProbeMap(obj.mod);

			if (probeMod != null && probeMod.instrument != undefined && probeMod.instrument != null) {
				probeMod.instrument(methodMap);
			}
		} else if (obj.msg) {
			if (typeof obj.msg === 'string') {
				var m = JSON.parse(obj.msg);

				// collector version 10.5.2
				if (m.collector)
					processCollectorVersion(m.collector.version);
			}
		} else if (obj.cmd == 'version') {
			// collector version 10.7 and later
			var params = obj.prms;

			if (params.collector)
				processCollectorVersion(params.collector);
			if (params.arfp)
				processArfpVersion(params.arfp);
			logger.info("Command connection Response: %s", message.toString());
			isCommandConnected = true;
            logTraceEventStartFlag = true;
            logTraceEventEndFlag = true;
            logMetricsSendFlag = true;
            // we need to wait for agent configuration message
            setTimeout(checkMetricReportingStatus, metricsEnableDelay);
			sendDataConnMsg();
			speakPingProbeTimer = setInterval(speakPingProbe, 2000);
		}
	}
}

function processCollectorVersion(version) {
	logger.info("Probe connected to collector agent version: %s", version);

	try {
		if (checkMetricsCompat && semver.gte(version, METRICS_COMPATIBILITY.minCollectorVersion)) {
			metricsReportingEnabled = true;
		}
	}
	catch (e) {
		// ignore
	}
}

function processArfpVersion(version) {
	logger.debug("Collector agent implements ARF protocol version: %s", version);

	if (checkMetricsCompat && semver.gte(version, METRICS_COMPATIBILITY.minArfpVersion)) {
		metricsReportingEnabled = true;
	}
}

//close the commandClient and dataClient if the speak message has not been received till the maxPingDelay time gap
function speakPingProbe() {

	try{
		var currentTime = new Date().getTime();
		//logger.info('current time millis %d', currentTime);

		if (lastPingResponseTime < (currentTime - maxPingDelay)) {
			var logMsg = "Timeout for Speak Message. CommandClient state: " + (commandClient != null ? commandClient.readyState : "null");
			if (commandClient != null && commandClient.readyState !== 'close' && commandClient.readyState !== 'readOnly') {
				//commandClient.removeAllListeners('close');
				logMsg += " Command connection is closed.";
				commandClient.end();
				if (dataClient != null && dataClient.readyState !== 'close' && dataClient.readyState !== 'readOnly') {
					logMsg += " Data connection is closed.";
					dataClient.end();
				}
			}
			logMsg += " from Probe End. Reconnect attempt count: " + reconnectAttempt + " lastPingResponseTime: " + lastPingResponseTime;
			logger.info(logMsg);

			if(tryReconnect && (commandClient == null || commandClient.readyState === "readOnly") && reconnectAttempt++ > 30) {
				lastPingResponseTime = new Date().getTime();
				reconnectAttempt = 1;
				closeCommandClient();
				reconnectCommandClient();
			}
		}
	} catch(e){
		logger.error("Unable to reestablish connection due to error in speak probe. Error is:" + e);
	}


}

var sendDataConnMsg = function (isReconnection) {



	dataClient = net.connect(currentPort, currentHost, function () {
		if (isReconnection) {
			logger.info('Data connection re-established with collector agent');
		}
		else {
			logger.info('Data connection established with collector agent');
		}

		isDataConnected = true;
		lastPingResponseTime = new Date().getTime();

		dataClient.write(generateMsg(dataConnMsg()));

		// report platform metrics once data connection is up
		// and collector agent compatibility has be verified
		setTimeout(probeClientUtil.reportPlatformMetrics, (metricsEnableDelay + 2000));
	});
	dataClient.on('data', function (data) {
		logger.info("Server data on Data Conn: %s", data.toString());
		//client.end();
	});
	dataClient.on('end', function () {
		logger.info('Data connection disconnected');
		isDataConnected = false;
	});
	dataClient.on('error', function (error) {
		logger.info('[Data Conn Error]' + error);
		isDataConnected = false;
		if(error.message.includes('This socket has been ended by the other party')){
            reconnectAttempt = 1;
            closeCommandClient();
            reconnectCommandClient();
        }
	});
	dataClient.on('close', function () {
		logger.info('Data connection Closed');
		isDataConnected = false;

		setTimeout(function () {
			if (commandClient && commandClient.readyState.toString() === 'open') {
                dataClient.removeAllListeners('connect');
                sendDataConnMsg(true);
            }
			else {
				logger.info('Data Connection not re-established since Command Connection is down');
			}
		}, 2000);
	});


};


var generateMsg = function (object) {
	var origJson = JSON.stringify(object);
	return origJson + os.EOL;
};

function checkBufferSize(msg) {
	if (currBuffSize + msg.length > maxAllowedBuffSize)
		return 0;
	else return 1;
}


// Write the buffer to the socket every 500 msecs
setInterval(function () {
	if (buff.length > 0 && !buffFull) {
	   if (!isDataConnected) {
            logger.warn(' writeToSocketByAnonymous - Cannot send message as data connection is down.' + buff.join(os.EOL));
            return;
        }
		dataClient.write(buff.join(os.EOL) + os.EOL);
		buff = [];
		currBuffSize = 0;
	}
	buffFull = false;
}, 500); // 500 msecs


function writeToSocket(msg) {
	var res = checkBufferSize(msg);
	if (res == 1) { // space still available
		buff.push(msg);
	}
	else {
	    if (!isDataConnected) {
            logger.warn(' writeToSocket - Cannot send message as data connection is down.' + buff.join(os.EOL));
            return;
        }
		buffFull = true;
		dataClient.write(buff.join(os.EOL) + os.EOL);
		buff = [];
		currBuffSize = 0;
		buff.push(msg); // clear the buff is it exceeds capacity and push the new incoming message
	}
	currBuffSize += msg.length;
}


function startTrace(name, time, txid, evtid, params) {
	/*This method should convert the incoming invocationData to relevant object*/

	//Check to find if the command connection is up: Only then proceed with the further execution of this function
	if (commandClient == null || commandClient.readyState.toString() !== 'open') {
		if (logTraceEventStartFlag) {
			logger.info('Start Trace cannot be initiated since the command connection is down.');
			logTraceEventStartFlag = false;
		}
		return;
	}


	if (dataClient == null) {
		sendDataConnMsg();
	}

	var object = {
		op: 'fnC',
		fn: name,
		ts: time,
		tid: txid,
		seq: evtid
	};
	if (params && Object.keys(params).length > 0) {
		object.prms = params;
	}
	var msg = JSON.stringify(object);
	writeToSocket(msg);
}

function endTrace(name, time, txid, evtid, params, errormsg) {


	//Check to find if the command connection is up: Only then proceed with the further execution of this function
	if (commandClient == null || commandClient.readyState.toString() !== 'open') {
		if (logTraceEventEndFlag) {
			logger.info('End Trace cannot be initiated since the command connection is down.');
			logTraceEventEndFlag = false;
		}
		return;
	}

	if (dataClient == null) {
		sendDataConnMsg();
	}

	var object = {
		op: 'fnR',
		fn: name,
		ts: time,
		tid: txid,
		cseq: evtid
	};
	if (params && Object.keys(params).length > 0) {
		object.prms = params;
	}

	if (errormsg && Object.keys(errormsg).length > 0) {
		object.exc = errormsg;
	}

	var msg = JSON.stringify(object);
	writeToSocket(msg);

}

virtualstack.on('tracer-start', function (event) {
	startTrace(event.name, event.time, event.txid, event.evtid, event.params);
});
virtualstack.on('tracer-finish', function (event) {
	endTrace(event.name, event.time, event.txid, event.evtid, event.params, event.errormsg);
});

module.exports.startTrace = startTrace;
module.exports.endTrace = endTrace;

function sendRequire(event) {
	if (!isCommandConnected) {
		logger.debug('Cannot send require event as command connection is down.');
		defer(sendRequire, event);
		return;
	}

	if (!isDataConnected) {
		logger.debug('Cannot send require event as data connection is down.');
		defer(sendRequire, event);
		return;
	}

	var object = {
		op: 'require',
		mod: event.module
	};
	if (event.params) {
		if (Object.keys(event.params).length > 0) {
			object.prms = event.params;
		}
	}
	var msg = generateMsg(object);
	dataClient.write(msg);
	logger.debug("Sending " + msg);

}

function defer(f) { setTimeout(f.bind.apply(f, arguments), 2000); }

collectorMsg.on('send-require', function (event) {
	sendRequire(event);
});

collectorMsg.on('receive-require', function (event) {
	collectorMsg.receiveRequire(event);
});

collectorMsg.on('send-metric', function (metric) {
	sendMetricInternal(metric);
});

function sendMetricInternal(metric) {

	//Check to find if the command connection is up: Only then proceed with the further execution of this function
	if (commandClient == null || commandClient.readyState.toString() !== 'open') {
		if (logMetricsSendFlag) {
			logger.debug('Metrics cannot be sent since the command connection is down.');
			logMetricsSendFlag = false;
		}
		return;
	}

	if (dataClient == null) {
		sendDataConnMsg();
	}

	if (metricsReportingEnabled) {
		var msg = generateMsg(metric);
		logger.debug("sending: <%s>", msg);
		dataClient.write(msg);
	}
};

function checkMetricReportingStatus() {
	if (config.metrics.enabled && !metricsReportingEnabled) {
		if (!config.metrics.checkCompatibility) {
			metricsReportingEnabled = true;
		} else {
			logger.warn("Probe has disabled runtime metrics reporting because it could not validate Collector Agent version. Please check probe compatibility guide.")
		}
	}
}

findDeletedAttributes = function(prms,customAttributes){

	var deletedAttr = {};

	Object.keys(prms).forEach(function(key){

		if(key && key.startsWith('attribute.decoration') && !customAttributes.hasOwnProperty(key)){
			deletedAttr[key] = "null";
		}

	});

	return deletedAttr;
}

readExternalAttributes = function(path){
	
	var extAttributes = {};
	try {
	delete require.cache[require.resolve(path)];
	var extConfig = require(path);

	Object.keys(extConfig).forEach(function(key){
		if(!probeUtil.isArray(extConfig[key]) && !probeUtil.isObject(extConfig[key])){
		
			extAttributes["attribute\.decoration\.ext\."+key] = extConfig[key];
		}

	});

    } catch(e){
		logger.error('got exception: %s while reading attributes from path: %s', e.message, path);
	}
	return extAttributes;
}

module.exports.sendMetric = sendMetricInternal;