ca-apm-probe/lib/probes/http.js

CA APM Node.js Agent monitors real-time health and performance of Node.js applications

/**
 * Copyright (c) 2015 CA. All rights reserved.
 *
 * This software and all information contained therein is confidential and proprietary and
 * shall not be duplicated, used, disclosed or disseminated in any way except as authorized
 * by the applicable license agreement, without the express written permission of CA. All
 * authorized reproductions must be marked with this language.
 *
 * EXCEPT AS SET FORTH IN THE APPLICABLE LICENSE AGREEMENT, TO THE EXTENT
 * PERMITTED BY APPLICABLE LAW, CA PROVIDES THIS SOFTWARE WITHOUT WARRANTY
 * OF ANY KIND, INCLUDING WITHOUT LIMITATION, ANY IMPLIED WARRANTIES OF
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT WILL CA BE
 * LIABLE TO THE END USER OR ANY THIRD PARTY FOR ANY LOSS OR DAMAGE, DIRECT OR
 * INDIRECT, FROM THE USE OF THIS SOFTWARE, INCLUDING WITHOUT LIMITATION, LOST
 * PROFITS, BUSINESS INTERRUPTION, GOODWILL, OR LOST DATA, EVEN IF CA IS
 * EXPRESSLY ADVISED OF SUCH LOSS OR DAMAGE.
 */

var util = require('util');
var agent = require('../agent');
var proxy = require('../proxy');
var bagent = require('../browser-agent');
var logger = require("../logger.js");
var config = require('../configdata').getConfigData();
var path = require('path');
var fs = require('fs');

// for http correlation between traces
// decorate http client request by adding custom header
var isClientRequestDecEnabled = config.http.client.requestDecorationEnabled;

var requestHeadersToCollect = config.http.requestHeaders || [];
var { baSnippet, isResponseDecEnabled, isAutoInjectionEnabled, isCorGuidEnabled, maxCookieExpInMs, allowedResContentType } = bagent.initializeBrowserAgentParameters();

var CORRELATION_HEADER_KEY = 'CorID';
var CORRELATION_HEADER_KEY_LOWERCASE = CORRELATION_HEADER_KEY.toLowerCase();
var DEFAULT_HOST = 'localhost';
var DEFAULT_PORT = '80';
var DEFAULT_PROTOCOL = 'http:';

var targetModule = new Object;
var serverNum = 1;
var isSnippetInjected = false;

// TODO - for 'connect' we might be interested in finding timing till socket close, not just connect event
var CLIENT_SUCCESS_EVENTS = { 'response': true, 'connect': true, 'information': true, 'upgrade': true };
var CLIENT_FAILURE_EVENTS = { 'error': true };

module.exports = function (http) {
  targetModule.http = http;
  targetModule.methodMap = getMethodsWithProbes();
  var methodMap = targetModule.methodMap;
  var isGqlUrl;
  // server probe

  proxy.after(http, 'createServer', function (module, args, server) {
    server['__CA_id'] = serverNum++;
    logger.debug('HTTP server %d created', server['__CA_id']);
    var stackTrace = (new Error('Location')).stack.substring(6);
    logger.debug(stackTrace);
  });

  if (http.ServerResponse) {
    proxy.before(http.ServerResponse.prototype, 'write', function (obj, args, storage) {
      if (!isSnippetInjected && isResponseDecEnabled && isAutoInjectionEnabled && obj._headerSent === false) {
        if (args[0] && bagent.validateRespIfJson(args[0].toString('utf8'))) {
          originalHeaders = args[0];
          args[0] = bagent.prepareScript(storage.get('ctx')) + originalHeaders;
        }
      }
    });
  }

  proxy.before(http.Server.prototype, ['on', 'addListener'],
    function caHTTPBeforeHook(server, args) {

      if (logger.isDebug()) {
        logger.debug('HTTP server ' + server['__CA_id'] + ': ' + args[0] + ' listener added.');
        logger.debug((new Error('Location')).stack.substring(6));
      }


      if (methodMap[0] == "skip_instrument") {
        return;
      }

      // store ref to server so we can pull current connections
      agent.httpServer = server;

      // agent provides data such as protocol, default port
      // it differs across http vs https
      var httpAgent = http.globalAgent;

      var kContextPropertyName = '__CA_APM_PROBE_HTTP_CONTEXT__';
      var context = server[kContextPropertyName];
      if (context == null) {
        // Index [0] is current connection counter, [1] is the previous one.
        context = { connectionCounts: [0, 0] };
        Object.defineProperty(server, kContextPropertyName, { value: context });
      }

      function getEventNameFormatted(req) {
        var name = 'http.' + req.method;
        return name;
      }

      function getEventArgs(req) {
        var eargs = {
          url: req.url,
          hdrs: {}
        };
        var corHeaderValue = req.headers[CORRELATION_HEADER_KEY_LOWERCASE];
        if (corHeaderValue) {
          if (logger.isDebug()) {
            logger.debug('Noticed correlation header in incoming http request, corid: %s'
              , corHeaderValue);
          }
          eargs.corId = corHeaderValue;
        }

        requestHeadersToCollect.forEach(function (h) {
          var v = req.headers[h.toLowerCase()];
          if (v) {
            eargs.hdrs[h] = v;
          }
        });

        var hostHeader = req.headers['host'];

        if (hostHeader) {
          var hostPort = hostHeader.split(':');
          eargs.hostName = hostPort[0] || DEFAULT_HOST;
          eargs.hostPort = hostPort[1] || (httpAgent.defaultPort && httpAgent.defaultPort.toString()) || DEFAULT_PORT;
        } else {
          eargs.hostName = DEFAULT_HOST;
          eargs.hostPort = (httpAgent.defaultPort && httpAgent.defaultPort.toString()) || DEFAULT_PORT;
        }

        return eargs;
      }

      function logHttpUrl(req, eargs) {
        if (logger.isDebug()) {
          var protocol = httpAgent.protocol || DEFAULT_PROTOCOL;
          var url = protocol + '//' + eargs.hostName + ':' + eargs.hostPort + req.url;
          logger.debug('http requested url: %s', url);
          logger.debug('http headers collected: %s', JSON.stringify(eargs.hdrs));
        }
      }

      function processRequest(req, storage) {
        var eventNameFormatted = getEventNameFormatted(req);

        var eventArgs = getEventArgs(req);

        isGqlUrl = (eventArgs.url == agent.graphQLUrl || eventArgs.url == '/graphql' || eventArgs.url == '/gql' || eventArgs.url == process.env.GRAPHQL_URL);

        if (req.method == 'POST' && isGqlUrl) {
          eventArgs.skipTrace = true;
        } else {
          eventArgs.skipTrace = false;
        }

        var ctx = agent.asynchEventStart(null, eventNameFormatted, eventArgs);
        ctx.hostName = eventArgs.hostName;
        ctx.corId = eventArgs.corId;
        storage.set('ctx', ctx);
        logHttpUrl(req, eventArgs);
        ctx.eventNameFormatted = eventNameFormatted;
        return ctx;
      }

      // will fail if cookie contains '='
      function getCookies(request) {
        var cookies = {};
        request.headers && request.headers.cookie && request.headers.cookie.split(';').forEach(function (cookie) {
          var parts = cookie.match(/(.*?)=(.*)$/)
          cookies[parts[1].trim()] = (parts[2] || '').trim();
        });
        return cookies;
      }

      function setCookie(response, key, value, path, age) {
        var cookies = response.getHeader('Set-Cookie');
        if (cookies == undefined)
          cookies = [];
        var exdate = new Date(getDateTimeMilliseconds() + age);
        cookieText = encodeURIComponent(key) + '=' + encodeURIComponent(value) + ';expires=' + exdate.toUTCString() + ';'
        if (path)
          cookieText += 'path=' + path + ';'
        cookies.push(cookieText);
        response.setHeader('Set-Cookie', cookies);
      }

      function getDateTimeMilliseconds() {
        return Date.now();
      }

      function shouldSendCookies(req, storage) {
        var cookies = getCookies(req);
        if (cookies['x-apm-brtm-response-bt-id'] == undefined)   // should check for bt-id cookie only .. decorated if it is AJAX
        {
          return true;
        }
        return false;
      }

      function contentTypeHtml(res) {
        if (allowedResContentType.match('text/html')) {
          return true;
        } else {
          return false;
        }
      }

      function lastSegmentOfUrl(pathname) {
        var pathNameArr, lastSeg = "", isTrailingSlash = false;
        if (typeof pathname !== 'string' || pathname.length < 1) {
          return lastSeg;
        }
        // If there is a trailing slash, then
        // 1. Set a flag
        // 2. Strip it and split by forward slash
        if (pathname[pathname.length - 1] === '/') {
          isTrailingSlash = true;
          pathname = pathname.slice(0, -1);
        }
        pathNameArr = pathname.split('/');
        if (pathNameArr.length > 0) {
          lastSeg = pathNameArr[pathNameArr.length - 1];
        }
        return (isTrailingSlash) ? lastSeg + '/' : lastSeg;
      }


      function processResponseForBrowserAgent(req, res, storage) {
        if (res.__CA_patched) {
          return;
        }
        // Browser Agent cookie or header additions
        var ctx = storage.get('ctx');
        serverTime = 'servertime=' + getDateTimeMilliseconds();
        setCookie(res, 'x-apm-brtm-servertime', serverTime, '/', maxCookieExpInMs)
        var headerValue;
        headerValue = bagent.getBaResponseBt(ctx, isGqlUrl);
        res.setHeader('x-apm-ba-response-bt', encodeURIComponent(headerValue));
        if (shouldSendCookies(req, storage)) {
          logger.debug("Cookie Corid=" + ctx.corId + " Expires:" + maxCookieExpInMs);
          setCookie(res, 'x-apm-brtm-response-bt-page-' + lastSegmentOfUrl(req.url), headerValue, '/', maxCookieExpInMs);
        }
        res.setHeader('Access-Control-Expose-Headers', 'x-apm-ba-response-bt');
        var ref = req.headers.referer;
        if (ref) {
          res.setHeader('Access-Control-Allow-Origin', ref);
        }

      }

      if (args[0] !== 'request' && args[0] !== 'upgrade') return;

      proxy.callback(args, -1, function caHTTPCallbackHook(obj, args, storage) {
        context.connectionCounts[0] += 1;

        if (agent.paused) return;

        var request = args[0];
        var res = args[1];
        var ctx = null;
        var eventNameFormatted = '';

        // handle multiple request listeners scenario
        // process request only once to send start event
        if (request.__CA_ctx) {
          ctx = request.__CA_ctx;
          storage.set('ctx', ctx);
        } else {
          ctx = processRequest(request, storage);
          request.__CA_ctx = ctx;
        }

        // response already patched to send finish event
        if (res.__CA_patched) {
          return;
        }

        eventNameFormatted = ctx.eventNameFormatted;
        // if browser decoration enabled and content = text/html
        if (isResponseDecEnabled && contentTypeHtml(res)) {
          processResponseForBrowserAgent(request, res, storage);
        }
        var errorObject = null;
        res.__CA_patched = true;

        var end_save = res.end;
        res.end = function () {
          var isFinished = res.finished;
          if (!isSnippetInjected && isResponseDecEnabled && isAutoInjectionEnabled && res._headerSent === false) {
            baEncrSnippet = bagent.prepareScript(ctx);
            if (ctx.gqlCtx) {
              if (arguments[0] && bagent.validateRespIfJson(arguments[0].toString('utf8')) && !arguments[0].includes('x-apm-ba-response-bt')) {
                endArguments = arguments[0];
                arguments[0] = baEncrSnippet + endArguments;
              }
            } else {
              if ((arguments[0] || arguments[0] == '')) {
                if (bagent.validateRespIfJson(arguments[0].toString('utf8')) && !arguments[0].includes('x-apm-ba-response-bt')) {
                  endArguments = arguments[0];
                  arguments[0] = baEncrSnippet + endArguments;
                }
              }
              else {
                isSnippetInjected = true;
                res.write(baEncrSnippet);
                isSnippetInjected = false;
              }
            }
          }
          end_save.apply(res, arguments);
          if (isFinished === false) {
            if (res.statusCode >= 400) {
              errorObject = new Object();
              errorObject['class'] = "Http " + res.statusCode;
              errorObject['msg'] = res.statusMessage;
            }
            if (eventNameFormatted == 'http.POST' && ctx.args.url.startsWith(agent.graphQLUrl)) {
              var gqlCtx = ctx.gqlCtx;
              if (gqlCtx) {
                gqlCtx = agent.asynchEventDone(gqlCtx, gqlCtx.name, null, new Object());
                if (gqlCtx != null) {
                  gqlCtx = agent.asynchEventFinish(gqlCtx);
                }
                ctx.gqlCtx = null;
              }
            }
            ctx = agent.asynchEventDone(ctx, eventNameFormatted, null, errorObject);
            if (ctx != null) {
              agent.asynchEventFinish(ctx);
            }
          }
        };

      });
    });


  if (isClientRequestDecEnabled) {
    // main probe into outgoing client request instance
    proxy.after(http, 'request', function (obj, args, rval, storage) {
      var clientReq = rval;

      if (typeof clientReq === 'object') {
        decorateClientRequest(storage.get('ctx'), clientReq);
      }
    });

    //in node8, we miss GET request tracing via http.request() probe.
    // special correlation probe for tracing http GET requests made via convenience method http.get(options, [cb])
    if (require('semver').satisfies(process.version, '>6.8.x') && http.ClientRequest && http.ClientRequest.prototype) {
      logger.debug('inserted correlation probe for clientRequest.end()');

      proxy.before(http.ClientRequest.prototype, 'end', function caHTTPClientReqHook(obj, args, storage) {
        var clientReq = obj;
        if (clientReq.path.startsWith('/apmia/datacollector/')) {
          return;
        }
        // GET request headers are sent on end() call. last chance for us to add correlation header.
        if (!clientReq._headerSent && !clientReq.getHeader(CORRELATION_HEADER_KEY) && clientReq.method === 'GET') {
          decorateClientRequest(storage.get('ctx'), clientReq);
        }
      });
    }
  }

  if (http.ClientRequest && http.ClientRequest.prototype) {

    // request.end() is used to signify that one is done with making request out
    proxy.after(http.ClientRequest.prototype, 'end', function caHTTPClientReqHook(obj, args, rval, storage) {

      var clientReq = obj;
      var hostHeader = clientReq.getHeader('host');
      var host;
      var port;
      var protocol = (clientReq.agent && clientReq.agent.protocol) || DEFAULT_PROTOCOL;
      var path = clientReq.path;
      var url = path;
      if (hostHeader) {
        var hostPort = hostHeader.split(':');
        host = hostPort[0] || DEFAULT_HOST;
        port = hostPort[1] || (clientReq.agent && clientReq.agent.defaultPort && clientReq.agent.defaultPort.toString()) || DEFAULT_PORT;
      } else {
        host = DEFAULT_HOST;
        port = (clientReq.agent && clientReq.agent.defaultPort && clientReq.agent.defaultPort.toString()) || DEFAULT_PORT;
      }

      if (path.startsWith('/apmia/datacollector/')) {
        return;
      }

      if (!path.startsWith(protocol + '//')) {

        // FIX DE289181: handle path with starting pattern of host:port and not having protocol prefix
        if (!path.startsWith('/')) {
          var path2 = path;
          var endPos = path2.indexOf("/");
          if (endPos < 0) {
            path = "";
          } else {
            path = path2.substring(endPos);
            path2 = path2.substring(0, endPos);
          }

          var hostPort2 = path2.split(':');
          if (hostPort2[1]) {
            host = hostPort2[0] || host;
            port = hostPort2[1] || port;
          } else {
            path = '/' + path2 + path;
          }
        }

        url = protocol + '//' + host + ':' + port + path;
      }

      var eventArgs = {
        host: host,
        port: port,
        method: clientReq.method,
        path: path,
        url: url
      };

      var eventNameFormatted = 'httpclient.request';
      var ctx = storage.get('ctx');
      if (logger.isDebug()) {
        if (ctx) {
          logger.debug('%s[%d %d %d]: %s', eventNameFormatted, ctx.txid, ctx.lane, ctx.evtid, eventArgs.url);
        }
        else {
          logger.debug('%s - no context', eventNameFormatted);
          logger.debug((new Error('No context')).stack);
        }
      }
      ctx = agent.asynchEventStart(storage.get('ctx'), eventNameFormatted, eventArgs);
      storage.set('ctx', ctx);

      // patch emit method
      var originalEmit = clientReq.emit;

      clientReq.emit = function () {
        beforeClientResponseHook(clientReq, arguments, ctx);
        var returnValue = originalEmit.apply(this, arguments);
        if (ctx != null) agent.asynchEventFinish(ctx);
        return returnValue;
      };

    });
  };

  // one time response event is emitted by ClientRequest instance when it is
  // gets response back from server
  var beforeClientResponseHook = function (obj, args, ctx) {
    var clientReq = obj;

    if (clientReq.__CA_patched || methodMap[0] == "skip_instrument") {
      return;
    }
    if (ctx.args.path.startsWith('/apmia/datacollector/')) {
      return;
    }
    var eventType = args[0];
    var res = args[1];
    var eventArgs = null; // also a flag for finishing trace
    var errorObject = null;

    logger.debug('noticed http client request event: ' + eventType);

    if (CLIENT_SUCCESS_EVENTS[eventType] && res) {
      eventArgs = {
        resCode: res.statusCode,
        resMsg: res.statusMessage
      };

      if (eventArgs.resCode >= 400 && eventArgs.resCode <= 599) {
        errorObject = {
          class: "Http " + eventArgs.resCode,
          msg: eventArgs.resMsg
        };
      }
    } else if (CLIENT_FAILURE_EVENTS[eventType] && res) {
      eventArgs = {};

      // res is an error object
      errorObject = {
        class: "HttpClientError",
        msg: res.message
      }

    } else if (eventType === 'close') {
      eventArgs = {};
    }

    if (eventArgs) {
      var eventNameFormatted = 'httpclient.request';

      if (ctx) {
        if (logger.isDebug()) {
          logger.debug('%s[%d %d %d]: callback', eventNameFormatted, ctx.txid, ctx.lane, ctx.evti);
        }

        ctx = agent.asynchEventDone(ctx, eventNameFormatted, eventArgs, errorObject);
        clientReq.__CA_patched = true;
      }
      else {
        if (logger.isDebug()) {
          logger.debug('%s - no context: callback', eventNameFormatted);
          logger.debug((new Error('No context')).stack);
        }
      }
    }
  };

};

function decorateClientRequest(ctx, clientReq) {
  if (ctx && ctx.corIdObj) {

    var corid = ctx.corIdObj.getOutgoingCorId();
    clientReq.setHeader(CORRELATION_HEADER_KEY, corid);
    logger.debug('Decorated outgoing http request with correlation header %s: %s',
      CORRELATION_HEADER_KEY, corid);
  }
}

function getMethodsWithProbes() {
  if (!targetModule.methodMap) {
    var mt = new Object;
    mt[0] = 'http#all';
    targetModule.methodMap = mt;
  }
  return targetModule.methodMap;
}

function instrument(methodMap) {
  targetModule.methodMap = methodMap;
}

module.exports.getMethodsWithProbes = getMethodsWithProbes;

module.exports.instrument = instrument.bind(module);