loganalysis/lib/scala.js

Collects stdout and stderr messages from Node.js applications and sends them to the IBM SmartCloud Analytics - Log Analysis service.

/*
 * © Copyright IBM Corp. 2014
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var http = require('http');

var https = require('https');

var util = require('util');

var cloudApp = require('./cloud_app');

var logger = require('./logger').newLogger('scala');

var restClient = require('./rest_client');

var os = require('os');

const SCALA_CLIENT_NAME = 'name';

const BULK_MESSAGE_EMIT_THRESHOLD = 'scala_bulkMessageEmitThreshold';

const BULK_MESSAGE_EMIT_INTERVAL = 'scala_bulkMessageEmitInterval';

const DISABLE_BULK_EMIT_MESSAGES = 'scala_disableBulkEmitMessages';

const DISABLE_AUTOMATIC_CLIENT_SET_UP = 'scala_disableAutomaticClientSetUp';

const INITIALIZATION_RETRY_INTERVAL = 'scala_initializationRetryInterval';

const NODE_EVENT_TYPE_TTL = "scala_ttl";

const SCALA_SERVICE_LABEL = "scala_service_label";

const SCALA_CLIENT_OPTION_NAMES = [
  BULK_MESSAGE_EMIT_THRESHOLD,
  BULK_MESSAGE_EMIT_INTERVAL,
  DISABLE_BULK_EMIT_MESSAGES ,
  DISABLE_AUTOMATIC_CLIENT_SET_UP,
  INITIALIZATION_RETRY_INTERVAL,
  NODE_EVENT_TYPE_TTL
];

const DEFAULT_MESSAGE_THRESHOLD_VALUE = 250000;

const DEFAULT_MESSAGE_SEND_INTERVAL = 10000;

const DEFAULT_INITIALIZATION_RETRY_INTERVAL = 10000;

const DEFAULT_NODE_EVENT_TYPE_TTL = '365d';

const NODE_JS_GENERIC_EVENT_TYPE_NAME = 'node_logs';

const UPDATE_NODE_EVENT_TYPE_TTL_JSON = '{ "ttl" : "%s"}';

const NODE_JS_GENERIC_EVENT_TYPE_JSON = '{ \
  "' + NODE_JS_GENERIC_EVENT_TYPE_NAME + '" : { \
    "_source" : { \
      "compress" : true \
    }, \
    "_ttl" : { \
      "enabled" : true, \
      "default" : "%s" \
    }, \
    "properties" : { \
      "bm_appName" : { \
        "type" : "string" \
      }, \
      "bm_instId" : { \
        "type" : "string" \
      }, \
      "bm_instIdx" : { \
        "type" : "string" \
      }, \
      "bm_appGUID" : { \
        "type" : "string" \
      }, \
      "severity" : { \
        "index" : "not_analyzed", \
        "type" : "string" \
      }, \
      "datetime" : { \
        "type" : "date", \
        "format" : "yyyy-MM-dd\'T\'HH:mm:ss.SSS Z" \
      }, \
      "message" : { \
        "index" : "not_analyzed", \
        "type" : "string" \
      }, \
      "source" : { \
        "index": "not_analyzed", \
        "type": "string", \
        "index_options": "docs", \
        "omit_norms": true \
      } \
    } \
  } \
}';

const SCALA_NODE_JS_EVENT_TYPE_URL = '/Unity/dciface/v1/eventtypes/' + NODE_JS_GENERIC_EVENT_TYPE_NAME;

const SCALA_NODE_JS_BULK_EVENT_POST_URL = '/Unity/dciface/v1/events/' + NODE_JS_GENERIC_EVENT_TYPE_NAME + '?bulk=true';

const SCALA_NODE_JS_EVENT_URL = '/Unity/dciface/v1/events/' + NODE_JS_GENERIC_EVENT_TYPE_NAME;

const BULK_MESSAGES_SEPARATOR = '{"create":{}}\n';

//+1 for \n appended to the messages
const BULK_MESSAGES_SEPARATOR_LENGTH = Buffer.byteLength(BULK_MESSAGES_SEPARATOR) + 1;

function ScalaClient(options) {
  logger.debug('Entering ScalaClient');
  this.name = options[SCALA_CLIENT_NAME];
  this.webClient = restClient.createRestClient(options);
  this.bm_source = options.bm_source;
  this.bm_appGUID = options.bm_appGUID;
  this.options = options;
  this.initialized = false;
  this.buffer = [];
  this.messagesLength = 0;
  this.disableBulkEmitMessages = options[DISABLE_BULK_EMIT_MESSAGES] ? options[DISABLE_BULK_EMIT_MESSAGES] : false;
  this.bulkMessageEmitThreshold = options[BULK_MESSAGE_EMIT_THRESHOLD] || DEFAULT_MESSAGE_THRESHOLD_VALUE;
  this.bulkMessageEmitInterval = options[BULK_MESSAGE_EMIT_INTERVAL] || DEFAULT_MESSAGE_SEND_INTERVAL;
  this.initializationRetryInterval = options[INITIALIZATION_RETRY_INTERVAL] || DEFAULT_INITIALIZATION_RETRY_INTERVAL;
  this.nodeEventTypeTTL = options[NODE_EVENT_TYPE_TTL] || DEFAULT_NODE_EVENT_TYPE_TTL;
  for (var key in options) {
    if (key == 'password') {
      continue;
    }
    logger.debug('create scala client ' + key + ' = ' + options[key]);
  }
  this.bulkEmitInProgress = false;
  logger.debug('Exiting ScalaClient');
}

ScalaClient.prototype.pushToBuffer = function (jsonMsg) {
  logger.debug('Entering ScalaClient.prototype.pushToBuffer');
  this.buffer.push(jsonMsg);
  this.messagesLength += (Buffer.byteLength(jsonMsg) + BULK_MESSAGES_SEPARATOR_LENGTH);
  if (this.messagesLength >= this.bulkMessageEmitThreshold) {
    logger.debug('Buffer threshold reached, flushing forcefully!');
    this.bulkEmitInProgress = false;
    this.flush();
  }
  logger.debug('Exiting ScalaClient.prototype.pushToBuffer');
}

ScalaClient.prototype.clearBuffer = function () {
  logger.debug('Entering ScalaClient.prototype.clearBuffer');
  this.buffer = [];
  this.messagesLength = 0;
  logger.debug('Exiting ScalaClient.prototype.clearBuffer');
}

ScalaClient.prototype.getNodeEventTypeTTL = function () {
  return this.nodeEventTypeTTL;
};

ScalaClient.prototype.setDisableBulkEmitMessages = function (disableBulkEmitMessages) {
  logger.debug('Entering ScalaClient.prototype.setDisableBulkEmitMessages');
  if (this.disableBulkEmitMessages == disableBulkEmitMessages) {
    return;
  }
  this.disableBulkEmitMessages = disableBulkEmitMessages;
  if (this.bulkEmitIntervalId) {
    clearInterval(this.bulkEmitIntervalId);
    this.bulkEmitIntervalId = null;
  }
  if (!this.disableBulkEmitMessages) {
    scalaClient.bulkEmitIntervalId = setInterval(function (scalaClient) {
      logger.debug('Invoking bulkEmit after interval ' + scalaClient.bulkMessageEmitInterval);
      bulkEmit(scalaClient);
    }, this.bulkMessageEmitInterval, this);
  }
  logger.debug('Exiting ScalaClient.prototype.setDisableBulkEmitMessages');
};

ScalaClient.prototype.getInitializationRetryInterval = function () {
  return this.initializationRetryInterval;
};

ScalaClient.prototype.getName = function () {
  return this.name;
};

/**
 * Send one message to scala server
 * @param message raw message contents, of string literal type
 * @param level message level
 */
ScalaClient.prototype.emit = function (msg, level) {
  logger.debug('Entering ScalaClient.prototype.emit');
  logger.debug('Log to be pushed to SCALA: ' + msg);
  var jsonObj = {
    message: msg,
    severity: level,
    datetime: this.formateDate(new Date()),
    source: cloudApp.bm_source,
    bm_appGUID: this.webClient.app_guid,//bm_appGUID: cloudApp.app_guid,
    bm_instId: cloudApp.instance_id,
    bm_appName: cloudApp.application_name,
    bm_instIdx: cloudApp.instance_index
  };
  var jsonMsg = JSON.stringify(jsonObj);

  //If the client is not initialized and the buffer size is less than the threshold value, the client will cache the messages in the buffer temporarily.
  if (!this.initialized) {
    logger.debug('Pushing a message to buffer for bulk-Emitting, as the client it not yet initialized');
    this.pushToBuffer(jsonMsg);
    return;
  }

  if (this.disableBulkEmitMessages) {
    //flush the buffer, as it may cache the messages before the the client is initialized
    if (this.messagesLength > 0) {
      this.flush();
    }
    var webClient = this.webClient;
    var _this = this;
    webClient.postOnce(SCALA_NODE_JS_EVENT_URL, jsonMsg, function (res) {
      res.on('error', function (e) {
        logger.error('Failed to send messages to scala server due to ' + (e && e.message ? e.message : ''));
      });
      if (res.statusCode != 201 && res.statusCode != 200) {
        var resBody = '';
        res.on('data', function (data) {
          resBody += data;
        });
        res.on('end', function () {
          logger.error('Failed to send messages to scala server, statusCode = ' + res.statusCode + ", response body = " + resBody);
          if (res.statusCode == 302) {
            logger.info('Received statuc code 302 in ScalaClient.prototype.emit, pushing to buffer, for bulk emitting later');
            webClient.reinit();
            this.pushToBuffer(jsonMsg);
          }
        });
      } else {
        logger.debug('Succssfully pushed message to SCALA');
        //Ignore the response data
        res.resume();
      }
    }, null, {error: sendMessageErrorCallback});
  } else {
    logger.debug('Pushing a message to buffer for bulk-Emitting');
    this.pushToBuffer(jsonMsg);
  }
  logger.debug('Exiting ScalaClient.prototype.emit');
};

/**
 *  Flush messages in the buffer to scala server
 */
ScalaClient.prototype.flush = function () {
  bulkEmit(this);
};

function bulkEmit(scalaClient) {
  logger.debug('Entering bulkEmit');
  if (scalaClient.webClient.reinitInProgress === true) {
    logger.debug('Restclient authenitcation is in progress, exiting bulkEmit');
    return;
  }
  if (scalaClient.bulkEmitInProgress === true) {
    logger.debug('bulkEmit is already in progress, exiting bulkEmit');
    return;
  }
  if (scalaClient.buffer.length == 0) {
    logger.debug('Nothing in the buffer, exiting bulkEmit');
    return;
  }

  //clear the buffer in case the scala client fails to initialize
  if (!scalaClient.initialized) {
    logger.error(scalaClient.buffer.length + ' logging records in the buffer are dropped as the scala client failed to initialize');
    scalaClient.clearBuffer();
    return;
  }
  try {
    scalaClient.bulkEmitInProgress = true;
    var webClient = scalaClient.webClient;
    var messages = scalaClient.buffer;
    var msgLength = scalaClient.messagesLength;
    scalaClient.clearBuffer();
    var req = webClient.post(SCALA_NODE_JS_BULK_EVENT_POST_URL, function (res) {
      res.on('error', function (e) {
        logger.error('Failed to send messages to scala server due to ' + (e && e.message ? e.message : ''));
      });
      var resBody = '';
      res.on('data', function (data) {
        resBody += data;
      });
      res.on('end', function () {
        logger.debug('bulkEmit to scala server, statusCode = ' + res.statusCode + ", response body = " + resBody);
        if (res.statusCode != 201 && res.statusCode != 200) {
          if (res.statusCode == 302) {
            logger.info('Received status code 302 in bulkEmit, trying to re-login');
            webClient.reinit();
            scalaClient.buffer = messages.concat(scalaClient.buffer);
            scalaClient.messagesLength += msgLength;
          }
        } else {
          logger.debug('Successfully bulkEmitted buffer of length ' + msgLength + ' to SCALA');
          res.resume();
        }
      });
    }, {headers: {'Content-Length': msgLength}}, {error: sendMessageErrorCallback  });
    for (var j = 0, length = messages.length; j < length; j++) {
      req.write(BULK_MESSAGES_SEPARATOR);
      req.write(messages[j]);
      req.write('\n');
    }
    req.end();
  } catch(e) {
      logger.error('BulkEmit falled due to ' + e);
  } finally {
    scalaClient.bulkEmitInProgress = false;
  }
  logger.debug('Exiting bulkEmit');
}

/**
 * Query Scala server and create node_application event type if not exists
 */
ScalaClient.prototype.setUp = function () {
  internalSetUp(this);
};

function internalSetUp(scalaClient) {
  logger.debug('Entering internalSetUp');
  var webClient = scalaClient.webClient;
  setTimeout(function(){
    var getReq = webClient.get(SCALA_NODE_JS_EVENT_TYPE_URL, function (res) {
      var getResponseData = '';
      res.on('data', function (data) {
        getResponseData += data;
      });
      res.on('end', function () {
        logger.info('Response for querying event type ' + getResponseData);
      });
      res.on('error', function (e) {
        logger.error('Failed to query event type due to ' + (e && e.message ? e.message : ''));
      });
      if (res.statusCode == 200) {
        //no action since the nodeAppLog event has been created
        logger.info('Event type ' + NODE_JS_GENERIC_EVENT_TYPE_NAME + ' exists in scala server ' + webClient.getServerName());
        internalInitialize(scalaClient);
      } else if (res.statusCode == 404) {
        createEventType(scalaClient);
      } else {
        logger.error('Failed to query event type ' + NODE_JS_GENERIC_EVENT_TYPE_NAME + ' on the server ' + webClient.getServerName() + ', it will be  retried after ' + scalaClient.getInitializationRetryInterval());
        logger.error('The error response headers for querying event type is ' + util.inspect(res.headers));
        setTimeout(internalSetUp, scalaClient.getInitializationRetryInterval(), scalaClient);
      }
    });
    getReq.on('error', function (e) {
      logger.error('Failed to query event type ' + NODE_JS_GENERIC_EVENT_TYPE_NAME + ' due to ' + (e && e.message ? e.message : '') + ', it will be retried after ' + scalaClient.getInitializationRetryInterval());
      setTimeout(internalSetUp, scalaClient.getInitializationRetryInterval(), scalaClient);
    });
    getReq.end();
  }, scalaClient.getInitializationRetryInterval());
  logger.debug('Exiting internalSetUp');
}

function createEventType(scalaClient) {
  logger.debug('Entering createEventType');
  var webClient = scalaClient.webClient;
  //Create event type
  var nodeEventTypeJson = util.format(NODE_JS_GENERIC_EVENT_TYPE_JSON, scalaClient.getNodeEventTypeTTL());
  var putReq = webClient.put(SCALA_NODE_JS_EVENT_TYPE_URL, nodeEventTypeJson, function (res) {
    var putResponseData = '';
    res.on('data', function (data) {
      putResponseData += data;
    });
    res.on('end', function () {
      logger.info('Response for creating event type ' + putResponseData);
      putResponseData = null;
    });
    res.on('error', function (e) {
      logger.error('Failed to create event type ' + nodeEventTypeJson + ' due to ' + (e && e.message ? e.message : '') + ', it will be retried after ' + scalaClient.getInitializationRetryInterval());
      setTimeout(createEventType, scalaClient.getInitializationRetryInterval(), scalaClient);
    });

    var statusCode = res.statusCode;
    if (statusCode == 200) {
      logger.info('Successfully create ' + nodeEventTypeJson + ' on the server ' + webClient.getServerName());
      internalInitialize(scalaClient);
    } else {
      logger.error('Failed to create event type ' + nodeEventTypeJson + ' on the server  ' + webClient.getServerName() + ', response status code = ' + statusCode);
    }
  });

  putReq.on('error', function (e) {
    logger.error('Failed to create event type ' + nodeEventTypeJson + ' due to ' + (e && e.message ? e.message : '') + ', it will be retried after ' + scalaClient.getInitializationRetryInterval());
    setTimeout(createEventType, scalaClient.getInitializationRetryInterval(), scalaClient);
  });
  putReq.end();
  logger.debug('Exiting createEventType');
}

function internalInitialize(scalaClient) {
  logger.debug('Entering internalInitialize');
  scalaClient.initialized = true;
  if (!scalaClient.disableBulkEmitMessages) {
    scalaClient.bulkEmitIntervalId = setInterval(function () {
      logger.debug('Invoking bulkEmit after interval ' + scalaClient.bulkMessageEmitInterval);
      bulkEmit(scalaClient);
    }, scalaClient.bulkMessageEmitInterval, scalaClient);
  }
  logger.debug('Exiting internalInitialize');
}

ScalaClient.prototype.cleanUp = function () {
  logger.debug('Entering ScalaClient.prototype.cleanUp');
  if (this.bulkEmitIntervalId) {
    clearInterval(this.bulkEmitIntervalId);
    this.bulkEmitIntervalId = null;
  }
  logger.debug('Exiting ScalaClient.prototype.cleanUp');
};

function sendMessageErrorCallback(e) {
  logger.error('Failed to send messages to scala server, the error message is ' + (e && e.message ? e.message : ''));
}

/**
 * Return date in format: "yyyy-MM-dd'T'HH:mm:ss.SSS Z"
 */
ScalaClient.prototype.formateDate = function (date) {
  var ret = date.getFullYear() + '-';

  var month = date.getMonth() + 1;
  ret += (month > 9 ? month : '0' + month) + '-';

  var day = date.getDate();
  ret += (day > 9 ? day : '0' + day) + 'T';

  var hours = date.getHours();
  ret += (hours > 9 ? hours : '0' + hours) + ':';

  var minutes = date.getMinutes();
  ret += (minutes > 9 ? minutes : '0' + minutes) + ':';

  var seconds = date.getSeconds();
  ret += (seconds > 9 ? seconds : '0' + seconds) + '.';

  var millisecs = date.getMilliseconds();
  if (millisecs < 9) {
    ret += '00';
  } else if (millisecs < 99) {
    ret += '0';
  }
  ret += millisecs + ' ' + date.toString().match(/([-\+][0-9]+)\s/)[1];

  return ret;
};

module.exports.createScalaClient = function (options) {
  logger.debug('Entering ScalaClient.prototype.createScalaClient');
  var clientOptions = JSON.parse(JSON.stringify(options));

  //Backward Compatibility for serviceURL option name
  if (clientOptions.serviceURL && !clientOptions.url) {
    clientOptions.url = clientOptions.serviceURL;
  }

  //Add environment configurations if not specified in the options
  for (var j = 0, jLength = SCALA_CLIENT_OPTION_NAMES.length; j < jLength; j++) {
    var optionName = SCALA_CLIENT_OPTION_NAMES[j];
    if (clientOptions[optionName]) {
      continue;
    }
    if (process.env[optionName]) {
      clientOptions[optionName] = process.env[optionName];
    }
  }
  var scalaClient = new ScalaClient(clientOptions);
  if (!process.env[DISABLE_AUTOMATIC_CLIENT_SET_UP]) {
    scalaClient.setUp();
  }
  return scalaClient;
  logger.debug('Exiting ScalaClient.prototype.createScalaClient');
};

/**
 * Read the scala services from VCAP_SERVICES environment variable, and create scala clients.
 * Accepted format :
 *
 * @returns {Array}
 */
module.exports.createCloudAwareScalaClients = function () {
  logger.debug('Entering ScalaClient.prototype.createCloudAwareScalaClients');
  var scalaClients = [];
  var scalaServices = cloudApp.findVCAPServicesByType(process.env[SCALA_SERVICE_LABEL] || "MonitoringAndAnalytics");
  logger.info('Creating ' + scalaServices.length + ' SCALA clients');
  for (var i = 0, iLength = scalaServices.length; i < iLength; i++) {
    var name = scalaServices[i].name;
    var credentials = scalaServices[i].credentials;

    if (credentials && credentials.scala_url && credentials.scala_password && credentials.scala_userid) {
      var clientOptions = {
        url: credentials.scala_url,
        username: credentials.scala_userid,
        password: credentials.scala_password,
        //keyfile: credentials.keyfile,
        app_guid: credentials.app_guid,
        name: name,
        isCloud: cloudApp.isRunninginCloud
      }

      for (var j = 0, jLength = SCALA_CLIENT_OPTION_NAMES.length; j < jLength; j++) {
        var optionName = SCALA_CLIENT_OPTION_NAMES[j];
        if (process.env[optionName]) {
          clientOptions[optionName] = process.env[optionName];
        }
      }
      var scalaClient = new ScalaClient(clientOptions);
      if (!process.env[DISABLE_AUTOMATIC_CLIENT_SET_UP]) {
        scalaClient.setUp();
      }
      scalaClients.push(scalaClient);
    } else {
      logger.error('Unable to find expected credentials from service ' + JSON.stringify(scalaServices[i]));
    }
  }
  logger.debug('Exiting ScalaClient.prototype.createCloudAwareScalaClients');
  return scalaClients;
};

module.exports.ScalaClient = ScalaClient;

module.exports.BULK_MESSAGE_EMIT_THRESHOLD = BULK_MESSAGE_EMIT_THRESHOLD;

module.exports.BULK_MESSAGE_EMIT_INTERVAL = BULK_MESSAGE_EMIT_INTERVAL;

module.exports.SCALA_CLIENT_NAME = SCALA_CLIENT_NAME;

module.exports.DISABLE_BULK_EMIT_MESSAGES = DISABLE_BULK_EMIT_MESSAGES;

module.exports.DISABLE_AUTOMATIC_CLIENT_SET_UP = DISABLE_AUTOMATIC_CLIENT_SET_UP;

module.exports.NODE_EVENT_TYPE_TTL = NODE_EVENT_TYPE_TTL;

module.exports.INITIALIZATION_RETRY_INTERVAL = INITIALIZATION_RETRY_INTERVAL;