joola.io.engine/lib/caching/fork_processDocument.js

joola.io's Framework Engine

var
  logger = require('joola.io.logger'),
  crypto = require('crypto');

Date.prototype.fixDate = function (force, direction, isZ) {
  date = this;
  if (!force)
    return date;

  var sign;
  if (!direction)
    sign = (date.getTimezoneOffset() > 0) ? 1 : -1;
  else
    sign = (date.getTimezoneOffset() > 0) ? -1 : 1;

  var offset = Math.abs(date.getTimezoneOffset());
  var hours = Math.floor(offset / 60);
  if (isZ) {
    date.addHours(hours);
  }
  date.setHours(date.getHours() + (sign * hours));

  return date;
};

String.prototype.clean = function () {
  var result = this;
  result = result.replace(/ /g, '');
  result = result.replace(/\./g, '');
  result = result.replace(/-/g, '');
  result = result.replace(/\\/g, '');
  result = result.replace(/\//g, '');
  result = result.replace(/:/g, '');
  result = result.replace(/{/g, '');
  result = result.replace(/}/g, '');
  result = result.replace(/"/g, '');
  return result;
};

try {
  logger.debug('Processing node started, pid: ' + process.pid);

  process.on('message', function (args) {
    args = require('JASON').parse(args);
    var dimensions = args.dimensions;
    var metrics = args.metrics;
    var chunk = args.chunk;

    var rowIndex = 1;
    chunk.forEach(function (row) {
      var document = {};

      document.date = new Date(row['date']).fixDate(true, true);

      var _date = new Date(JSON.parse(JSON.stringify(document.date)));

      var bucket = {};
      _date.setMilliseconds(0);
      bucket['second'] = new Date(_date);
      _date.setSeconds(0);
      bucket['minute'] = new Date(_date);
      _date.setMinutes(0);
      bucket['hour'] = new Date(_date);
      _date.setUTCHours(0, 0, 0, 0);
      bucket['day'] = new Date(_date);
      _date.setDate(1);
      bucket['month'] = new Date(_date);
      _date.setMonth(1);
      bucket['year'] = new Date(_date);

      document.timebucket = bucket;

      var rowKey = {date: document.date };
      //document.handled = false;
      dimensions.forEach(function (d) {
        var _name = (d.alias ? d.alias : d.name);
        if (row[_name] == '')
          row[_name] = '(not set)';
        document[_name] = row[_name];
        rowKey[_name] = row[_name];
      });
      metrics.forEach(function (m) {
        var _name = m.name;
        try {
          var value = parseFloat(row[_name]);
          if (isNaN(value))
            value = null;
          document[_name] = value;
        }
        catch (ex) {
          console.log('ex: ' + ex.message);
        }
      });

      try {
        var md5sum = crypto.createHash('md5');
        md5sum.update(JSON.stringify(rowKey).clean());
        var _key = md5sum.digest('hex');
        document.key = _key
        document.rowIndex = rowIndex;

        process.send({type: 'document', document: document});
        //var size = (Buffer.byteLength(JSON.stringify(document), 'utf-8') / 1024 / 1024);
        //size = (Math.round(size * 100) / 100).toString() + 'MB';
        //var message = rowIndex + '/' + chunk.length + ', size: ' + size;
      }
      catch (ex) {
        logger.error('Processing node error, pid: ' + process.pid + ': ' + ex);
        process.send({type: 'error', error: ex});
      }

      rowIndex++;
    });

    process.send({type: 'done'});
    logger.debug('Processing node done, pid: ' + process.pid);
  });
}
catch (ex) {
  logger.error('Processing node error, pid: ' + process.pid + ': ' + ex);
  process.send({type: 'error', error: ex});
}

process.send({type: 'init'});