loopback-connector-firehose/lib/firehose.js

Loopback.js connector for Amazon Kinesis Firehose

'use strict'
var AWS = require('aws-sdk');
var assert = require('assert');
var _ = require('lodash');

exports.initialize = function ( dataSource, cb ) {

  var settings = dataSource.settings;
  var connector = new FirehoseConnector( settings );
  dataSource.connector = connector;
  connector.DataAccessObject = function () {};

  for (var m in FirehoseConnector.prototype) {
    var method = FirehoseConnector.prototype[m];
    if ('function' === typeof method) {
      connector.DataAccessObject[m] = method.bind(connector);
      for(var k in method) { 
        connector.DataAccessObject[m][k] = method[k];
      }
    }
  }

  cb && cb();
}

var firehose;
function FirehoseConnector( settings ) {

  this.name = 'firehose';
  this._models = {};

  assert( typeof settings === 'object',
          'cannot initialize FirehoseConnector without a settings object' );
  assert( typeof settings.region === 'string',
          'cannot initialize FirehoseConnector without a region' );
  assert( typeof settings.DeliveryStreamName === 'string',
          'cannot initialize FirehoseConnector without a DeliveryStreamName' );
  assert( typeof settings.ColumnNames === 'object',
          'cannot initialize FirehoseConnector without a ColumnNames' );
  
  var connector = this;

  var region = this.region = settings.region;
  var DeliveryStreamName = this.DeliveryStreamName = settings.DeliveryStreamName;
  var ColumnNames = this.ColumnNames = settings.ColumnNames;
  
  AWS.config.region = region;

  firehose = new AWS.Firehose({apiVersion: '2015-08-04'});
}

FirehoseConnector.prototype.send = function ( data, cb ) {

  let columnNames = this.ColumnNames;

  let streamName = this.DeliveryStreamName;

  var hasDynamicDeliveryStream = function ( dataObj ) {
    if ( _.has( dataObj, 'DeliveryStreamName' ) )
      return true;
    return false;
  };

  var getDynamicDeliveryStream = function ( dataObj ) {
    return dataObj['DeliveryStreamName'];
  };

  var checkValidSchema = function ( schema, dataObj ) {
    let res = _.reduce( schema, function ( count, validParam ) {
      let hasParam = _.has( dataObj, validParam );
      if ( hasParam === true )
        return count + 1;
      return count;
    }, 0 );
    if ( res === schema.length ) {
      return true;
    }
    return false;
  };

  if ( checkValidSchema( columnNames, data ) === false )
    return cb( new Error( 'Invalid schema' ) );

  if ( hasDynamicDeliveryStream( data ) === true )
    streamName = getDynamicDeliveryStream( data );

  var convertJsonToRecord = function ( schema, dataObj ) {
    let res = '';
    _.forEach( schema, function ( field ) {
      res = res + dataObj[field] + '|';
    });
    res += '\n';
    return res;
  }

  var delimitedData = '';
  delimitedData = convertJsonToRecord( columnNames, data );

  var params = {
    DeliveryStreamName: streamName,
    Record: { Data: delimitedData }
  };

  firehose.putRecord( params, function ( err, res ) {
    if (err) {
      return cb( err );
    } else {
      return cb( null, res );
    }
  });
}

function setRemoting( fn, options ) {
  options = options || {};
  for (var opt in options) {
    if (options.hasOwnProperty(opt)) {
      fn[opt] = options[opt];
    }
  }
  fn.shared = true;
}

setRemoting( FirehoseConnector.prototype.send, {
  description: 'Send a pipe delimited data to Firehose service given a JSON',
  accepts: [
    {
      arg: 'data',
      type: 'object',
      description: 'Data to be sent',
      http: { source: 'body' }
    },
  ],
  returns: { arg: 'data', type: 'object', root: true },
  http: { verb: 'post', path: '/' }
});