firehoser_es5/lib/firehoser.js

es5 version of firehoser. A wrapper around AWS Kinesis Firehose with retry logic and custom queuing behavior.

'use strict';

var _get = function get(object, property, receiver) { if (object === null) object = Function.prototype; var desc = Object.getOwnPropertyDescriptor(object, property); if (desc === undefined) { var parent = Object.getPrototypeOf(object); if (parent === null) { return undefined; } else { return get(parent, property, receiver); } } else if ("value" in desc) { return desc.value; } else { var getter = desc.get; if (getter === undefined) { return undefined; } return getter.call(receiver); } };

var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();

var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();

function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } }

function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; }

function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; }

function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }

// require("babel-polyfill");

var _ = require('lodash');
var AWS = require('aws-sdk');
var async = require('async');
var JaySchema = require('jayschema');
var moment = require('moment');
var Promise = require('es6-promise').Promise;


var schemaValidator = new JaySchema();

var DeliveryStream = function () {
    function DeliveryStream(name) {
        var awsConfig = arguments.length <= 1 || arguments[1] === undefined ? null : arguments[1];
        var schema = arguments.length <= 2 || arguments[2] === undefined ? null : arguments[2];
        var retryInterval = arguments.length <= 3 || arguments[3] === undefined ? 1500 : arguments[3];
        var firehose = arguments.length <= 4 || arguments[4] === undefined ? null : arguments[4];
        var logger = arguments.length <= 5 || arguments[5] === undefined ? null : arguments[5];

        _classCallCheck(this, DeliveryStream);

        this.maxIngestion = 400;
        this.maxDrains = 3;
        this.maxRetries = 40;
        this.name = name;
        if (awsConfig !== null) {
            AWS.config.update(awsConfig);
        }
        this.schema = schema;
        this.retryInterval = retryInterval;
        this.firehose = firehose ? firehose : new AWS.Firehose({ params: { DeliveryStreamName: name } });
        this.log = logger ? logger : function () {};
    }

    _createClass(DeliveryStream, [{
        key: 'validateRecord',
        value: function validateRecord(record) {
            return schemaValidator.validate(record, this.schema);
        }
    }, {
        key: 'validateRecords',
        value: function validateRecords(records) {
            var _this = this;

            if (!this.schema) {
                return [records, []];
            }
            var validRecords = [];
            var invalidRecords = [];

            _.forEach(records, function (record) {
                var validationErrors = _this.validateRecord(record);
                if (_.isEmpty(validationErrors)) {
                    validRecords.push(record);
                } else {
                    var ve = validationErrors[0];
                    invalidRecords.push({
                        type: "schema",
                        originalRecord: record,
                        description: buildSchemaErrorDescription(ve),
                        details: ve
                    });
                }
            });
            return [validRecords, invalidRecords];
        }
    }, {
        key: 'formatRecord',
        value: function formatRecord(record) {
            return { Data: record + '\n' };
        }
    }, {
        key: 'putRecord',
        value: function putRecord(record) {
            return this.putRecords([record]);
        }
    }, {
        key: 'putRecords',
        value: function putRecords(records) {
            var _this2 = this;

            this.log('DeliveryStream.putRecords() called with ' + records.length + ' records.');
            return new Promise(function (resolve, reject) {
                // Validate records against a schema, if necessary.

                var _validateRecords = _this2.validateRecords(records);

                var _validateRecords2 = _slicedToArray(_validateRecords, 2);

                var validRecords = _validateRecords2[0];
                var invalidRecords = _validateRecords2[1];

                // Split the records into reasonably-sized chunks.

                records = _.map(validRecords, _this2.formatRecord);
                var chunks = _.chunk(records, _this2.maxIngestion);
                var tasks = [];
                for (var i = 0; i < chunks.length; i++) {
                    tasks.push(_this2.drain.bind(_this2, chunks[i]));
                }

                // Schedule the chunks all at the same time.
                _this2.log('Kicking off ' + tasks.length + ' calls to drain() for ' + records.length + ' records.');
                async.parallelLimit(tasks, _this2.maxDrains, function (err, results) {
                    var allErrors = invalidRecords.concat(_.flatten(results));
                    if (err || !_.isEmpty(allErrors)) {
                        return reject(allErrors);
                    }
                    return resolve();
                });
            });
        }
    }, {
        key: 'drain',
        value: function drain(records, cb) {
            var numRetries = arguments.length <= 2 || arguments[2] === undefined ? 0 : arguments[2];

            var leftovers = [];
            this.log('Draining ' + records.length + ' records.  Pass #' + (numRetries + 1));
            this.firehose.putRecordBatch({ Records: records }, function (firehoseErr, resp) {
                // Stuff broke!
                if (firehoseErr) {
                    return cb(null, {
                        type: "firehose",
                        description: "Internal aws-sdk error.",
                        details: firehoseErr,
                        originalRecord: null
                    });
                }

                // Not all records make it in, but firehose keeps on chugging!
                if (resp.FailedPutCount > 0) {}

                // Push errored records back into the next list.
                var _iteratorNormalCompletion = true;
                var _didIteratorError = false;
                var _iteratorError = undefined;

                try {
                    for (var _iterator = _.zip(records, resp.RequestResponses)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {
                        var _step$value = _slicedToArray(_step.value, 2);

                        var orig = _step$value[0];
                        var result = _step$value[1];

                        if (!_.isUndefined(result.ErrorCode)) {
                            this.log('Got ErrorCode ' + result.ErrorCode + ' for record ' + orig);
                            leftovers.push({
                                type: "firehose",
                                description: result.ErrorMessage,
                                details: {
                                    ErrorCode: result.ErrorCode,
                                    ErrorMessage: result.ErrorMessage
                                },
                                originalRecord: orig
                            });
                        }
                    }

                    // Recurse!
                } catch (err) {
                    _didIteratorError = true;
                    _iteratorError = err;
                } finally {
                    try {
                        if (!_iteratorNormalCompletion && _iterator.return) {
                            _iterator.return();
                        }
                    } finally {
                        if (_didIteratorError) {
                            throw _iteratorError;
                        }
                    }
                }

                if (leftovers.length && numRetries < this.maxRetries) {
                    // We're about to recurse, let the child handle storing error details.
                    leftovers = _.map(leftovers, function (leftover) {
                        return _.pick(leftover, ['originalRecord']);
                    });
                    return setTimeout(function () {
                        this.drain.bind(this, leftovers, cb, numRetries + 1);
                    }, this.retryInterval);
                } else {
                    return cb(null, leftovers);
                }
            });
        }
    }]);

    return DeliveryStream;
}();

var JSONDeliveryStream = function (_DeliveryStream) {
    _inherits(JSONDeliveryStream, _DeliveryStream);

    function JSONDeliveryStream() {
        _classCallCheck(this, JSONDeliveryStream);

        return _possibleConstructorReturn(this, Object.getPrototypeOf(JSONDeliveryStream).apply(this, arguments));
    }

    _createClass(JSONDeliveryStream, [{
        key: 'formatRecord',
        value: function formatRecord(record) {
            return _get(Object.getPrototypeOf(JSONDeliveryStream.prototype), 'formatRecord', this).call(this, JSON.stringify(record));
        }
    }]);

    return JSONDeliveryStream;
}(DeliveryStream);

var QueuableDeliveryStream = function (_DeliveryStream2) {
    _inherits(QueuableDeliveryStream, _DeliveryStream2);

    function QueuableDeliveryStream(name) {
        var _Object$getPrototypeO;

        var maxTime = arguments.length <= 1 || arguments[1] === undefined ? 30000 : arguments[1];
        var maxSize = arguments.length <= 2 || arguments[2] === undefined ? 500 : arguments[2];

        _classCallCheck(this, QueuableDeliveryStream);

        for (var _len = arguments.length, args = Array(_len > 3 ? _len - 3 : 0), _key = 3; _key < _len; _key++) {
            args[_key - 3] = arguments[_key];
        }

        var _this4 = _possibleConstructorReturn(this, (_Object$getPrototypeO = Object.getPrototypeOf(QueuableDeliveryStream)).call.apply(_Object$getPrototypeO, [this, name].concat(args)));

        _this4.queue = [];
        _this4.timeout = null;
        _this4.maxTime = maxTime;
        _this4.maxSize = maxSize;
        _this4.promise = null;
        setInterval(_this4.drainQueue.bind(_this4), _this4.maxTime);
        return _this4;
    }

    _createClass(QueuableDeliveryStream, [{
        key: 'putRecords',
        value: function putRecords(records) {
            var _queue,
                _this5 = this;

            this.log('QueuableDeliveryStream.putRecords() called with ' + records.length + ' records.');
            (_queue = this.queue).push.apply(_queue, _toConsumableArray(records));
            if (this.promise === null) {
                this.promise = new Promise(function (resolve, reject) {
                    _this5.resolver = resolve;
                    _this5.rejecter = reject;
                });
            }
            this.log('queue size is: ' + this.queue.length + ', maxSize is: ' + this.maxSize + '.');
            if (this.queue.length >= this.maxSize) {
                // Queue's full!
                this.log('queue is full, draining immediately.');
                setImmediate(this.drainQueue.bind(this));
            }
            return this.promise;
        }
    }, {
        key: 'drainQueue',
        value: function drainQueue() {
            var _this6 = this;

            this.log('Countdown timer expired or queue limit reached.');
            this.log('Time to drain the queue of ' + this.queue.length + ' records.');
            var toQueue = this.queue.splice(0, this.queue.length);
            if (!toQueue.length) {
                this.log('No records in queue, not draining anything.');
                return;
            }
            _get(Object.getPrototypeOf(QueuableDeliveryStream.prototype), 'putRecords', this).call(this, toQueue).then(this.resolver, this.rejecter).then(function () {
                _this6.promise = null;
                _this6.rejecter = null;
                _this6.resolver = null;
            });
        }
    }]);

    return QueuableDeliveryStream;
}(DeliveryStream);

var QueuableJSONDeliveryStream = function (_QueuableDeliveryStre) {
    _inherits(QueuableJSONDeliveryStream, _QueuableDeliveryStre);

    function QueuableJSONDeliveryStream() {
        _classCallCheck(this, QueuableJSONDeliveryStream);

        return _possibleConstructorReturn(this, Object.getPrototypeOf(QueuableJSONDeliveryStream).apply(this, arguments));
    }

    _createClass(QueuableJSONDeliveryStream, [{
        key: 'formatRecord',
        value: function formatRecord(record) {
            return _get(Object.getPrototypeOf(QueuableJSONDeliveryStream.prototype), 'formatRecord', this).call(this, JSON.stringify(record));
        }
    }]);

    return QueuableJSONDeliveryStream;
}(QueuableDeliveryStream);

function buildSchemaErrorDescription(ve) {
    if (ve.desc) {
        return ve.desc;
    }
    var field = ve.instanceContext.replace(/(#\/)|(#)/ig, "").replace(/\//g, ".");
    return (ve.kind || 'Error') + ' on \'' + field + '\'.  Expected ' + ve.constraintName + ' to be ' + ve.constraintValue + ', actual value was ' + ve.testedValue + '.';
}

function makeRedshiftTimestamp(input) {
    return moment(input).utc().format('YYYY-MM-DD HH:mm:ss');
}

module.exports = {
    DeliveryStream: DeliveryStream,
    JSONDeliveryStream: JSONDeliveryStream,
    QueuableDeliveryStream: QueuableDeliveryStream,
    QueuableJSONDeliveryStream: QueuableJSONDeliveryStream,
    makeRedshiftTimestamp: makeRedshiftTimestamp
};