midas-core/distribution/midas.js

Enrich data with APIs

'use strict';

Object.defineProperty(exports, "__esModule", {
    value: true
});
exports.default = undefined;

var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; };

var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); // Utils


// Extractors


// Loaders

// import GoogleSpreadSheetLoader from './Loader/GoogleSpreadSheetLoader';

var _ClassLoader = require('./Utils/ClassLoader');

var _ClassLoader2 = _interopRequireDefault(_ClassLoader);

var _ArrayUtils = require('./Utils/ArrayUtils');

var _ArrayUtils2 = _interopRequireDefault(_ArrayUtils);

var _objectPath = require('object-path');

var _objectPath2 = _interopRequireDefault(_objectPath);

var _colors = require('colors');

var _colors2 = _interopRequireDefault(_colors);

var _recursiveIterator = require('recursive-iterator');

var _recursiveIterator2 = _interopRequireDefault(_recursiveIterator);

var _jsonpath = require('jsonpath');

var _jsonpath2 = _interopRequireDefault(_jsonpath);

var _RateLimiter = require('./Utils/RateLimiter');

var _RateLimiter2 = _interopRequireDefault(_RateLimiter);

var _JSONExtractor = require('./Extractor/JSONExtractor');

var _JSONExtractor2 = _interopRequireDefault(_JSONExtractor);

var _CSVExtractor = require('./Extractor/CSVExtractor');

var _CSVExtractor2 = _interopRequireDefault(_CSVExtractor);

var _XLSXExtractor = require('./Extractor/XLSXExtractor');

var _XLSXExtractor2 = _interopRequireDefault(_XLSXExtractor);

var _GoogleSpreadSheetExtractor = require('./Extractor/GoogleSpreadSheetExtractor');

var _GoogleSpreadSheetExtractor2 = _interopRequireDefault(_GoogleSpreadSheetExtractor);

var _MySQLExtractor = require('./Extractor/MySQLExtractor');

var _MySQLExtractor2 = _interopRequireDefault(_MySQLExtractor);

var _JSONLoader = require('./Loader/JSONLoader');

var _JSONLoader2 = _interopRequireDefault(_JSONLoader);

var _CSVLoader = require('./Loader/CSVLoader');

var _CSVLoader2 = _interopRequireDefault(_CSVLoader);

var _XLSXLoader = require('./Loader/XLSXLoader');

var _XLSXLoader2 = _interopRequireDefault(_XLSXLoader);

var _MySQLLoader = require('./Loader/MySQLLoader');

var _MySQLLoader2 = _interopRequireDefault(_MySQLLoader);

var _flat = require('flat');

var _flat2 = _interopRequireDefault(_flat);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }

var Midas = function () {
    function Midas(config) {
        _classCallCheck(this, Midas);

        this.config = config;
        this.config._source = this.config.source[Object.keys(this.config.source)[0]];
        this.config._source.type = Object.keys(this.config.source)[0];
        this.config._target = this.config.target[Object.keys(this.config.target)[0]];
        this.config._target.type = Object.keys(this.config.target)[0];
        this.rp = null;
    }

    _createClass(Midas, [{
        key: '_report_status',
        value: function _report_status(status) {
            console.log(status.message);
        }
    }, {
        key: '_chain_enrichers',
        value: function _chain_enrichers(enrichers, data) {
            var self = this;
            async function chain_enricher_promises(promises, self) {
                var result = [];
                var enrichment_target = null;
                var intermediate_result = null;
                var path_expression = '';
                var parent_object_path = '';
                var parent = {};

                var _iteratorNormalCompletion = true;
                var _didIteratorError = false;
                var _iteratorError = undefined;

                try {
                    for (var _iterator = promises[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {
                        var promise = _step.value;


                        var jp_source_data = _jsonpath2.default.nodes(data, promise.getConfig().input_parameter);
                        var jp_source_data_2 = _jsonpath2.default.nodes(data, '$..price');

                        var enriched_property_name = promise.getConfig().target_property;
                        var _iteratorNormalCompletion2 = true;
                        var _didIteratorError2 = false;
                        var _iteratorError2 = undefined;

                        try {
                            for (var _iterator2 = jp_source_data[Symbol.iterator](), _step2; !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true) {
                                var node = _step2.value;


                                var node_path = node.path.join('.').substring(2); // Remove leading $. from path string for object path 
                                enrichment_target = node.value;

                                promise.setData(enrichment_target);
                                intermediate_result = enrichment_target;
                                intermediate_result = await promise.process(intermediate_result);
                                var deep_copied_intermediate_result = JSON.parse(JSON.stringify(intermediate_result));

                                // Check node type 
                                if (_typeof(node.value) !== 'object' && !Array.isArray(node.value)) {
                                    path_expression = _jsonpath2.default.stringify(node.path);
                                    var node_path_deep_copy = JSON.parse(JSON.stringify(node.path));
                                    //node_path_deep_copy.pop();
                                    node_path_deep_copy.shift();
                                    parent_object_path = node_path_deep_copy.join('.');

                                    // get parent and check if it's an array
                                    parent = _jsonpath2.default.parent(data, path_expression);
                                    if (Array.isArray(parent)) {
                                        var enriched_intermediate_object = { value: node.value };
                                        enriched_intermediate_object[enriched_property_name] = deep_copied_intermediate_result;
                                        _objectPath2.default.set(data, parent_object_path, enriched_intermediate_object);
                                    } else {
                                        parent[enriched_property_name] = deep_copied_intermediate_result;
                                    }
                                }

                                if (_typeof(node.value) === 'object' && !Array.isArray(node.value)) {
                                    _objectPath2.default.set(data, node_path + '.' + enriched_property_name, deep_copied_intermediate_result);
                                }

                                if (Array.isArray(node.value)) {
                                    path_expression = _jsonpath2.default.stringify(node.path);
                                    parent = _jsonpath2.default.parent(data, path_expression);
                                    parent[enriched_property_name] = deep_copied_intermediate_result;
                                }

                                // check if there is some kind of rate limit defined within the configuration
                                if (typeof promise.getConfig().rate_limit !== 'undefined' && typeof promise.getConfig().rate_limit !== 'undefined') {
                                    await _RateLimiter2.default.async_stall(promise.getConfig().rate_limit.number_of_requests, promise.getConfig().rate_limit.time_window);
                                }
                            }
                        } catch (err) {
                            _didIteratorError2 = true;
                            _iteratorError2 = err;
                        } finally {
                            try {
                                if (!_iteratorNormalCompletion2 && _iterator2.return) {
                                    _iterator2.return();
                                }
                            } finally {
                                if (_didIteratorError2) {
                                    throw _iteratorError2;
                                }
                            }
                        }
                    }
                } catch (err) {
                    _didIteratorError = true;
                    _iteratorError = err;
                } finally {
                    try {
                        if (!_iteratorNormalCompletion && _iterator.return) {
                            _iterator.return();
                        }
                    } finally {
                        if (_didIteratorError) {
                            throw _iteratorError;
                        }
                    }
                }

                return { data: result };
            }
            return chain_enricher_promises(enrichers, self);
        }
    }, {
        key: '_instantiate_enricher_classes',
        value: function _instantiate_enricher_classes(enrichers, enricher_classes, data) {
            var _this = this;

            var instances = [];
            enrichers.forEach(function (enricher, index) {
                instances.push(new enricher_classes[index][enricher.name](_this._enricher_add_utils(), data, enrichers[index].config));
            });
            return instances;
        }
    }, {
        key: '_enricher_add_utils',
        value: function _enricher_add_utils() {
            var rp = require('request-promise');
            return rp;
        }
    }, {
        key: '_extractor',
        value: function _extractor() {
            var extractor_type = this.config._source.type.toLowerCase();

            if (extractor_type === 'json') {
                return new _JSONExtractor2.default(this.config);
            }

            if (extractor_type === 'csv') {
                return new _CSVExtractor2.default(this.config);
            }

            if (extractor_type === 'xlsx') {
                return new _XLSXExtractor2.default(this.config);
            }

            if (extractor_type === 'googlespreadsheet') {
                return new _GoogleSpreadSheetExtractor2.default(this.config);
            }

            if (extractor_type === 'mysql') {
                return new _MySQLExtractor2.default(this.config);
            }
        }
    }, {
        key: '_loader',
        value: function _loader() {
            var loader_type = this.config._target.type.toLowerCase();

            if (loader_type === 'json') {
                return new _JSONLoader2.default(this.config);
            }

            if (loader_type === 'csv') {
                return new _CSVLoader2.default(this.config);
            }

            if (loader_type == 'xlsx') {
                return new _XLSXLoader2.default(this.config);
            }

            if (loader_type == 'mysql') {
                return new _MySQLLoader2.default(this.config);
            }
        }
    }, {
        key: '_extract_data',
        value: function _extract_data() {
            return this._extractor().get_data_promise();
        }
    }, {
        key: '_match_path',
        value: function _match_path(path_generic, path_instance) {
            var path_generic_split = path_generic.split('.');
            var path_instance_split = path_instance.split('.');

            var isNumber = function isNumber(val) {
                return (/^\d+$/.test(val)
                );
            };

            // Remove starting $.
            path_generic_split.shift();

            // Compare paths
            var result = true;
            path_generic_split.forEach(function (path_element, index) {
                if (path_element === '[:]') {
                    if (!isNumber(path_instance_split[index])) {
                        result = false;
                    }
                } else {
                    result = result && path_instance_split[index] === path_element && path_generic_split.length == path_instance_split.length;
                }
            });

            return result;
        }
    }, {
        key: 'touch',
        value: function touch() {
            var _this2 = this;

            // LOG
            this._report_status({ message: '👑👑 midas data enrichment process started'.yellow.bold });
            this._report_status({ message: this.config.name + ' || ' + this._extractor().get_type() + '->' + this._loader().get_type() + '\n' });

            var data_promise = this._extract_data();

            // TODO: Fix hardcoded path
            var enricher_classes = _ClassLoader2.default.load_classes(this.config.enrichers);
            var enrichers = this._instantiate_enricher_classes(this.config.enrichers, enricher_classes, {});

            // LOG
            this._report_status({ message: '🔗🔗 Enrichment chain'.bold + '' });
            var status_message = '';
            this.config.enrichers.forEach(function (enricher, index) {
                if (index < _this2.config.enrichers.length - 1) {
                    status_message += enricher.name + ' -> ';
                } else {
                    status_message += enricher.name;
                }
            });
            status_message = status_message.split(' -> ').join(' -> ');
            this._report_status({ message: status_message });

            return data_promise.then(function (data_set) {

                data_set = _ArrayUtils2.default.arrayify(data_set);
                var enrichers_chain = [];
                var enrichment = enrichers_chain.push(_this2._chain_enrichers(enrichers, data_set));

                return Promise.all(enrichers_chain).then(function (enriched_items) {

                    var result = _this2._loader().load(data_set);
                    return result.then(function (res) {
                        // LOG
                        _this2._report_status({ message: '\n✅ midas data enrichment process done'.green.bold });
                        return Promise.resolve(true);
                    });
                }).catch(function (err) {
                    console.log(err);
                    console.log('Something went wrong'.red);
                    return Promise.resolve(false);
                });
            });
            // end of function
        }
    }]);

    return Midas;
}();

exports.default = Midas;