midas-core/distribution/Wizard/wizard.js

Enrich data with APIs

'use strict';

Object.defineProperty(exports, "__esModule", {
	value: true
});
exports.default = undefined;

var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; };

var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); // Utils


// Utils


var _fsExtra = require('fs-extra');

var _fsExtra2 = _interopRequireDefault(_fsExtra);

var _recursiveIterator = require('recursive-iterator');

var _recursiveIterator2 = _interopRequireDefault(_recursiveIterator);

var _camelcase = require('camelcase');

var _camelcase2 = _interopRequireDefault(_camelcase);

var _TypeChecker = require('./../Utils/TypeChecker');

var _TypeChecker2 = _interopRequireDefault(_TypeChecker);

var _ArrayUtils = require('./../Utils/ArrayUtils');

var _ArrayUtils2 = _interopRequireDefault(_ArrayUtils);

var _csvjson = require('csvjson');

var _csvjson2 = _interopRequireDefault(_csvjson);

var _xlsx = require('xlsx');

var _xlsx2 = _interopRequireDefault(_xlsx);

var _readJsonSync = require('read-json-sync');

var _readJsonSync2 = _interopRequireDefault(_readJsonSync);

var _jsonpath = require('jsonpath');

var _jsonpath2 = _interopRequireDefault(_jsonpath);

var _inquirer = require('inquirer');

var _inquirer2 = _interopRequireDefault(_inquirer);

var _colors = require('colors');

var _colors2 = _interopRequireDefault(_colors);

var _prettyjson = require('prettyjson');

var _prettyjson2 = _interopRequireDefault(_prettyjson);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }

var Wizard = function () {
	function Wizard() {
		_classCallCheck(this, Wizard);
	}

	_createClass(Wizard, [{
		key: '_transform_to_config_object',
		value: function _transform_to_config_object(object) {
			var config = { source: {}, target: {}, enrichers: [] };
			var rate_limit = { number_of_requests: -1, time_window: 's' };

			// Source
			config.name = object.name;
			config.source[object.source_type] = {};
			config.source[object.source_type].path = object.source_path;

			// Enricher
			var enricher_name = this._has_custom_enricher(object) ? (0, _camelcase2.default)(object.custom_enricher) : (0, _camelcase2.default)(object.enricher);

			// check if rate limit is set
			if (typeof object.rate_limit !== 'undefined') {
				var limits = object.rate_limit.split('/');

				if (['s', 'm', 'h'].indexOf(limits[1]) > -1) {
					rate_limit.number_of_requests = limits[0];
					rate_limit.time_window = limits[1];
				}
			}

			if (object.enricher !== 'no') {
				config.enrichers = [{
					name: enricher_name,
					path: this._resolve_enricher_parent_path(enricher_name),
					config: {
						input_parameter: object.source_object_path,
						target_property: object.target_property_name,
						rate_limit: rate_limit
					}
				}];
			} else {
				config.enrichers = [{
					name: "NAME_OF_YOUR_ENRICHER",
					path: "ABSOLUTE_PATH_TO_YOUR_ENRICHER",
					config: {
						input_parameter: object.source_object_path,
						target_property: object.target_property_name,
						rate_limit: rate_limit
					}
				}];
			}

			// Add configs for pre-defined enricher
			if (!this._has_custom_enricher(object) && enricher_name === 'openWeather') {
				config.enrichers[0].config['api_key'] = object.enricher_config_api_key;
			}

			// Target
			if (object.target_type === 'alter source') {
				config.target[object.source_type] = {};
				config.target[object.source_type].path = object.source_path;
			} else {
				// ONLY TARGET FOR ALPHA
				config.target[object.source_type] = {};
				config.target[object.source_type].path = object.source_path;

				//config.target[object.target_type] = {};
				//config.target[object.target_type].path = object.target_path;
			}

			return config;
		}
	}, {
		key: '_has_custom_enricher',
		value: function _has_custom_enricher(answers) {
			return typeof answers.custom_enricher !== 'undefined';
		}
	}, {
		key: '_resolve_enricher_parent_path',
		value: function _resolve_enricher_parent_path(name) {
			var base_path = __dirname + '/../Enrichers/';
			if (name === 'openWeather') {
				return base_path;
			}
			return process.cwd();
		}
	}, {
		key: '_create_enricher',
		value: function _create_enricher(name) {
			name = (0, _camelcase2.default)(name);
			var enricher_template = '\n"use strict";\nvar Enricher = class Enricher {\n\n  constructor(rp, inputData, config) {\n    //npm request-promise is used for handling requests\n    //see: https://github.com/request/request-promise\n    this.rp = rp;\n    //loads inputData of the target file specified in the source object path in your config\n    this.inputData = inputData;\n    //loads config for this enrichment\n    this.config = config;\n  }\n\n  getConfig() {\n    return this.config;\n  }\n\n  getName() {\n    return \'Enricher\';\n  }\n\n  setData(inputData) {\n    this.inputData = inputData;\n  }\n\n  process(inputData) {\n\n    if (typeof inputData != \'undefined\' && inputData != null) {\n      this.inputData = inputData;\n    }\n\n    // Do stuff here\n\n\n    return Promise.resolve(this.inputData);    \n  }\n\n  // eof class\n};\n\nmodule.exports.Enricher = Enricher;\n\t\t';

			var new_enricher_source = enricher_template.replace(/Enricher/g, name);
			var enricher_path = process.cwd() + '/' + name + '.js';
			_fsExtra2.default.writeFileSync(enricher_path, new_enricher_source);
			return enricher_path;
		}
	}, {
		key: '_process_answers',
		value: function _process_answers(answers) {
			var _this = this;

			var config_object = this._transform_to_config_object(answers);
			var config_string = JSON.stringify(config_object, null, '\t');
			var config_save_path = process.cwd() + '/' + config_object.name.toLowerCase().replace(/\s/g, "_") + '_midas.json';

			if (this._has_custom_enricher(answers)) {
				this._create_enricher(answers.custom_enricher);
			}

			var config_save_state = _fsExtra2.default.outputJson(config_save_path, config_object, { spaces: '\t' });
			config_save_state.then(function () {
				console.log('');
				console.log('--------------------------------------------------------------');
				console.log('✅ New enrichment job successfully created '.green.bold);
				console.log('');
				console.log(' ⚠️ You can freely customize your enrichment job using your own enrichers and config'.cyan);
				console.log('Check out https://midas.science/guide'.cyan);
				console.log('');

				// Notify user that s/he has to add an enricher themself
				if (!_this._has_custom_enricher(answers)) {
					console.log('');
					console.log(' ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️');
					console.log('Currently you didn\'t specify any enricher for your process'.bold);
					console.log('Create or add one before start your enrichement process'.bold);
					console.log(' ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️');
					console.log('');
					console.log('');
				}

				console.log('Start enrichment process via '.green.bold);
				console.log('midas enrich -c "' + config_save_path + '"');
				console.log('--------------------------------------------------------------');
				console.log('');
			}).catch(function (err) {
				return console.log('Something went wrong');
			});
		}
	}, {
		key: 'start',
		value: function start() {
			var _this2 = this;

			var questions = _inquirer2.default.prompt([{
				type: 'input',
				name: 'name',
				message: "Chose a name for your enrichment process:"
			}, {
				type: 'list',
				name: 'source_type',
				message: 'What\'s your data source?',
				choices: ['CSV', 'JSON', 'XLSX'],
				filter: function filter(val) {
					return val.toLowerCase();
				}
			}, {
				when: function when(response) {
					return response.source_type === 'csv';
				},
				type: 'input',
				name: 'csv_delimiter',
				message: "What\'s the delimiter of your CSV file?"
			}, {
				when: function when(response) {
					return true;
				},
				type: 'input',
				name: 'source_path',
				message: "Where is your file located? (Enter the absolute path)",
				validate: function validate(value) {
					if (_fsExtra2.default.existsSync(value)) {
						return true;
					}
					return 'File not found :/';
				}
			}, {
				when: function when(response) {
					return true;
				},
				type: 'input',
				name: 'source_object_path',
				message: function message(response) {
					if (response.json_path_validate === 'n') {
						return 'Just try again';
					} else {
						return 'Define the JSON path of your input parameter';
					}
				},
				validate: function validate(value, response) {

					// Extract data schema
					var data_set = null;
					if (response.source_type === 'xlsx') {
						var wb = _xlsx2.default.readFile(response.source_path, { type: "file" });
						data_set = _xlsx2.default.utils.sheet_to_json(wb.Sheets[wb.SheetNames[0]], { raw: false, header: 1 });
						data_set = _ArrayUtils2.default.arrayObjectify(data_set);
					} else if (response.source_type === 'csv') {

						var options = {
							delimiter: response.csv_delimiter,
							quote: '"'
						};

						var _data = _fsExtra2.default.readFileSync(response.source_path, { encoding: 'utf8' });
						data_set = _csvjson2.default.toObject(_data, options);
					} else if (response.source_type === 'json') {
						data_set = (0, _readJsonSync2.default)(response.source_path);
					}

					var jp_source_data = null;

					try {
						jp_source_data = _jsonpath2.default.nodes(data_set, value);
					} catch (e) {
						return 'Please enter a valid json path';
					}

					var example_data_point = 'Not found';
					if (jp_source_data.length > 0 && typeof jp_source_data[0].value !== 'undefined') {
						example_data_point = jp_source_data[0].value;
					} else {
						return 'Please enter a valid json path';
					}

					var example_data_point_type = (typeof example_data_point === 'undefined' ? 'undefined' : _typeof(example_data_point)).bold;
					if (Array.isArray(example_data_point)) {
						example_data_point_type = 'array'.bold;
					}

					console.log('');
					console.log('');
					console.log('This will be the input for your enricher:');
					console.log('Example (first row / data point):');
					console.log('');
					console.log('Type ' + example_data_point_type);
					if ((typeof example_data_point === 'undefined' ? 'undefined' : _typeof(example_data_point)) === 'object') {
						console.log(_prettyjson2.default.render(example_data_point));
					} else {
						console.log(example_data_point.blue);
					}
					console.log('');
					console.log('');

					return true;
				}
			}, {
				type: 'list',
				name: 'enricher',
				message: 'Do you want to create a new enricher?',
				choices: ['Yes', 'No'],
				filter: function filter(val) {
					var return_value = val;
					if (val === 'Yes') {
						return_value = 'custom';
					}
					return (0, _camelcase2.default)(return_value);
				}
			}, {
				when: function when(response) {
					return response.enricher === 'custom';
				},
				type: 'input',
				name: 'custom_enricher',
				message: 'Specify a name for your new enricher',
				filter: function filter(val) {
					return val;
				}
			}, {
				when: function when(response) {
					return true;
				},
				type: 'input',
				name: 'target_property_name',
				message: "Name of the enriched property?"
			}, {
				when: function when(response) {
					return true;
				},
				type: 'list',
				name: 'rate_limit_yesno',
				message: "Do you need rate limiting?",
				choices: ['Yes', 'No'],
				filter: function filter(val) {
					return val.toLowerCase();
				}
			}, {
				when: function when(response) {
					return response.rate_limit_yesno === 'yes';
				},
				type: 'input',
				name: 'rate_limit',
				message: "Define your rate limit via NUMBER_OF_REQUESTS/TIME_UNIT (e.g. 100/s would mean midas will make a maximum of 100 requests per second)",
				filter: function filter(val) {
					return val.toLowerCase();
				},
				validate: function validate(value, response) {
					var split = value.split('/');

					if (split.length != 2) {
						return false;
					}

					if (['s', 'm', 'h'].indexOf(split[1]) <= -1) {
						return false;
					}

					if (/^\d+$/.test(split[0]) == false) {
						return false;
					}

					return true;
				}
			}]);

			questions.then(function (answers) {
				_this2._process_answers(answers);
			});
			// eof
		}
	}]);

	return Wizard;
}();

exports.default = Wizard;