@wbg-mde/r-factory
Version:
Metadata editor R integration module
126 lines (125 loc) • 6.46 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const path = require("path");
const fs = require("fs");
const mkdirp = require("mkdirp");
const R = require("@wbg-mde/r-script");
const streamEqual = require("stream-equal");
const config_1 = require("./config");
const import_config_1 = require("../modules/import/import-config");
const repository_1 = require("@wbg-mde/repository");
var _ = require('lodash');
class DestringTest {
constructor() {
this.inputDatasetDirectory = config_1.TestConfig.inputPath.dataset;
this.inputJsonDirectory = config_1.TestConfig.inputPath.json;
this.inputCSVDirectory = config_1.TestConfig.inputPath.csv;
this.inputFinalJsonDirectory = config_1.TestConfig.inputPath.finaljson;
this.outputDatasetDirectory = config_1.TestConfig.outputPath.dataset;
this.outputJsonDirectory = config_1.TestConfig.outputPath.json;
this.outputCSVDirectory = config_1.TestConfig.outputPath.csv;
this.libraryPath = '';
this.memoryLimit = 36000;
mkdirp.sync(this.inputDatasetDirectory);
mkdirp.sync(this.inputJsonDirectory);
mkdirp.sync(this.inputCSVDirectory);
mkdirp.sync(this.inputFinalJsonDirectory);
mkdirp.sync(this.outputDatasetDirectory);
mkdirp.sync(this.outputJsonDirectory);
mkdirp.sync(this.outputCSVDirectory);
}
execute(variableNames, callback) {
try {
fs.readdir(this.inputDatasetDirectory, (err, files) => {
if (files) {
this.executeFiles(files, 0, variableNames, (result) => {
callback(result);
});
}
});
}
catch (e) {
console.log('execute error >> ' + e);
callback({ result: 'error' });
}
}
executeFiles(files, index, variableNames, callback) {
try {
let file = files[index];
let fileId = "F" + (index + 1);
let datafile = path.join(this.inputDatasetDirectory, file);
let inputJsonFile = path.join(this.inputJsonDirectory, fileId + '.json');
let inputCSVFile = path.join(this.inputCSVDirectory, fileId + '.csv');
let finalJsonFile = path.join(this.inputFinalJsonDirectory, fileId + '.json');
let outputCSVFile = path.join(this.outputCSVDirectory, fileId + '.csv');
let workingDirectory = path.dirname(config_1.TestConfig.scriptPath.import);
R(config_1.TestConfig.scriptPath.import).data(datafile, path.extname(file).slice(1), fileId, inputJsonFile, import_config_1.ImportConfig.frequencyCalcLimit, this.libraryPath, this.memoryLimit, workingDirectory).call((err, rowCount) => {
if (err) {
callback({ result: 'error' });
console.log('import file R script execution error - file( ' + file + ' ):' + err.toString());
}
else {
R(config_1.TestConfig.scriptPath.exportCSV).data(datafile, path.extname(file).slice(1), inputCSVFile, this.libraryPath, this.memoryLimit).call((err, resp) => {
if (err) {
callback({ result: 'error' });
console.log('write csv error - file( ' + file + ' ):' + err.toString());
}
else {
this.formatJson(inputJsonFile, finalJsonFile);
let workingDirectory = path.dirname(config_1.TestConfig.scriptPath.resequence);
R(config_1.TestConfig.scriptPath.destring)
.data(inputCSVFile, finalJsonFile, outputCSVFile, variableNames, this.libraryPath, workingDirectory)
.call((err, resp) => {
if (err) {
callback({ result: 'error' });
console.log('Destring variable error - file( ' + file + ' ):' + err.toString());
}
else {
console.log(resp);
var readStream1 = fs.createReadStream(inputCSVFile);
var readStream2 = fs.createReadStream(outputCSVFile);
streamEqual(readStream1, readStream2, (err, equal) => {
console.log('completed >> ' + file);
if (equal) {
console.log('input & output files are same');
}
else {
console.log('input & output files are not same');
}
index += 1;
if (index < files.length) {
this.executeFiles(files, index, variableNames, callback);
}
else {
callback({ result: 'ok' });
}
});
}
});
}
});
}
});
}
catch (e) {
console.log('execute files error >> ' + e);
callback({ result: 'error' });
}
}
formatJson(filePath, outPath) {
let outstream = repository_1.Repository_Utility
.readFile(filePath)
.toString();
let jsonData = JSON.parse(outstream);
let formattedDataset = _.map(jsonData, (data) => {
return {
name: data.name,
internalName: data.name,
type: (data.varFormat) ? data.varFormat.type : ["numeric"],
dataType: data.varType
};
});
repository_1.Repository_Utility.writeFile(outPath, JSON.stringify(formattedDataset));
}
}
exports.DestringTest = DestringTest;