@wbg-mde/r-factory
Version:
Metadata editor R integration module
154 lines (153 loc) • 8.58 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const path = require("path");
const fs = require("fs");
const mkdirp = require("mkdirp");
const R = require("@wbg-mde/r-script");
const streamEqual = require("stream-equal");
const config_1 = require("./config");
const test_utils_1 = require("./test.utils");
const import_config_1 = require("../modules/import/import-config");
const repository_1 = require("@wbg-mde/repository");
var _ = require('lodash');
class ImportTest {
constructor() {
this.libraryPath = '';
this.memoryLimit = 36000;
mkdirp.sync(config_1.TestConfig.inputPath.dataset);
mkdirp.sync(config_1.TestConfig.inputPath.json);
mkdirp.sync(config_1.TestConfig.inputPath.csv);
mkdirp.sync(config_1.TestConfig.inputPath.finaljson);
mkdirp.sync(config_1.TestConfig.outputPath.dataset);
mkdirp.sync(config_1.TestConfig.outputPath.csv);
mkdirp.sync(config_1.TestConfig.outputPath.json);
}
execute(callback) {
try {
fs.readdir(config_1.TestConfig.inputPath.dataset, (err, files) => {
if (files) {
this.executeFiles(files, 0, (result) => {
callback(result);
});
}
});
}
catch (e) {
console.log('execute error >> ' + e);
callback({ result: 'error' });
}
}
executeFiles(files, index, callback) {
try {
let file = files[index];
let fileId = "F" + (index + 1);
let inputdataset = path.join(config_1.TestConfig.inputPath.dataset, file);
let inputJson = path.join(config_1.TestConfig.inputPath.json, fileId + '.json');
let inputcsv = path.join(config_1.TestConfig.inputPath.csv, fileId + '.csv');
let finaljson = path.join(config_1.TestConfig.inputPath.finaljson, fileId + '.json');
let outputdataset = path.join(config_1.TestConfig.outputPath.dataset, file);
let workingDirectory = path.dirname(config_1.TestConfig.scriptPath.export);
let outputJson = path.join(config_1.TestConfig.outputPath.json, fileId + '.json');
let outputcsv = path.join(config_1.TestConfig.outputPath.csv, fileId + '.csv');
const varProps = ["name", "internalName", "labl", "val", "dcml", "width", "type", "intrvl", "dataType"];
R(config_1.TestConfig.scriptPath.import).data(inputdataset, path.extname(file).slice(1), fileId, inputJson, import_config_1.ImportConfig.frequencyCalcLimit, this.libraryPath, this.memoryLimit, workingDirectory).call((err, rowCount) => {
if (err) {
callback({ result: 'error' });
console.log('import file R script execution error - file( ' + file + ' ):' + err.toString());
}
else {
R(config_1.TestConfig.scriptPath.exportCSV).data(inputdataset, path.extname(file).slice(1), inputcsv, this.libraryPath, this.memoryLimit).call((err, resp) => {
if (err) {
callback({ result: 'error' });
console.log('write csv error - file( ' + file + ' ):' + err.toString());
}
else {
test_utils_1.TestUtility.formatVariable(inputJson, finaljson, varProps);
this.formatJson(inputJson, finaljson);
R(config_1.TestConfig.scriptPath.export)
.data(inputcsv, finaljson, outputdataset, path.extname(file).slice(1), 8, this.libraryPath, this.memoryLimit, workingDirectory)
.call((err, resp) => {
if (err) {
callback({ result: 'error' });
console.log('export dta error - file( ' + file + ' ):' + err.toString());
}
else {
R(config_1.TestConfig.scriptPath.import).data(outputdataset, path.extname(file).slice(1), fileId, outputJson, import_config_1.ImportConfig.frequencyCalcLimit, this.libraryPath, this.memoryLimit, workingDirectory).call((err, rowCount) => {
if (err) {
callback({ result: 'error' });
console.log('import out dta error: ' + err.toString());
}
else {
R(config_1.TestConfig.scriptPath.exportCSV).data(outputdataset, path.extname(file).slice(1), outputcsv, this.libraryPath, this.memoryLimit).call((err, resp) => {
if (err) {
callback({ result: 'error' });
console.log('write csv out error: ' + err.toString());
}
else {
var readStream1 = fs.createReadStream(inputcsv);
var readStream2 = fs.createReadStream(outputcsv);
streamEqual(readStream1, readStream2, (err, equal) => {
console.log('completed >> ' + file);
if (equal) {
console.log('input & output files are same');
}
else {
console.log('input & output files are not same');
}
index += 1;
if (index < files.length) {
this.executeFiles(files, index, callback);
}
else {
callback({ result: 'ok' });
}
});
}
});
}
});
}
});
}
});
}
});
}
catch (e) {
console.log('execute files error >> ' + e);
callback({ result: 'error' });
}
}
formatJson(filePath, outPath) {
let outstream = repository_1.Repository_Utility
.readFile(filePath)
.toString();
let jsonData = JSON.parse(outstream);
let formattedDataset = _.map(jsonData, (data) => {
let catgry = new Array();
if (data.catgry) {
catgry = _.map(data.catgry, (category) => {
if (category && category.catValu && category.catValu !== 'NA') {
return {
catValu: category.catValu,
labl: category.labl
};
}
});
}
return {
name: data.name,
internalName: data.name,
labl: data.labl,
val: catgry,
dcml: data.dcml,
width: (data.location) ? data.location.width : [0],
type: (data.varFormat) ? data.varFormat.type : ["numeric"],
intrvl: data.intrvl,
dataType: data.varType
};
});
repository_1.Repository_Utility.writeFile(outPath, JSON.stringify(formattedDataset));
}
}
exports.ImportTest = ImportTest;