@codingtools/cdt
Version:
CLI for Developers
154 lines • 7.24 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
const command_1 = require("@oclif/command");
const avsc_1 = tslib_1.__importDefault(require("avsc"));
const chalk_1 = tslib_1.__importDefault(require("chalk"));
const fs_1 = tslib_1.__importDefault(require("fs")); // includes all from avro-js and some more
const json_2_csv_1 = tslib_1.__importDefault(require("json-2-csv"));
const logger_1 = tslib_1.__importDefault(require("../utilities/logger"));
const utilities_1 = tslib_1.__importDefault(require("../utilities/utilities"));
class Avro extends command_1.Command {
/*
* input,output, and operation are all must
* */
async run() {
const { args, flags } = this.parse(Avro);
this.checkParameters(flags, args);
this.executeCommand(flags, args);
}
// to check required parameters passed or not
checkParameters(flags, args) {
if (!flags.file)
logger_1.default.error(this, 'Input file is not provided');
if (flags.command) // if -c flag have value, then override
args.command = flags.command;
if (!args.command)
logger_1.default.error(this, 'Command is empty or not provided, supported:' + Avro.SupportedCommands);
else // if exists then make Lower Case
args.command = args.command.toLowerCase();
// output is not mendatory for 'get_schema' command
if (args.command !== Avro.GET_SCHEMA && !flags.output)
logger_1.default.error(this, 'Output file is not provided');
}
executeCommand(flags, args) {
switch (args.command) {
case Avro.GET_SCHEMA:
return this.getSchema(flags, args);
case Avro.TO_JSON:
return this.toJson(flags, args);
case Avro.TO_AVRO:
return this.toAvro(flags, args);
case Avro.TO_CSV:
return this.toCsv(flags, args);
default:
logger_1.default.error(this, 'Unsupported Command, supported: ' + Avro.SupportedCommands);
}
}
// tslint:disable-next-line:no-unused
getSchema(flags, args) {
avsc_1.default.createFileDecoder(flags.file)
.on('metadata', function (type) {
let output = type.schema();
let schemaStr = JSON.stringify(output);
if (flags.output) {
// @ts-ignore
utilities_1.default.writeStringToFile(this, flags.output, schemaStr);
}
else {
// @ts-ignore
logger_1.default.success(this, `${chalk_1.default.yellow('Avro Schema')}\n${JSON.stringify(output, null, ' ')}`);
}
});
}
// tslint:disable-next-line:no-unused
toJson(flags, args) {
logger_1.default.progressStart(this, 'Converting Avro To Json');
// setTimeout(() => {
logger_1.default.progressStop(this, ' Converting Avro To Json');
utilities_1.default.truncateFile(this, flags.output);
avsc_1.default.createFileDecoder(flags.file)
.on('data', function (recordStr) {
// @ts-ignore
utilities_1.default.appendStringToFile(this, flags.output, JSON.stringify(recordStr));
});
logger_1.default.success(this, `${chalk_1.default.blue('Json')} written to file: ${chalk_1.default.green(flags.output)}`); // this will output error and exit command
// }, 1000)
}
// tslint:disable-next-line:no-unused
toCsv(flags, args) {
logger_1.default.progressStart(this, 'Converting Avro To Csv');
// setTimeout(() => {
logger_1.default.progressStop(this, ' Converting Avro To Csv');
utilities_1.default.truncateFile(this, flags.output);
let prependHeader = true; // only write on the first line
avsc_1.default.createFileDecoder(flags.file)
.on('data', function (recordStr) {
// @ts-ignore
let json = JSON.parse(JSON.stringify(recordStr));
json_2_csv_1.default.json2csv(json, (err, csv) => {
if (csv) {
// @ts-ignore
utilities_1.default.appendStringToFile(this, flags.output, csv + '\n');
}
if (err) {
// @ts-ignore
logger_1.default.error(this, err.toString());
}
}, { prependHeader });
prependHeader = false;
});
logger_1.default.success(this, `${chalk_1.default.blue('Csv')} written to file: ${chalk_1.default.green(flags.output)}`); // this will output error and exit command
// }, 300)
}
toAvro(flags, args) {
if (!flags.schemaType)
logger_1.default.error(this, 'Schema file is not provided');
logger_1.default.progressStart(this, 'Generating Avro');
// setTimeout(() => {
logger_1.default.progressStop(this, ' Generating Avro');
let schema = avsc_1.default.parse(flags.schemaType);
let avroEncoder = new avsc_1.default.streams.BlockEncoder(schema);
avroEncoder.pipe(fs_1.default.createWriteStream(flags.output));
// We write the records to the block encoder, which will take care of serializing them
// into an object container file.
let inputString = utilities_1.default.getInputString(this, flags, args);
let jsonStr = this.convertAvroJsonToValidJson(inputString);
let jsonObjects = JSON.parse(jsonStr);
jsonObjects.forEach(function (data) {
if (schema.isValid(data)) {
avroEncoder.write(data);
}
else {
// @ts-ignore
logger_1.default.warn(this, `${chalk_1.default.yellow('[SKIPPING RECORD]')} schema is invalid: ${chalk_1.default.yellowBright(JSON.stringify(data))}`);
}
});
logger_1.default.success(this, `${chalk_1.default.blue('Avro')} written to file: ${chalk_1.default.green(flags.output)}`); // this will output error and exit command
avroEncoder.end();
// }, 300)
}
convertAvroJsonToValidJson(json) {
let jsonStr = '[' + json + ']';
jsonStr = jsonStr.replace(/[\s\n]+/mg, '');
jsonStr = jsonStr.replace(/\}\{/mg, '},{');
return jsonStr;
}
}
exports.default = Avro;
Avro.description = 'Avro Utility command';
Avro.GET_SCHEMA = 'get_schema';
Avro.TO_JSON = 'to_json';
Avro.TO_AVRO = 'to_avro';
Avro.TO_CSV = 'to_csv';
// do not change order otherwise we need to change order in getCommand() also
Avro.SupportedCommands = [Avro.GET_SCHEMA, Avro.TO_JSON, Avro.TO_AVRO, Avro.TO_CSV];
Avro.flags = {
help: command_1.flags.help({ char: 'h' }),
command: command_1.flags.string({ char: 'c', description: `commands supported: ${Avro.SupportedCommands}` }),
file: command_1.flags.string({ char: 'f', description: 'input file path' }),
output: command_1.flags.string({ char: 'o', description: 'output file path' }),
schemaType: command_1.flags.string({ char: 't', description: 'schema type file path' }),
};
Avro.args = [{ name: 'command' }]; // operation type
//# sourceMappingURL=avro.js.map