datakit
Version:
Simple JavaScript toolkit for data transform across JSON, CSV and YAML.
751 lines • 27.3 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (g && (g = 0, op[0] && (_ = 0)), _) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.writeYamlSync = exports.writeYaml = exports.writeJsonSync = exports.writeJson = exports.writeCsvSync = exports.writeCsv = exports.writeFile = exports.toYaml = exports.toCsv = exports.toJson = exports.readYamlSync = exports.readJsonSync = exports.readCsvSync = exports.readYaml = exports.readJson = exports.readCsv = exports.readFile = exports.fromYaml = exports.fromCsv = exports.fromJson = exports.fromObject = void 0;
var utils_1 = require("./lib/utils");
var YAML = require("yaml");
// @ts-ignore
var papaparse_1 = __importDefault(require("papaparse"));
;
/**
* Convert a regular JavaScript object to 'tablular data'.
* Each element of the matches a field from the object.
*
* @param obj - The JavaScript object to convert to a dataframe.
*
* @returns Returns an array with an element describing each field.
*/
function fromObject(obj) {
return Object.keys(obj)
.map(function (fieldName) { return ({
field: fieldName,
value: obj[fieldName],
}); });
}
exports.fromObject = fromObject;
;
/**
* Deserialize JSON text to a JavaScript array.
*
* @param jsonTextString The JSON text to deserialize.
*
* @returns Returns an array of JavaScript objects that were deserialized from the JSON text.
*/
function fromJson(jsonTextString, config) {
if (!(0, utils_1.isString)(jsonTextString)) {
throw new Error("Expected 'jsonTextString' parameter to 'datakit.fromJson' to be a string containing data encoded in the JSON format.");
}
if (config) {
if (!(0, utils_1.isObject)(config)) {
throw new Error("Expected 'config' parameter to 'datakit.fromJson' to be an object with configuration options for JSON deserialization.");
}
}
var records = JSON.parse(jsonTextString);
if (config && config.parser) {
for (var _i = 0, _a = Object.keys(config.parser); _i < _a.length; _i++) {
var columnName = _a[_i];
var parsingFunction = config.parser[columnName];
for (var _b = 0, records_1 = records; _b < records_1.length; _b++) {
var record = records_1[_b];
record[columnName] = parsingFunction(record[columnName]);
}
}
}
return records;
}
exports.fromJson = fromJson;
;
/**
* Deserialize CSV text to a JavaScript array.
* Each element of the array contains fields that match the columns from the CSV data.
*
* @param csvTextString The CSV text to deserialize.
* @param [config] Optional configuration options for parsing the CSV data.
*
* @returns Returns an array of JavaScript objects that were deserialized from the CSV text.
*/
function fromCsv(csvTextString, config) {
if (!(0, utils_1.isString)(csvTextString)) {
throw new Error("Expected 'csvTextString' parameter to 'datakit.fromCsv' to be a string containing data encoded in the CSV format.");
}
if (!config) {
config = {};
}
else {
if (!(0, utils_1.isObject)(config)) {
throw new Error("Expected 'config' parameter to 'datakit.fromCsv' to be an object with configuration options for CSV deserialization.");
}
config = Object.assign({}, config); // Copy config so we can default values as necessary.
}
if (config.columnNames) {
if (!(0, utils_1.isArray)(config.columnNames)) {
throw new Error("Expected 'columnNames' field of 'config' parameter to 'datakit.fromCsv' to be an array of strings that specifies column names to read from the CSV data.");
}
for (var _i = 0, _a = config.columnNames; _i < _a.length; _i++) {
var columnName = _a[_i];
if (!(0, utils_1.isString)(columnName)) {
throw new Error("Expected 'columnNames' field of 'config' parameter to 'datakit.fromCsv' to be an array of strings that specify column names.");
}
}
}
if (config.dynamicTyping === undefined) {
config.dynamicTyping = true;
}
if (config.skipEmptyLines === undefined) {
config.skipEmptyLines = true;
}
var parsed = papaparse_1.default.parse(csvTextString, config);
var rows = parsed.data;
if (rows.length === 0) {
return [];
}
rows = rows.map(function (row) { return row.map(function (cell) { return (0, utils_1.isString)(cell) ? cell.trim() : cell; }); });
var columnNames;
if (config && config.columnNames) {
columnNames = config.columnNames;
}
else {
columnNames = rows.shift().filter(function (columnName) { return columnName && columnName.length !== undefined && columnName.length > 0; });
}
var records = rows.map(function (row) {
var record = {};
for (var i = 0; i < columnNames.length; ++i) {
record[columnNames[i]] = row[i];
}
return record;
});
if (config.parser) {
for (var _b = 0, _c = Object.keys(config.parser); _b < _c.length; _b++) {
var columnName = _c[_b];
var parsingFunction = config.parser[columnName];
for (var _d = 0, records_2 = records; _d < records_2.length; _d++) {
var record = records_2[_d];
record[columnName] = parsingFunction(record[columnName]);
}
}
}
return records;
}
exports.fromCsv = fromCsv;
/**
* Deserialize YAML text to a JavaScript array.
*
* @param yamlTextString The YAML text to deserialize.
*
* @returns Returns an array of JavaScript objects that were deserialized from the YAML text.
*/
function fromYaml(yamlTextString) {
if (!(0, utils_1.isString)(yamlTextString)) {
throw new Error("Expected 'yamlTextString' parameter to 'datakit.fromYaml' to be a string containing data encoded in the YAML format.");
}
return YAML.parse(yamlTextString);
}
exports.fromYaml = fromYaml;
/**
* Like fs.readFile but returns a promise for the file data.
*
* @param filePath Path to the file to be loaded.
*
* @returns A promise that is resolved with the contents of the file.
*
* @example
* <pre>
*
* const data = await datakit.readFile("some-data-file.txt");
* console.log(data);
* </pre>
*
*/
function readFile(filePath) {
if (!filePath || !(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.readFile' function to be a string.");
}
return new Promise(function (resolve, reject) {
var fs = require("fs");
fs.readFile(filePath, 'utf8', function (err, fileData) {
if (err) {
reject(err);
return;
}
resolve(fileData);
});
});
}
exports.readFile = readFile;
/**
* Asynchronously deserialize a CSV file to a JavaScript array.
* Each element of the array contains fields that match the columns from the CSV file.
*
* @param filePath Path to the file to be loaded.
* @param [config] Optional configuration file for parsing.
*
* @returns Returns a promise for the loaded data.
*
* @example
* <pre>
*
* const data = await datakit.readCsv("my-data-file.csv");
* console.log(data);
* </pre>
*
* const config = {
* dynamicTyping: true,
* // ... other options ...
* };
* const data = await datakit.readCsv("my-data-file.csv", config);
* console.log(data);
* </pre>
*/
function readCsv(filePath, config) {
return __awaiter(this, void 0, void 0, function () {
var fileData;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!filePath || !(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.readCsv' function to be a string.");
}
if (config) {
if (!(0, utils_1.isObject)(config)) {
throw new Error("Expected optional 'config' parameter to 'datakit.readCsv' to be an object with configuration options for CSV deserialization.");
}
}
return [4 /*yield*/, readFile(filePath)];
case 1:
fileData = _a.sent();
return [2 /*return*/, fromCsv(fileData, config)];
}
});
});
}
exports.readCsv = readCsv;
/**
* Asynchronously deserialize a JSON file to a JavaScript array.
*
* @param filePath Path to the file to be loaded.
*
* @returns Returns a promise for the loaded data.
*
* @example
* <pre>
*
* const data = await datakit.readJson("my-data-file.json");
* console.log(data);
* </pre>
*/
function readJson(filePath) {
return __awaiter(this, void 0, void 0, function () {
var fileData;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!filePath || !(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.readJson' function to be a string.");
}
return [4 /*yield*/, readFile(filePath)];
case 1:
fileData = _a.sent();
return [2 /*return*/, fromJson(fileData)];
}
});
});
}
exports.readJson = readJson;
/**
* Asynchronously deserialize a YAML file to a JavaScript array.
*
* @param filePath Path to the file to be loaded.
*
* @returns Returns a promise for the loaded data.
*
* @example
* <pre>
*
* const data = await datakit.readYaml("my-data-file.yaml");
* console.log(data);
* </pre>
*/
function readYaml(filePath) {
return __awaiter(this, void 0, void 0, function () {
var fileData;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!filePath || !(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.readYaml' function to be a string.");
}
return [4 /*yield*/, readFile(filePath)];
case 1:
fileData = _a.sent();
return [2 /*return*/, fromYaml(fileData)];
}
});
});
}
exports.readYaml = readYaml;
/**
* Synchronously deserialize a CSV file to a JavaScript array.
* Each element of the array contains fields that match the columns from the CSV file.
*
* @param filePath Path to the file to be loaded.
* @param [config] Optional configuration file for parsing.
*
* @returns Returns the loaded data.
*
* @example
* <pre>
*
* const data = datakit.readCsvSync("my-data-file.csv");
* console.log(data);
* </pre>
*
* @example
* <pre>
*
* const config = {
* dynamicTyping: true,
* // ... other options ...
* };
* const data = datakit.readCsvSync("my-data-file.csv", config);
* console.log(data);
* </pre>
*/
function readCsvSync(filePath, config) {
if (!filePath || !(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.readCsvSync' function to be a string.");
}
if (config) {
if (!(0, utils_1.isObject)(config)) {
throw new Error("Expected optional 'config' parameter to 'datakit.readCsvSync' to be an object with configuration options for CSV parsing.");
}
}
var fs = require("fs");
return fromCsv(fs.readFileSync(filePath, 'utf8'), config);
}
exports.readCsvSync = readCsvSync;
/**
* Synchronously deserialize a JSON file to a JavaScript array.
*
* @param filePath Path to the file to be loaded.
*
* @returns Returns the loaded data.
*
* @example
* <pre>
*
* const data = datakit.readJsonSync("my-data-file.json");
* console.log(data);
* </pre>
*/
function readJsonSync(filePath) {
if (!filePath || !(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.readJsonSync' function to be a string.");
}
var fs = require("fs");
return fromJson(fs.readFileSync(filePath, 'utf8'));
}
exports.readJsonSync = readJsonSync;
/**
* Synchronously deserialize a YAML file to a JavaScript array.
*
* @param filePath Path to the file to be loaded.
*
* @returns Returns the loaded data.
*
* @example
* <pre>
*
* const data = datakit.readYamlSync("my-data-file.yaml");
* console.log(data);
* </pre>
*/
function readYamlSync(filePath) {
if (!filePath || !(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.readYamlSync' function to be a string.");
}
var fs = require("fs");
return fromYaml(fs.readFileSync(filePath, 'utf8'));
}
exports.readYamlSync = readYamlSync;
;
/**
* Serialize a JavaScript array to the JSON data format.
*
* @param input The data to be serialized.
*
* @return Returns a string in the JSON data format that represents the data.
*
* @example
* <pre>
*
* const data = ... JavaScript data ...;
* const jsonData = datakit.toJson(data);
* console.log(jsonData);
* </pre>
*/
function toJson(input, config) {
if (config) {
if (!(0, utils_1.isObject)(config)) {
throw new Error("Expected optional 'config' parameter to 'datakit.toJson' to be an object with configuration options for JSON serialization.");
}
if (config.formatter) {
// Make a copy of the input so that we can change the formatting.
input = input.map(function (record) { return Object.assign({}, record); });
for (var _i = 0, _a = Object.keys(config.formatter); _i < _a.length; _i++) {
var columnName = _a[_i];
var formatterFunction = config.formatter[columnName];
for (var _b = 0, input_1 = input; _b < input_1.length; _b++) {
var record = input_1[_b];
record[columnName] = formatterFunction(record[columnName]);
}
}
}
}
return JSON.stringify(input, null, 4);
}
exports.toJson = toJson;
;
/**
* Serialize a JavaScript array to the CSV data format.
* Columns in the CSV file match fields from the objects in the array.
*
* @param input The data to be serialized.
*
* @return Returns a string in the CSV data format that represents the data.
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* const csvData = datakit.toCsv(data);
* console.log(csvData);
* </pre>
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* const config = {
* header: false,
* // ... other options ...
* };
* const csvData = datakit.toCsv(data, config);
* console.log(csvData);
* </pre>
*/
function toCsv(input, config) {
if (!input || !(0, utils_1.isArray)(input)) {
throw new Error("Expected 'input' parameter to 'datakit.toCsv' to be a JavaScript array.");
}
if (config) {
if (!(0, utils_1.isObject)(config)) {
throw new Error("Expected optional 'config' parameter to 'datakit.toCsv' to be an object with configuration options for CSV serialization.");
}
if (config.formatter) {
// Make a copy of the input so that we can change the formatting.
input = input.map(function (record) { return Object.assign({}, record); });
for (var _i = 0, _a = Object.keys(config.formatter); _i < _a.length; _i++) {
var columnName = _a[_i];
var formatterFunction = config.formatter[columnName];
for (var _b = 0, input_2 = input; _b < input_2.length; _b++) {
var record = input_2[_b];
record[columnName] = formatterFunction(record[columnName]);
}
}
}
}
var columnNames = (config && config.columnNames) || (input && input.length > 0 && Object.keys(input[0])) || [];
var headerLine = config === undefined || config.header === undefined || config.header
? [columnNames]
: [];
var data = input.map(function (obj) { return columnNames.map(function (columnName) { return obj[columnName]; }); });
var rows = headerLine.concat(data);
return papaparse_1.default.unparse(rows, config);
}
exports.toCsv = toCsv;
/**
* Serialize a JavaScript array to the YAML data format.
*
* @param input The data to be serialized.
*
* @return Returns a string in the YAML data format that represents the data.
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* const yamlData = datakit.toYaml(data);
* console.log(yamlData);
* </pre>
*/
function toYaml(input) {
return YAML.stringify(input);
}
exports.toYaml = toYaml;
/**
* Like fs.writeFile but returns a promise for completion of the asynchronous file write.
*
* @param filePath Path to the file to be written.
* @param data Data to be written the to file.
*
* @returns A promise that is resolved when the file has been written.
*
* @example
* <pre>
*
* const data = "... JavaScript string with text data ...";
* await datakit.writeFile("some-data-file.txt", data);
* </pre>
*
*/
function writeFile(filePath, data) {
if (!filePath || !(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.writeFile' function to be a string.");
}
return new Promise(function (resolve, reject) {
var fs = require("fs");
fs.writeFile(filePath, data, function (err) {
if (err) {
reject(err);
return;
}
resolve();
});
});
}
exports.writeFile = writeFile;
/**
* Asynchronously serialize a JavaScript array to a CSV file.
* The fields in the objects of the array become the columns in the CSV file.
*
* @param filePath Path to the file to be written.
* @param [config] Optional configuration file for parsing.
*
* @return Returns a promise that resolves when the file has been written.
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* await datakit.writeCsv("my-data-file.csv", data);
* </pre>
*
* @example
* <pre>
* const config = {
* // ... Options for serialization ...
* };
* const data = [ ... JavaScript array of data ... ];
* await datakit.writeCsv("my-data-file.csv", config);
* </pre>
*/
function writeCsv(filePath, input, config) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.writeCsv' to be a string that specifies the path of the file to write to the local file system.");
}
if (!input || !(0, utils_1.isArray)(input)) {
throw new Error("Expected 'input' parameter to 'datakit.writeCsv' to be a JavaScript array.");
}
if (config) {
if (!(0, utils_1.isObject)(config)) {
throw new Error("Expected optional 'config' parameter to 'datakit.writeCsv' to be an object with configuration options for CSV serialization.");
}
}
return [4 /*yield*/, writeFile(filePath, toCsv(input, config))];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
}
exports.writeCsv = writeCsv;
/**
* Synchronously serialize a JavaScript array to a CSV file.
* The fields in the objects of the array become the columns in the CSV file.
*
* @param filePath Path to the file to be written.
* @param [config] Optional configuration file for parsing.
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* datakit.writeCsvSync("my-data-file.csv", data);
* </pre>
*
* @example
* <pre>
*
* const config = {
* // ... Options for serialization ...
* };
* const data = [ ... JavaScript array of data ... ];
* datakit.writeCsvSync("my-data-file.csv", config);
* </pre>
*/
function writeCsvSync(filePath, input, config) {
if (!(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.writeCsvSync' to be a string that specifies the path of the file to write to the local file system.");
}
if (!input || !(0, utils_1.isArray)(input)) {
throw new Error("Expected 'input' parameter to 'datakit.writeCsvSync' to be a JavaScript array.");
}
if (config) {
if (!(0, utils_1.isObject)(config)) {
throw new Error("Expected optional 'config' parameter to 'datakit.writeCsvSync' to be an object with configuration options for CSV serialization.");
}
}
var fs = require("fs");
fs.writeFileSync(filePath, toCsv(input, config));
}
exports.writeCsvSync = writeCsvSync;
/**
* Asynchronously serialize a JavaScript array to a JSON file.
*
* @param filePath Path to the file to be written.
*
* @return Returns a promise that resolves when the file has been written.
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* await datakit.writeJson("my-data-file.json", data);
* </pre>
*/
function writeJson(filePath, input) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.writeJson' to be a string that specifies the path of the file to write to the local file system.");
}
return [4 /*yield*/, writeFile(filePath, toJson(input))];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
}
exports.writeJson = writeJson;
/**
* Synchronously serialize a JavaScript array to a JSON file.
*
* @param filePath Path to the file to be written.
*
* @return Returns a promise that resolves when the file has been written.
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* datakit.writeJsonSync("my-data-file.json", data);
* </pre>
*/
function writeJsonSync(filePath, input) {
if (!(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.writeJsonSync' to be a string that specifies the path of the file to write to the local file system.");
}
var fs = require("fs");
fs.writeFileSync(filePath, toJson(input));
}
exports.writeJsonSync = writeJsonSync;
/**
* Asynchronously serialize a JavaScript array to a YAML file.
*
* @param filePath Path to the file to be written.
*
* @return Returns a promise that resolves when the file has been written.
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* await datakit.writeYaml("my-data-file.yaml", data);
* </pre>
*/
function writeYaml(filePath, input) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.writeYaml' to be a string that specifies the path of the file to write to the local file system.");
}
return [4 /*yield*/, writeFile(filePath, toYaml(input))];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
}
exports.writeYaml = writeYaml;
/**
* Synchronously serialize a JavaScript array to a Yaml file.
*
* @param filePath Path to the file to be written.
*
* @return Returns a promise that resolves when the file has been written.
*
* @example
* <pre>
*
* const data = [ ... JavaScript array of data ... ];
* datakit.writeYamlSync("my-data-file.yaml", data);
* </pre>
*/
function writeYamlSync(filePath, input) {
if (!(0, utils_1.isString)(filePath)) {
throw new Error("Expected 'filePath' parameter to 'datakit.writeYamlSync' to be a string that specifies the path of the file to write to the local file system.");
}
var fs = require("fs");
fs.writeFileSync(filePath, toYaml(input));
}
exports.writeYamlSync = writeYamlSync;
//# sourceMappingURL=index.js.map