datainout
Version:
Import and reports data
697 lines (685 loc) • 25.6 kB
JavaScript
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __defProps = Object.defineProperties;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getOwnPropSymbols = Object.getOwnPropertySymbols;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __propIsEnum = Object.prototype.propertyIsEnumerable;
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __spreadValues = (a, b) => {
for (var prop in b || (b = {}))
if (__hasOwnProp.call(b, prop))
__defNormalProp(a, prop, b[prop]);
if (__getOwnPropSymbols)
for (var prop of __getOwnPropSymbols(b)) {
if (__propIsEnum.call(b, prop))
__defNormalProp(a, prop, b[prop]);
}
return a;
};
var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var __async = (__this, __arguments, generator) => {
return new Promise((resolve, reject) => {
var fulfilled = (value) => {
try {
step(generator.next(value));
} catch (e) {
reject(e);
}
};
var rejected = (value) => {
try {
step(generator.throw(value));
} catch (e) {
reject(e);
}
};
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
step((generator = generator.apply(__this, __arguments)).next());
});
};
// src/importers/readers/BaseReader.ts
var BaseReader_exports = {};
__export(BaseReader_exports, {
BaseReader: () => BaseReader
});
module.exports = __toCommonJS(BaseReader_exports);
// src/helpers/excel.helper.ts
var exceljs = __toESM(require("exceljs"), 1);
var SYNTAX = {
VARIABLE_TABLE_SYNTAX: "$$",
VARIABLE_SYNTAX: "$",
INDEX_COLUMN_TABLE_SYNTAX: "$$**",
END_TABLE_SYNYAX: "$$br"
};
var DEFAULT_BEGIN_TABLE = -1;
var DEFAULT_END_TABLE = -1;
var DEFAULT_COLUMN_INDEX = 1;
var ReaderExceljsHelper = class _ReaderExceljsHelper {
constructor(opts) {
this.isSampleExcel = true;
this.isStop = false;
var _a;
this.onCell = opts == null ? void 0 : opts.onCell;
this.onRow = opts == null ? void 0 : opts.onRow;
this.onSheet = opts == null ? void 0 : opts.onSheet;
this.isSampleExcel = (_a = opts == null ? void 0 : opts.isSampleExcel) != null ? _a : true;
this.templateManager = opts.templateManager;
}
load(arg) {
return __async(this, null, function* () {
const workBook = new exceljs.Workbook();
if (arg instanceof Buffer) yield workBook.xlsx.load(arg);
else yield workBook.xlsx.readFile(arg);
for (let i = 0; !this.isStop && i < workBook.worksheets.length; i++) {
const workSheet = workBook.getWorksheet(i + 1);
if (workSheet) yield this.eachSheet(workSheet, i + 1);
}
});
}
eachSheet(sheet, sheetIndex) {
return __async(this, null, function* () {
var _a, _b, _c, _d;
this.templateManager.SheetIndex = sheetIndex - 1;
const sheetDesc = this.templateManager.SheetTemplate;
const trackingRows = [];
let columnIndex = DEFAULT_COLUMN_INDEX;
let beginTable = DEFAULT_BEGIN_TABLE;
let endTable = DEFAULT_END_TABLE;
for (let i = 1; !this.isStop && i <= sheet.rowCount; i++) {
const row = sheet.getRow(i);
this.templateManager.defineActualTableStartRow(_ReaderExceljsHelper.beginTableAt(row, sheetDesc, this.isSampleExcel));
this.templateManager.defineActualTableStartRow(_ReaderExceljsHelper.endTableAt(row, sheetDesc, this.isSampleExcel));
columnIndex = (_a = _ReaderExceljsHelper.columnTableIndex(row, sheetDesc, this.isSampleExcel)) != null ? _a : columnIndex;
endTable = (_b = this.templateManager.ActualTableEndRow) != null ? _b : DEFAULT_END_TABLE;
beginTable = (_c = this.templateManager.ActualTableStartRow) != null ? _c : DEFAULT_BEGIN_TABLE;
const section = _ReaderExceljsHelper.getSection(row, beginTable, endTable);
const cells = [];
for (let j = 0; j < row.cellCount; j++) {
if (!((_d = row.getCell(j + 1)) == null ? void 0 : _d.value)) continue;
const cell = this.convertCell(row.getCell(j + 1), section, beginTable, endTable);
cells.push(cell);
if (this.onCell) yield this.onCell(cell);
}
if (this.onRow) {
const rowDataHelper = this.convertRow(row, section, cells);
if (i === 1) trackingRows.push(rowDataHelper);
if (i === sheet.rowCount) trackingRows.push(rowDataHelper);
yield this.onRow(rowDataHelper);
}
}
if (this.onSheet)
yield this.onSheet({
beginTableAt: beginTable != null ? beginTable : 1,
columnIndex,
endTableAt: endTable,
sheetIndex,
name: sheet.name,
detail: sheet,
rowCount: sheet.rowCount,
lastestRow: trackingRows[1],
firstRow: trackingRows[0]
});
});
}
convertCell(cell, section, beginTableAt, endTableAt) {
var _a;
const isVariable = _ReaderExceljsHelper.isVariable(cell.value);
return {
address: _ReaderExceljsHelper.getAddress(cell.address, section, isVariable),
detail: cell,
isVariable,
rowIndex: +cell.fullAddress.row,
section,
label: _ReaderExceljsHelper.getLabel(cell.value + "", isVariable),
variableValue: isVariable ? _ReaderExceljsHelper.getVariableValue(cell) : void 0,
beginTableAt,
endTableAt,
formula: (_a = cell.formula) != null ? _a : void 0
};
}
convertRow(row, section, cells) {
var _a, _b, _c, _d;
return {
detail: row,
rowIndex: row.number,
section,
cells,
beginTableAt: (_b = (_a = cells[0]) == null ? void 0 : _a.beginTableAt) != null ? _b : DEFAULT_BEGIN_TABLE,
endTableAt: (_d = (_c = cells[0]) == null ? void 0 : _c.endTableAt) != null ? _d : DEFAULT_END_TABLE
};
}
static getSection(row, beginTable, endTable) {
const rowIndex = typeof row === "number" ? row : row.number;
let section = "header";
if (beginTable === DEFAULT_BEGIN_TABLE) section = "header";
else if (beginTable !== DEFAULT_BEGIN_TABLE && rowIndex < beginTable) section = "header";
else if (rowIndex > endTable && endTable !== DEFAULT_END_TABLE) section = "footer";
else if (rowIndex > beginTable) section = "table";
return section;
}
/** Get begin table at by row */
static beginTableAt(row, sheetOpts, isSampleExcel = true) {
if (isSampleExcel)
for (let i = 0; i < row.cellCount; i++) {
const cellValue = row.getCell(i + 1).value;
if (cellValue === void 0 || typeof cellValue !== "string") continue;
if (cellValue.includes(SYNTAX.INDEX_COLUMN_TABLE_SYNTAX)) return row.number;
if (cellValue.includes(SYNTAX.VARIABLE_TABLE_SYNTAX)) return row.number - 1;
}
else if (sheetOpts) return sheetOpts.beginTableAt;
return void 0;
}
/** Get end table at by row */
static endTableAt(row, sheetOpts, isSampleExcel = true) {
var _a;
if (isSampleExcel)
for (let i = 0; i < row.cellCount; i++) {
const cellValue = row.getCell(i + 1).value;
if (cellValue === void 0 || typeof cellValue !== "string") continue;
if (cellValue.includes(SYNTAX.END_TABLE_SYNYAX)) return row.number - 1;
if (!cellValue.includes(SYNTAX.INDEX_COLUMN_TABLE_SYNTAX) && cellValue.includes(SYNTAX.VARIABLE_TABLE_SYNTAX)) return row.number;
}
else if (sheetOpts) {
const cellvalues = (_a = row == null ? void 0 : row.values) != null ? _a : [];
if (cellvalues && !cellvalues[sheetOpts.keyTableAt] && row.number > sheetOpts.beginTableAt) return row.number - 1;
}
return void 0;
}
static isNullableRow(row) {
var _a, _b;
return (_b = (_a = row == null ? void 0 : row.values) == null ? void 0 : _a.reduce((acc, val) => acc && !!!val, true)) != null ? _b : false;
}
/** Get key of table by row */
static columnTableIndex(row, sheetOpts, isSampleExcel = true) {
if (isSampleExcel)
for (let i = 0; i < row.cellCount; i++) {
const cellValue = row.getCell(i + 1).value;
if (cellValue === void 0 || typeof cellValue !== "string") continue;
if ((cellValue + "").includes(SYNTAX.INDEX_COLUMN_TABLE_SYNTAX)) return row.number + 1;
}
else if (sheetOpts) return sheetOpts.keyTableAt;
return void 0;
}
static isVariable(cellValue) {
cellValue = cellValue + "";
if (cellValue.includes(SYNTAX.INDEX_COLUMN_TABLE_SYNTAX)) return false;
return cellValue.includes(SYNTAX.VARIABLE_SYNTAX);
}
static getLabel(label, isVariable) {
if (isVariable) return void 0;
if (label.includes(SYNTAX.INDEX_COLUMN_TABLE_SYNTAX)) return label.split("->").pop();
return label;
}
static getAddress(address, section, isVariable) {
return !isVariable ? address : section !== "header" ? address.split(/\d+/)[0] : address;
}
static getVariableValue(cell) {
const cellValue = (cell.value + "").replace("$$", "$");
let fieldName = "";
let type = "string";
fieldName = cellValue.split("$")[1];
if (fieldName.includes("->")) {
const args = fieldName.split("->");
fieldName = args[0];
type = args[1].toLowerCase();
}
return { fieldName, type };
}
static splitAddress(address) {
const col = address.split(/\d+/)[0];
const row = address.split(/[a-zA-Z]/)[1];
return { col, row };
}
};
// src/helpers/parse-type.ts
var TypeParser = class {
constructor(opts) {
var _a;
this.dateFormat = (_a = opts == null ? void 0 : opts.dateFormat) != null ? _a : "DD-MM-YYYY";
}
boolean(val) {
return val === "true";
}
date(val) {
return new Date(val);
}
number(val) {
return +val;
}
object(val) {
return JSON.parse(val);
}
string(val) {
return val + "";
}
};
// src/common/core/error-message-default.ts
var ERROR_MESSAGE = {
MESSAGE_REQUIRED_CELL: "must be required",
MESSAGE_INVALID_CELL: "validate failed"
};
// src/common/error/ValidateError.ts
var ValidateImportError = class extends Error {
constructor(opts) {
super(opts.message);
this.address = opts == null ? void 0 : opts.address;
this.keyField = opts.keyField;
this.value = opts.value;
}
};
// src/helpers/validate-cell-importer.ts
function validateCellImport(value, cellDes, address, keyField) {
const isNullValue = value === void 0 || value === null;
const ValidateImportErrorOpts = {
keyField,
address,
value
};
if (isNullValue && cellDes.required === true) {
const message = `${ERROR_MESSAGE.MESSAGE_REQUIRED_CELL}`;
throw new ValidateImportError(__spreadProps(__spreadValues({}, ValidateImportErrorOpts), {
message
}));
}
if (cellDes.validate) {
const result = cellDes.validate(value);
if (result instanceof Error) throw result;
const { isValid, message } = result;
if (isValid) {
const defaultMessage = `${ERROR_MESSAGE.MESSAGE_INVALID_CELL}`;
throw new ValidateImportError(__spreadProps(__spreadValues({}, ValidateImportErrorOpts), {
message: message != null ? message : defaultMessage
}));
}
}
}
// src/helpers/convert-row-to-table-data-v2.ts
var ConvertorRows2TableData = class {
constructor(opts) {
this.tableCols = [];
this.container = { header: {}, footer: {}, table: [] };
this.section = "header";
this.addresses = { header: {}, footer: {}, table: [] };
this.onTrigger = () => __async(this, null, function* () {
});
this.onErrors = () => __async(this, null, function* () {
});
var _a, _b, _c, _d;
this.chunkSize = (_a = opts.chunkSize) != null ? _a : 10;
this.typeParser = (_b = opts.typeParser) != null ? _b : new TypeParser();
this.templateManager = opts.templateManager;
this.onErrors = (_c = opts == null ? void 0 : opts.onErrors) != null ? _c : this.onErrors;
this.onTrigger = (_d = opts == null ? void 0 : opts.onTrigger) != null ? _d : this.onTrigger;
this.tableCols = this.templateManager.GroupCells.table.map((e) => e.keyName);
}
push(row) {
return __async(this, null, function* () {
if (row === null) return yield this.complete();
const section = this.getSection(row.number);
let values = {};
let address = {};
for (let i = 0; i < row.cellCount; i++) {
const cell = row.getCell(i + 1);
const keyName = this.getKeyField(cell, section);
if (!keyName) continue;
values[keyName] = cell.value;
address[keyName] = cell.address;
}
if (Object.keys(values).length > 0) this.addContainer(section, values, address);
const isTrigger = this.isTrigger(section, row.number);
if (isTrigger) {
const data = yield this.pop(section);
if (data) yield this.onTrigger(section, data);
}
this.section = section;
});
}
pop(section) {
return __async(this, null, function* () {
const data = Array.isArray(this.container[section]) ? this.container[section] : [this.container[section]];
const address = Array.isArray(this.addresses[section]) ? this.addresses[section] : [this.addresses[section]];
const error = [];
if (section === "table") {
this.container.table = [];
this.addresses.table = [];
} else {
this.container[section] = {};
this.addresses[section] = {};
}
for (let i = 0; i < data.length; i++) {
const { errors, row } = this.formatValue(data[i], address[i], section);
if (row) data[i] = row;
if (errors && errors.length > 0) error.push(...errors);
}
if (error.length !== 0) yield this.onErrors(error);
return !data[0] || Object.keys(data[0]).length === 0 ? null : section === "table" ? data : data[0];
});
}
getSection(rowIndex) {
var _a, _b;
const beginTable = (_a = this.templateManager.ActualTableStartRow) != null ? _a : DEFAULT_BEGIN_TABLE;
const endTable = (_b = this.templateManager.ActualTableEndRow) != null ? _b : DEFAULT_END_TABLE;
const section = ReaderExceljsHelper.getSection(rowIndex, beginTable, endTable);
return section;
}
isTrigger(section, rowIndex) {
var _a, _b, _c;
const beginTable = (_a = this.templateManager.ActualTableStartRow) != null ? _a : DEFAULT_BEGIN_TABLE;
const chunkSize = (_c = (_b = this.container) == null ? void 0 : _b.table) == null ? void 0 : _c.length;
if (beginTable === rowIndex) return true;
else if (section === "footer" && this.section === "table") return true;
else if (this.chunkSize <= chunkSize && section === "table") return true;
return false;
}
addContainer(section, data, address) {
if (!this.addresses[section] && section === "table") this.addresses.table = [];
if (!this.addresses[section] && section !== "table") this.addresses[section] = {};
if (section === "table") {
this.container.table.push(data);
this.addresses.table.push(address);
} else {
this.container[section] = __spreadValues(__spreadValues({}, this.container[section]), data);
this.addresses[section] = __spreadValues(__spreadValues({}, this.addresses[section]), address);
}
}
formatValue(data, address, section) {
var _a;
const row = __spreadValues({}, data);
const errors = [];
if (!this.templateManager.GroupCells[section]) return { row: null, errors: null };
(_a = this.templateManager.GroupCells[section]) == null ? void 0 : _a.forEach((cell) => {
let value = row[cell.keyName];
if (cell.setValue) value = cell.setValue(value, row);
if (cell.type && cell.type !== "virtual") value = this.typeParser[cell.type](value);
try {
validateCellImport(value, cell, address[cell.keyName], cell.keyName);
row[cell.keyName] = value;
} catch (error) {
errors.push(error);
}
});
return { row, errors };
}
getKeyField(cell, section) {
if (section === "table") {
return this.tableCols[cell.fullAddress.col - 1];
}
if (section === "header" && this.templateManager.GroupCells.header) {
const headerCells = this.templateManager.GroupCells.header;
const index = headerCells.findIndex((e) => e.address === cell.address);
if (index >= 0) return headerCells[index].keyName;
else return "";
}
if (this.templateManager.GroupCells.footer) {
const footerCells = this.templateManager.GroupCells.footer;
const index = footerCells.findIndex((e) => {
var _a, _b, _c, _d;
const rownum = ((_a = this.templateManager.ActualTableEndRow) != null ? _a : DEFAULT_END_TABLE) + ((_c = (_b = e.fullAddress) == null ? void 0 : _b.row) != null ? _c : 0);
return cell.address === `${(_d = e.fullAddress) == null ? void 0 : _d.col}${rownum}`;
});
if (index >= 0) return footerCells[index].keyName;
else return "";
}
return null;
}
complete() {
return __async(this, null, function* () {
let data = yield this.pop(this.section);
if (data) yield this.onTrigger(this.section, data);
data = yield this.pop("footer");
if (data) yield this.onTrigger("footer", data);
yield this.onTrigger(this.section, null);
});
}
};
// src/importers/ImporterHandler.ts
var ImporterHandler = class {
constructor(eachRow = false) {
this.eachRow = false;
this.eachRow = eachRow;
}
run(tabledata, filterImporter, setGlobalError = (err) => {
}) {
return __async(this, null, function* () {
try {
if (tabledata instanceof Error) yield this.catch(tabledata);
else if (tabledata.header) yield this.handleHeader(tabledata.header, filterImporter);
else if (tabledata.footer) yield this.handleHeader(tabledata.footer, filterImporter);
else if (tabledata.table && !this.eachRow) yield this.handleChunk(tabledata.table, filterImporter);
else if (tabledata.table && this.eachRow)
for (let i = 0; i < tabledata.table.length; i++)
try {
yield this.handleRow(tabledata.table[i], filterImporter);
} catch (error) {
yield this.catch(error);
setGlobalError(error);
}
} catch (e) {
yield this.catch(e);
setGlobalError(e);
}
});
}
catch(error) {
return __async(this, null, function* () {
});
}
handleChunk(chunk, filter) {
return __async(this, null, function* () {
});
}
handleRow(data, filter) {
return __async(this, null, function* () {
});
}
handleHeader(header, filter) {
return __async(this, null, function* () {
});
}
handleFooter(footer, filter) {
return __async(this, null, function* () {
});
}
};
// src/common/core/RingPromise.ts
var import_stream = require("stream");
var RingPromise = class {
constructor(ringSize, task) {
this.ring = [];
this.index = 0;
this.ring = new Array(ringSize).fill(Promise.resolve());
this.task = task;
}
run(...arg) {
return __async(this, null, function* () {
this.index = (this.index + 1) % this.ring.length;
this.ring[this.index] = this.task(...arg);
if (this.index === this.ring.length - 1) yield Promise.all(this.ring);
});
}
stream() {
const that = this;
return new import_stream.Writable({
objectMode: true,
write(arg, _encoding, callback) {
return __async(this, null, function* () {
try {
yield that.run(...arg);
} catch (err) {
return callback(err);
}
callback();
});
},
final(callback) {
Promise.all(that.ring).then(() => callback()).catch((err) => callback(err));
}
});
}
};
// src/common/core/QueueData.ts
var QueueData = class {
constructor(size = 50) {
this.cells = [];
this.resolveFunc = () => {
};
this.size = size;
this.waitingFunc = this.createWaiter();
}
createWaiter() {
return new Promise((resolve) => {
this.resolveFunc = resolve;
});
}
waiting() {
return __async(this, null, function* () {
if (this.cells.length < this.size) return;
yield this.waitingFunc;
});
}
shift() {
const data = this.cells.shift();
if (this.cells.length === this.size - 1) {
this.resolveFunc();
}
return data;
}
add(data) {
this.cells.push(data);
if (this.cells.length === this.size) {
this.waitingFunc = this.createWaiter();
}
}
};
// src/importers/readers/BaseReader.ts
var BaseReader = class {
constructor(opts) {
this.handler = {};
this.queueData = new QueueData(100);
this.ringPromise = {};
var _a;
this.type = opts.type;
this.typeParser = (_a = opts.typeParser) != null ? _a : new TypeParser();
this.templateManager = opts.templateManager;
this.convertorRows2TableData = new ConvertorRows2TableData({
onTrigger: (sect, data) => __async(this, null, function* () {
return yield this.onTrigger(sect, data);
}),
onErrors: (err) => __async(this, null, function* () {
return yield this.onErrors(err);
}),
chunkSize: opts == null ? void 0 : opts.chunkSize,
templateManager: this.templateManager
});
}
run(templateManager, arg, handler, opts) {
return __async(this, null, function* () {
var _a;
this.convertorRows2TableData = new ConvertorRows2TableData({
onTrigger: (sect, data) => __async(this, null, function* () {
return yield this.onTrigger(sect, data);
}),
onErrors: (err) => __async(this, null, function* () {
return yield this.onErrors(err);
}),
chunkSize: opts == null ? void 0 : opts.chunkSize,
templateManager: this.templateManager
});
this.ringPromise = new RingPromise((_a = opts == null ? void 0 : opts.jobCount) != null ? _a : 1, this.createTask());
this.options = opts;
this.handler = handler;
this.templateManager = templateManager;
yield this.load(arg);
this.consumeData();
});
}
consumeData() {
return __async(this, null, function* () {
while (this.globalError === void 0) {
const queueData = this.queueData.shift();
if (queueData && queueData.data) yield this.ringPromise.run(queueData == null ? void 0 : queueData.data, queueData == null ? void 0 : queueData.filter);
else if (!(queueData == null ? void 0 : queueData.data)) break;
else if (!queueData) yield new Promise((r) => setTimeout(r, 2));
if (this.globalError) {
console.error("Error occurred, stopping the reader.");
break;
}
}
});
}
getType() {
return this.type;
}
createTask() {
return (data, filter) => __async(this, null, function* () {
if (this.handler instanceof ImporterHandler) yield this.handler.run(data, filter, this.setGlobalError);
else
for (let i = 0; i < this.handler.length; i++) {
try {
data = yield this.handler[i](data, filter);
} catch (error) {
this.setGlobalError(error);
}
}
});
}
onTrigger(section, data) {
return __async(this, null, function* () {
var _a;
const filter = {
section,
sheetIndex: (_a = this.templateManager.SheetInformation.sheetIndex) != null ? _a : 0,
sheetName: this.templateManager.SheetInformation.sheetName,
isHasNext: data !== null
};
yield this.queueData.waiting();
this.queueData.add({ data, filter });
});
}
onErrors(errors) {
return __async(this, null, function* () {
var _a;
errors = Array.isArray(errors) ? errors : [errors];
const task = this.createTask();
if ((_a = this.options) == null ? void 0 : _a.ignoreErrors) this.setGlobalError(errors[0]);
else
for (let i = 0; i < errors.length; i++) {
yield task(errors[i], null);
}
});
}
setGlobalError(err) {
var _a;
if (!((_a = this.options) == null ? void 0 : _a.ignoreErrors) && this.globalError) this.globalError = err;
}
};
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
BaseReader
});