@nocobase/plugin-action-import
Version:
Import records using excel templates. You can configure which fields to import and templates will be generated automatically.
531 lines (529 loc) • 21.1 kB
JavaScript
/**
* This file is part of the NocoBase (R) project.
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
* Authors: NocoBase Team.
*
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
* For more information, please refer to: https://www.nocobase.com/agreement.
*/
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var xlsx_importer_exports = {};
__export(xlsx_importer_exports, {
XlsxImporter: () => XlsxImporter
});
module.exports = __toCommonJS(xlsx_importer_exports);
var XLSX = __toESM(require("xlsx"));
var import_lodash = __toESM(require("lodash"));
var import_database = require("@nocobase/database");
var import_events = __toESM(require("events"));
var import_errors = require("../errors");
var import_lodash2 = __toESM(require("lodash"));
var import_utils = require("../utils");
class XlsxImporter extends import_events.default {
constructor(options) {
super();
this.options = options;
if (typeof options.columns === "string") {
options.columns = JSON.parse(options.columns);
}
if (options.columns.length == 0) {
throw new Error(`columns is empty`);
}
this.repository = options.repository ? options.repository : options.collection.repository;
this.logger = options.logger;
this.loggerService = new import_utils.LoggerService({ logger: this.logger });
}
repository;
loggerService;
logger;
async beforePerformImport(data, options) {
return data;
}
async validateBySpaces(data, ctx) {
var _a;
if ((_a = ctx == null ? void 0 : ctx.space) == null ? void 0 : _a.can) {
await ctx.space.can({
data: (data == null ? void 0 : data.slice(1)) || [],
columns: this.options.columns.map((column) => column.dataIndex),
collection: this.options.collection.name,
ctx
});
}
}
async validate(ctx) {
this.validateColumns(ctx);
const data = await this.getData(ctx);
await this.validateBySpaces(data, ctx);
return data;
}
async run(options = {}) {
var _a, _b, _c, _d;
const hasExternalTransaction = !!options.transaction;
const { db } = this.options.collectionManager;
if (hasExternalTransaction) {
try {
const data = await this.loggerService.measureExecutedTime(
async () => this.validate(options.context),
"Validation completed in {time}ms"
);
const imported = await this.loggerService.measureExecutedTime(
async () => this.performImport(data, options),
"Data import completed in {time}ms"
);
(_a = this.logger) == null ? void 0 : _a.info(`Import completed successfully, imported ${imported} records`);
if (db) {
await this.loggerService.measureExecutedTime(
async () => this.resetSeq(options),
"Sequence reset completed in {time}ms"
);
}
return imported;
} catch (error) {
(_b = this.logger) == null ? void 0 : _b.error(`Import failed: ${this.renderErrorMessage(error)}`, {
originalError: error.stack || error.toString()
});
throw error;
}
}
try {
const data = await this.loggerService.measureExecutedTime(
async () => this.validate(options.context),
"Validation completed in {time}ms"
);
const imported = await this.loggerService.measureExecutedTime(
async () => this.performImport(data, options),
"Data import completed in {time}ms"
);
(_c = this.logger) == null ? void 0 : _c.info(`Import completed successfully, imported ${imported} records`);
if (db) {
await this.loggerService.measureExecutedTime(
async () => this.resetSeq({}),
"Sequence reset completed in {time}ms"
);
}
return imported;
} catch (error) {
(_d = this.logger) == null ? void 0 : _d.error(`Import failed: ${this.renderErrorMessage(error)}`, {
originalError: error.stack || error.toString()
});
throw error;
}
}
async resetSeq(options) {
const { transaction } = options;
const { db } = this.options.collectionManager;
const collection = this.options.collection;
const autoIncrementAttribute = collection.model.autoIncrementAttribute;
if (!autoIncrementAttribute) {
return;
}
const field = this.options.collection.getField(autoIncrementAttribute);
if (field && !(field instanceof import_database.NumberField)) {
return;
}
let hasImportedAutoIncrementPrimary = false;
for (const importedDataIndex of this.getColumnsByPermission(options == null ? void 0 : options.context)) {
if (importedDataIndex.dataIndex[0] === autoIncrementAttribute) {
hasImportedAutoIncrementPrimary = true;
break;
}
}
if (!hasImportedAutoIncrementPrimary) {
return;
}
let tableInfo = collection.getTableNameWithSchema();
if (typeof tableInfo === "string") {
tableInfo = {
tableName: tableInfo
};
}
const autoIncrInfo = await db.queryInterface.getAutoIncrementInfo({
tableInfo,
fieldName: autoIncrementAttribute,
transaction
});
const maxVal = await collection.model.max(autoIncrementAttribute, { transaction });
if (maxVal == null) {
return;
}
if (typeof autoIncrInfo.currentVal === "number" && maxVal <= autoIncrInfo.currentVal) {
return;
}
const queryInterface = db.queryInterface;
await queryInterface.setAutoIncrementVal({
tableInfo,
columnName: collection.model.rawAttributes[autoIncrementAttribute].field,
currentVal: maxVal,
seqName: autoIncrInfo.seqName,
transaction
});
this.emit("seqReset", { maxVal, seqName: autoIncrInfo.seqName });
}
getColumnsByPermission(ctx) {
var _a, _b, _c;
const columns = this.options.columns;
const fields = (_c = (_b = (_a = ctx == null ? void 0 : ctx.permission) == null ? void 0 : _a.can) == null ? void 0 : _b.params) == null ? void 0 : _c.fields;
if (!Array.isArray(fields)) {
return columns;
}
return columns.filter((x) => import_lodash2.default.includes(fields, x.dataIndex[0]));
}
validateColumns(ctx) {
var _a, _b, _c;
const columns = this.getColumnsByPermission(ctx);
if (columns.length === 0) {
throw new import_errors.ImportValidationError("Columns configuration is empty");
}
for (const column of columns) {
if (!Array.isArray(column == null ? void 0 : column.dataIndex) || column.dataIndex.length === 0) {
throw new import_errors.ImportValidationError("Columns configuration is empty");
}
if (column.dataIndex.length > 2) {
throw new import_errors.ImportValidationError("Invalid field: {{field}}", {
field: column.dataIndex.join(".")
});
}
const [fieldName, filterKey] = column.dataIndex;
if (typeof fieldName !== "string" || fieldName.trim() === "") {
throw new import_errors.ImportValidationError("Invalid field: {{field}}", { field: String(fieldName) });
}
const field = this.options.collection.getField(fieldName);
if (!field) {
throw new import_errors.ImportValidationError("Field not found: {{field}}", { field: fieldName });
}
if (column.dataIndex.length > 1) {
if (typeof field.isRelationField !== "function" || !field.isRelationField()) {
throw new import_errors.ImportValidationError("Invalid field: {{field}}", {
field: column.dataIndex.join(".")
});
}
if (typeof filterKey !== "string" || filterKey.trim() === "") {
throw new import_errors.ImportValidationError("Invalid field: {{field}}", {
field: column.dataIndex.join(".")
});
}
const targetCollection = (_a = field.targetCollection) == null ? void 0 : _a.call(field);
if (!targetCollection) {
throw new import_errors.ImportValidationError("Field not found: {{field}}", {
field: column.dataIndex.join(".")
});
}
const targetField = targetCollection.getField(filterKey);
const isValidAttribute = (_c = (_b = targetCollection.model) == null ? void 0 : _b.getAttributes()) == null ? void 0 : _c[filterKey];
if (!targetField && !isValidAttribute) {
throw new import_errors.ImportValidationError("Field not found: {{field}}", {
field: `${fieldName}.${filterKey}`
});
}
}
}
}
async performImport(data, options) {
const chunkSize = this.options.chunkSize || 1e3;
const chunks = import_lodash.default.chunk(data.slice(1), chunkSize);
let handingRowIndex = 1;
let imported = 0;
const total = data.length - 1;
if (this.options.explain) {
handingRowIndex += 1;
}
let chunkRows;
while ((chunkRows = chunks.shift()) !== void 0) {
await this.handleChuckRows(chunkRows, options, { handingRowIndex, context: options == null ? void 0 : options.context });
handingRowIndex += chunkRows.length;
imported += chunkRows.length;
this.emit("progress", {
total,
current: imported
});
}
return imported;
}
getModel() {
return this.repository instanceof import_database.RelationRepository ? this.repository.targetModel : this.repository.model;
}
async handleRowValuesWithColumns(row, rowValues, options, columns, rowIndex = 1) {
for (let index = 0; index < columns.length; index++) {
const column = columns[index];
const field = this.options.collection.getField(column.dataIndex[0]);
if (!field) {
throw new import_errors.ImportValidationError("Import validation. Field not found", {
field: column.dataIndex[0]
});
}
const str = row[index];
const dataKey = column.dataIndex[0];
const fieldOptions = field.options;
const interfaceName = fieldOptions.interface;
const InterfaceClass = this.options.collectionManager.getFieldInterface(interfaceName);
if (!InterfaceClass) {
rowValues[dataKey] = str;
continue;
}
const interfaceInstance = new InterfaceClass(field.options);
const ctx = options.context ? Object.create(options.context) : {};
ctx.transaction = options.transaction;
ctx.field = field;
if (column.dataIndex.length > 1) {
ctx.associationField = field;
ctx.targetCollection = field.targetCollection();
ctx.filterKey = column.dataIndex[1];
}
try {
rowValues[dataKey] = str == null ? null : await interfaceInstance.toValue(this.trimString(str), ctx);
} catch (error) {
throw new import_errors.ImportValidationError("Failed to parse field {{field}} in row {{rowIndex}}: {{message}}", {
rowIndex,
field: dataKey,
message: error.message
});
}
}
const model = this.getModel();
const guard = import_database.UpdateGuard.fromOptions(model, {
...options,
action: "create",
underscored: this.repository.collection.options.underscored
});
rowValues = this.repository instanceof import_database.RelationRepository ? model.callSetters(guard.sanitize(rowValues || {}), options) : guard.sanitize(rowValues);
}
async handleChuckRows(chunkRows, runOptions, options) {
var _a, _b;
const { handingRowIndex: chunkStartRowIndex = 1 } = options;
const db = this.options.collectionManager.db;
const externalTransaction = runOptions == null ? void 0 : runOptions.transaction;
const transaction = externalTransaction || (db ? await db.sequelize.transaction() : null);
const chunkRunOptions = { ...runOptions, transaction };
const columns = this.getColumnsByPermission(options == null ? void 0 : options.context);
const translate = (message) => {
var _a2;
if ((_a2 = options.context) == null ? void 0 : _a2.t) {
return options.context.t(message, { ns: "action-import" });
} else {
return message;
}
};
const rows = [];
let inChunkRowIndex = 0;
try {
for (let i = 0; i < chunkRows.length; i++) {
inChunkRowIndex = i;
const row = chunkRows[i];
const rowValues = {};
await this.handleRowValuesWithColumns(row, rowValues, chunkRunOptions, columns, chunkStartRowIndex + i);
rows.push({
...this.options.rowDefaultValues || {},
...rowValues
});
}
await this.loggerService.measureExecutedTime(
async () => this.performInsert({
values: rows,
transaction,
context: options == null ? void 0 : options.context
}),
"Record insertion completed in {time}ms"
);
await new Promise((resolve) => setTimeout(resolve, 5));
if (!externalTransaction) {
await (transaction == null ? void 0 : transaction.commit());
}
} catch (error) {
if (!externalTransaction) {
await (transaction == null ? void 0 : transaction.rollback());
}
if (error.name === "ImportValidationError") {
throw error;
}
if (error.name === "SequelizeUniqueConstraintError") {
throw new Error(`${translate("Unique constraint error, fields:")} ${JSON.stringify(error.fields)}`);
}
const failedRowIndex = chunkStartRowIndex + inChunkRowIndex;
if ((_a = error.params) == null ? void 0 : _a.rowIndex) {
error.params.rowIndex = failedRowIndex;
}
(_b = this.logger) == null ? void 0 : _b.error(`Import error at row ${failedRowIndex}: ${error.message}`, {
rowIndex: failedRowIndex,
rowData: chunkRows[inChunkRowIndex],
originalError: error.stack || error.toString()
});
throw new import_errors.ImportError(`Import failed at row ${failedRowIndex}`, {
rowIndex: failedRowIndex,
rowData: chunkRows[inChunkRowIndex],
cause: error
});
}
return;
}
async performInsert(insertOptions) {
const { values, transaction, context } = insertOptions;
const instances = await this.loggerService.measureExecutedTime(
async () => this.getModel().bulkCreate(values, {
transaction,
hooks: insertOptions.hooks == void 0 ? true : insertOptions.hooks,
returning: true,
context
}),
"Row {{rowIndex}}: bulkCreate completed in {time}ms"
);
if (this.repository instanceof import_database.RelationRepository) {
await this.associateRecords(instances, import_lodash2.default.omit(insertOptions, "values"));
}
const db = this.options.collectionManager.db;
for (let i = 0; i < instances.length; i++) {
const instance = instances[i];
const value = values[i];
if (insertOptions.hooks !== false) {
await this.loggerService.measureExecutedTime(
async () => {
await db.emitAsync(`${this.repository.collection.name}.afterCreate`, instance, {
transaction
});
await db.emitAsync(`${this.repository.collection.name}.afterSave`, instance, {
transaction
});
instance.clearChangedWithAssociations();
},
`Row ${i + 1}: afterSave event emitted in {time}ms`,
"debug"
);
}
await this.loggerService.measureExecutedTime(
async () => (0, import_database.updateAssociations)(instance, value, { transaction }),
`Row ${i + 1}: updateAssociations completed in {time}ms`,
"debug"
);
if ((context == null ? void 0 : context.skipWorkflow) !== true) {
await this.loggerService.measureExecutedTime(
async () => {
await db.emitAsync(`${this.repository.collection.name}.afterCreateWithAssociations`, instance, {
transaction
});
await db.emitAsync(`${this.repository.collection.name}.afterSaveWithAssociations`, instance, {
transaction
});
instance.clearChangedWithAssociations();
},
`Row ${i + 1}: afterCreate event emitted in {time}ms`,
"debug"
);
}
instances[i] = null;
}
return instances;
}
async associateRecords(targets, options = {}) {
if (!(this.repository instanceof import_database.RelationRepository)) {
return;
}
const accessors = this.repository.accessors();
const sourceModel = await this.repository.getSourceModel();
if (!accessors || !sourceModel) {
throw new Error("Missing accessors or source model.");
}
if (accessors.addMultiple) {
await sourceModel[accessors.addMultiple](targets, options);
} else if (accessors.add) {
await Promise.all(
targets.map((target) => sourceModel[accessors.add](target, options))
);
} else if (accessors.set) {
if (targets.length > 1) {
throw new Error("Cannot associate multiple records to a single-valued relation.");
}
await sourceModel[accessors.set](targets[0], options);
} else {
throw new Error(`Unsupported association or no usable accessor on ${this.repository["association"]}`);
}
}
renderErrorMessage(error) {
let message = error.message;
if (error.parent) {
message += `: ${error.parent.message}`;
}
return message;
}
trimString(str) {
if (typeof str === "string") {
return str.trim();
}
return str;
}
getExpectedHeaders(ctx) {
const columns = this.getColumnsByPermission(ctx);
return columns.map((col) => col.title || col.defaultTitle);
}
async getData(ctx) {
const workbook = this.options.workbook;
const worksheet = workbook.Sheets[workbook.SheetNames[0]];
let data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null, blankrows: false });
this.options.workbook = null;
const expectedHeaders = this.getExpectedHeaders(ctx);
const { headerRowIndex, headers } = this.findAndValidateHeaders({ data, expectedHeaders });
if (headerRowIndex === -1) {
throw new import_errors.ImportValidationError("Headers not found. Expected headers: {{headers}}", {
headers: expectedHeaders.join(", ")
});
}
data = this.alignWithHeaders({ data, expectedHeaders, headerRowIndex });
const rows = data.slice(headerRowIndex + 1);
if (rows.length === 0) {
throw new import_errors.ImportValidationError("No data to import");
}
return [headers, ...rows];
}
alignWithHeaders(params) {
const { data, expectedHeaders, headerRowIndex } = params;
const headerRow = data[headerRowIndex];
const keepCols = expectedHeaders.map((header) => headerRow.indexOf(header));
if (keepCols.some((index) => index < 0)) {
throw new import_errors.ImportValidationError("Headers not found. Expected headers: {{headers}}", {
headers: expectedHeaders.join(", ")
});
}
return data.map((row) => keepCols.map((i) => row[i]));
}
findAndValidateHeaders(options) {
const { data, expectedHeaders } = options;
for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
const row = data[rowIndex];
const actualHeaders = row.filter((cell) => cell !== null && cell !== "");
const allHeadersFound = expectedHeaders.every((header) => actualHeaders.includes(header));
if (allHeadersFound) {
const orderedHeaders = expectedHeaders.filter((h) => actualHeaders.includes(h));
return { headerRowIndex: rowIndex, headers: orderedHeaders };
}
}
return { headerRowIndex: -1, headers: [] };
}
}
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {
XlsxImporter
});