n8n
Version:
n8n Workflow Automation Tool
456 lines • 23.2 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DataTableService = void 0;
const backend_common_1 = require("@n8n/backend-common");
const db_1 = require("@n8n/db");
const di_1 = require("@n8n/di");
const n8n_workflow_1 = require("n8n-workflow");
const csv_parser_service_1 = require("./csv-parser.service");
const data_table_column_repository_1 = require("./data-table-column.repository");
const data_table_file_cleanup_service_1 = require("./data-table-file-cleanup.service");
const data_table_rows_repository_1 = require("./data-table-rows.repository");
const data_table_size_validator_service_1 = require("./data-table-size-validator.service");
const data_table_repository_1 = require("./data-table.repository");
const data_table_types_1 = require("./data-table.types");
const data_table_column_not_found_error_1 = require("./errors/data-table-column-not-found.error");
const data_table_file_upload_error_1 = require("./errors/data-table-file-upload.error");
const data_table_name_conflict_error_1 = require("./errors/data-table-name-conflict.error");
const data_table_not_found_error_1 = require("./errors/data-table-not-found.error");
const data_table_validation_error_1 = require("./errors/data-table-validation.error");
const sql_utils_1 = require("./utils/sql-utils");
const role_service_1 = require("../../services/role.service");
let DataTableService = class DataTableService {
constructor(dataTableRepository, dataTableColumnRepository, dataTableRowsRepository, logger, dataTableSizeValidator, projectRelationRepository, roleService, csvParserService, fileCleanupService) {
this.dataTableRepository = dataTableRepository;
this.dataTableColumnRepository = dataTableColumnRepository;
this.dataTableRowsRepository = dataTableRowsRepository;
this.logger = logger;
this.dataTableSizeValidator = dataTableSizeValidator;
this.projectRelationRepository = projectRelationRepository;
this.roleService = roleService;
this.csvParserService = csvParserService;
this.fileCleanupService = fileCleanupService;
this.logger = this.logger.scoped('data-table');
}
async start() { }
async shutdown() { }
async createDataTable(projectId, dto) {
await this.validateUniqueName(dto.name, projectId);
const result = await this.dataTableRepository.createDataTable(projectId, dto.name, dto.columns);
if (dto.fileId) {
try {
await this.importDataFromFile(projectId, result.id, dto.fileId, dto.hasHeaders ?? true);
await this.fileCleanupService.deleteFile(dto.fileId);
}
catch (error) {
await this.deleteDataTable(result.id, projectId);
throw error;
}
}
this.dataTableSizeValidator.reset();
return result;
}
async importDataFromFile(projectId, dataTableId, fileId, hasHeaders) {
try {
const tableColumns = await this.getColumns(dataTableId, projectId);
const csvMetadata = await this.csvParserService.parseFile(fileId, hasHeaders);
const columnMapping = new Map();
csvMetadata.columns.forEach((csvColumn, index) => {
if (tableColumns[index]) {
columnMapping.set(csvColumn.name, tableColumns[index].name);
}
});
const csvRows = await this.csvParserService.parseFileData(fileId, hasHeaders);
const transformedRows = csvRows.map((csvRow) => {
const transformedRow = {};
for (const [csvColName, value] of Object.entries(csvRow)) {
const tableColName = columnMapping.get(csvColName);
if (tableColName) {
transformedRow[tableColName] = value;
}
}
return transformedRow;
});
if (transformedRows.length > 0) {
await this.insertRows(dataTableId, projectId, transformedRows);
}
}
catch (error) {
this.logger.error('Failed to import data from CSV file', { error, fileId, dataTableId });
throw new data_table_file_upload_error_1.FileUploadError(error instanceof Error ? error.message : 'Failed to read CSV file');
}
}
async updateDataTable(dataTableId, projectId, dto) {
await this.validateDataTableExists(dataTableId, projectId);
await this.validateUniqueName(dto.name, projectId);
await this.dataTableRepository.update({ id: dataTableId }, { name: dto.name });
return true;
}
async transferDataTablesByProjectId(fromProjectId, toProjectId) {
return await this.dataTableRepository.transferDataTableByProjectId(fromProjectId, toProjectId);
}
async deleteDataTableByProjectId(projectId) {
const result = await this.dataTableRepository.deleteDataTableByProjectId(projectId);
if (result) {
this.dataTableSizeValidator.reset();
}
return result;
}
async deleteDataTableAll() {
const result = await this.dataTableRepository.deleteDataTableAll();
if (result) {
this.dataTableSizeValidator.reset();
}
return result;
}
async deleteDataTable(dataTableId, projectId) {
await this.validateDataTableExists(dataTableId, projectId);
await this.dataTableRepository.deleteDataTable(dataTableId);
this.dataTableSizeValidator.reset();
return true;
}
async addColumn(dataTableId, projectId, dto) {
await this.validateDataTableExists(dataTableId, projectId);
const result = await this.dataTableColumnRepository.addColumn(dataTableId, dto);
await this.dataTableRepository.touchUpdatedAt(dataTableId);
return result;
}
async moveColumn(dataTableId, projectId, columnId, dto) {
await this.validateDataTableExists(dataTableId, projectId);
const existingColumn = await this.validateColumnExists(dataTableId, columnId);
await this.dataTableColumnRepository.moveColumn(dataTableId, existingColumn, dto.targetIndex);
return true;
}
async deleteColumn(dataTableId, projectId, columnId) {
await this.validateDataTableExists(dataTableId, projectId);
const existingColumn = await this.validateColumnExists(dataTableId, columnId);
await this.dataTableColumnRepository.deleteColumn(dataTableId, existingColumn);
await this.dataTableRepository.touchUpdatedAt(dataTableId);
return true;
}
async renameColumn(dataTableId, projectId, columnId, dto) {
await this.validateDataTableExists(dataTableId, projectId);
const existingColumn = await this.validateColumnExists(dataTableId, columnId);
return await this.dataTableColumnRepository.renameColumn(dataTableId, existingColumn, dto.name);
}
async getManyAndCount(options) {
return await this.dataTableRepository.getManyAndCount(options);
}
async getManyRowsAndCount(dataTableId, projectId, dto) {
await this.validateDataTableExists(dataTableId, projectId);
return await this.dataTableColumnRepository.manager.transaction(async (em) => {
const columns = await this.dataTableColumnRepository.getColumns(dataTableId, em);
const transformedDto = dto.filter
? { ...dto, filter: this.validateAndTransformFilters(dto.filter, columns) }
: dto;
const result = await this.dataTableRowsRepository.getManyAndCount(dataTableId, transformedDto, columns, em);
return {
count: result.count,
data: (0, sql_utils_1.normalizeRows)(result.data, columns),
};
});
}
async getColumns(dataTableId, projectId) {
await this.validateDataTableExists(dataTableId, projectId);
return await this.dataTableColumnRepository.getColumns(dataTableId);
}
async insertRows(dataTableId, projectId, rows, returnType = 'count') {
await this.validateDataTableSize();
await this.validateDataTableExists(dataTableId, projectId);
const result = await this.dataTableColumnRepository.manager.transaction(async (trx) => {
const columns = await this.dataTableColumnRepository.getColumns(dataTableId, trx);
const transformedRows = this.validateAndTransformRows(rows, columns);
return await this.dataTableRowsRepository.insertRows(dataTableId, transformedRows, columns, returnType, trx);
});
this.dataTableSizeValidator.reset();
await this.dataTableRepository.touchUpdatedAt(dataTableId);
return result;
}
async upsertRow(dataTableId, projectId, dto, returnData = false, dryRun = false) {
await this.validateDataTableSize();
await this.validateDataTableExists(dataTableId, projectId);
const result = await this.dataTableColumnRepository.manager.transaction(async (trx) => {
const columns = await this.dataTableColumnRepository.getColumns(dataTableId, trx);
const { data, filter } = this.validateAndTransformUpdateParams(dto, columns);
if (dryRun) {
return await this.dataTableRowsRepository.dryRunUpsertRow(dataTableId, data, filter, columns, trx);
}
const updated = await this.dataTableRowsRepository.updateRows(dataTableId, data, filter, columns, true, trx);
if (updated.length > 0) {
return returnData ? updated : true;
}
const inserted = await this.dataTableRowsRepository.insertRows(dataTableId, [data], columns, returnData ? 'all' : 'id', trx);
return returnData ? inserted : true;
});
if (!dryRun) {
this.dataTableSizeValidator.reset();
await this.dataTableRepository.touchUpdatedAt(dataTableId);
}
return result;
}
validateAndTransformUpdateParams({ filter, data }, columns) {
if (columns.length === 0) {
throw new data_table_validation_error_1.DataTableValidationError('No columns found for this data table or data table not found');
}
if (!filter?.filters || filter.filters.length === 0) {
throw new data_table_validation_error_1.DataTableValidationError('Filter must not be empty');
}
if (!data || Object.keys(data).length === 0) {
throw new data_table_validation_error_1.DataTableValidationError('Data columns must not be empty');
}
const [transformedData] = this.validateAndTransformRows([data], columns, false);
const transformedFilter = this.validateAndTransformFilters(filter, columns);
return { data: transformedData, filter: transformedFilter };
}
async updateRows(dataTableId, projectId, dto, returnData = false, dryRun = false) {
await this.validateDataTableSize();
await this.validateDataTableExists(dataTableId, projectId);
const result = await this.dataTableColumnRepository.manager.transaction(async (trx) => {
const columns = await this.dataTableColumnRepository.getColumns(dataTableId, trx);
const { data, filter } = this.validateAndTransformUpdateParams(dto, columns);
if (dryRun) {
return await this.dataTableRowsRepository.dryRunUpdateRows(dataTableId, data, filter, columns, trx);
}
return await this.dataTableRowsRepository.updateRows(dataTableId, data, filter, columns, returnData, trx);
});
if (!dryRun) {
this.dataTableSizeValidator.reset();
await this.dataTableRepository.touchUpdatedAt(dataTableId);
}
return result;
}
async deleteRows(dataTableId, projectId, dto, returnData = false, dryRun = false) {
await this.validateDataTableExists(dataTableId, projectId);
const result = await this.dataTableColumnRepository.manager.transaction(async (trx) => {
const columns = await this.dataTableColumnRepository.getColumns(dataTableId, trx);
if (!dto.filter?.filters || dto.filter.filters.length === 0) {
throw new data_table_validation_error_1.DataTableValidationError('Filter is required for delete operations to prevent accidental deletion of all data');
}
const transformedFilter = this.validateAndTransformFilters(dto.filter, columns);
return await this.dataTableRowsRepository.deleteRows(dataTableId, columns, transformedFilter, returnData, dryRun, trx);
});
if (!dryRun) {
this.dataTableSizeValidator.reset();
await this.dataTableRepository.touchUpdatedAt(dataTableId);
}
return result;
}
validateAndTransformRows(rows, columns, includeSystemColumns = false, skipDateTransform = false) {
const allColumns = includeSystemColumns
? [
...Object.entries(n8n_workflow_1.DATA_TABLE_SYSTEM_COLUMN_TYPE_MAP).map(([name, type]) => ({
name,
type,
})),
...columns,
]
: columns;
const columnNames = new Set(allColumns.map((x) => x.name));
const columnTypeMap = new Map(allColumns.map((x) => [x.name, x.type]));
return rows.map((row) => {
const transformedRow = {};
const keys = Object.keys(row);
for (const key of keys) {
if (!columnNames.has(key)) {
throw new data_table_validation_error_1.DataTableValidationError(`unknown column name '${key}'`);
}
transformedRow[key] = this.validateAndTransformCell(row[key], key, columnTypeMap, skipDateTransform);
}
return transformedRow;
});
}
validateAndTransformCell(cell, key, columnTypeMap, skipDateTransform = false) {
if (cell === null)
return null;
const columnType = columnTypeMap.get(key);
if (!columnType)
return cell;
const fieldType = data_table_types_1.columnTypeToFieldType[columnType];
if (!fieldType)
return cell;
const validationResult = (0, n8n_workflow_1.validateFieldType)(key, cell, fieldType, {
strict: false,
parseStrings: false,
});
if (!validationResult.valid) {
throw new data_table_validation_error_1.DataTableValidationError(`value '${String(cell)}' does not match column type '${columnType}': ${validationResult.errorMessage}`);
}
if (columnType === 'date') {
if (skipDateTransform && cell instanceof Date) {
return cell;
}
try {
const dateInISO = validationResult.newValue.toUTC().toISO();
return dateInISO;
}
catch {
throw new data_table_validation_error_1.DataTableValidationError(`value '${String(cell)}' does not match column type 'date'`);
}
}
return validationResult.newValue;
}
async validateDataTableExists(dataTableId, projectId) {
const existingTable = await this.dataTableRepository.findOneBy({
id: dataTableId,
project: {
id: projectId,
},
});
if (!existingTable) {
throw new data_table_not_found_error_1.DataTableNotFoundError(dataTableId);
}
return existingTable;
}
async validateColumnExists(dataTableId, columnId) {
const existingColumn = await this.dataTableColumnRepository.findOneBy({
id: columnId,
dataTableId,
});
if (existingColumn === null) {
throw new data_table_column_not_found_error_1.DataTableColumnNotFoundError(dataTableId, columnId);
}
return existingColumn;
}
async validateUniqueName(name, projectId) {
const hasNameClash = await this.dataTableRepository.existsBy({
name,
projectId,
});
if (hasNameClash) {
throw new data_table_name_conflict_error_1.DataTableNameConflictError(name);
}
}
validateAndTransformFilters(filterObject, columns) {
const transformedRows = this.validateAndTransformRows(filterObject.filters.map((f) => {
return {
[f.columnName]: f.value,
};
}), columns, true, true);
const transformedFilters = filterObject.filters.map((filter, index) => {
const transformedValue = transformedRows[index][filter.columnName];
if (['like', 'ilike'].includes(filter.condition)) {
if (transformedValue === null || transformedValue === undefined) {
throw new data_table_validation_error_1.DataTableValidationError(`${filter.condition.toUpperCase()} filter value cannot be null or undefined`);
}
if (typeof transformedValue !== 'string') {
throw new data_table_validation_error_1.DataTableValidationError(`${filter.condition.toUpperCase()} filter value must be a string`);
}
const valueWithWildcards = transformedValue.includes('%')
? transformedValue
: `%${transformedValue}%`;
return { ...filter, value: valueWithWildcards };
}
if (['gt', 'gte', 'lt', 'lte'].includes(filter.condition)) {
if (transformedValue === null || transformedValue === undefined) {
throw new data_table_validation_error_1.DataTableValidationError(`${filter.condition.toUpperCase()} filter value cannot be null or undefined`);
}
}
return { ...filter, value: transformedValue };
});
return { ...filterObject, filters: transformedFilters };
}
async validateDataTableSize() {
await this.dataTableSizeValidator.validateSize(async () => await this.dataTableRepository.findDataTablesSize());
}
async getDataTablesSize(user) {
const allSizeData = await this.dataTableSizeValidator.getCachedSizeData(async () => await this.dataTableRepository.findDataTablesSize());
const roles = await this.roleService.rolesWithScope('project', ['dataTable:listProject']);
const accessibleProjectIds = await this.projectRelationRepository.getAccessibleProjectsByRoles(user.id, roles);
const accessibleProjectIdsSet = new Set(accessibleProjectIds);
const accessibleDataTables = Object.fromEntries(Object.entries(allSizeData.dataTables).filter(([, dataTableInfo]) => accessibleProjectIdsSet.has(dataTableInfo.projectId)));
return {
totalBytes: allSizeData.totalBytes,
quotaStatus: this.dataTableSizeValidator.sizeToState(allSizeData.totalBytes),
dataTables: accessibleDataTables,
};
}
async generateDataTableCsv(dataTableId, projectId) {
const dataTable = await this.validateDataTableExists(dataTableId, projectId);
const columns = await this.dataTableColumnRepository.getColumns(dataTableId);
const { data: rows } = await this.dataTableRowsRepository.getManyAndCount(dataTableId, {
skip: 0,
}, columns);
const csvContent = this.buildCsvContent(rows, columns);
return {
csvContent,
dataTableName: dataTable.name,
};
}
buildCsvContent(rows, columns) {
const sortedColumns = [...columns].sort((a, b) => a.index - b.index);
const userHeaders = sortedColumns.map((col) => col.name);
const headers = ['id', ...userHeaders, 'createdAt', 'updatedAt'];
const csvRows = [headers.map((h) => this.escapeCsvValue(h)).join(',')];
for (const row of rows) {
const values = [];
values.push(this.escapeCsvValue(row.id));
for (const column of sortedColumns) {
const value = row[column.name];
values.push(this.escapeCsvValue(this.formatValueForCsv(value, column.type)));
}
values.push(this.escapeCsvValue(this.formatDateForCsv(row.createdAt)));
values.push(this.escapeCsvValue(this.formatDateForCsv(row.updatedAt)));
csvRows.push(values.join(','));
}
return csvRows.join('\n');
}
formatValueForCsv(value, columnType) {
if (value === null || value === undefined) {
return '';
}
if (columnType === 'date') {
if (value instanceof Date || typeof value === 'string') {
return this.formatDateForCsv(value);
}
}
if (columnType === 'boolean') {
return String(value);
}
if (columnType === 'number') {
return String(value);
}
return String(value);
}
formatDateForCsv(date) {
if (date instanceof Date) {
return date.toISOString();
}
const parsed = new Date(date);
return !isNaN(parsed.getTime()) ? parsed.toISOString() : String(date);
}
escapeCsvValue(value) {
const str = String(value);
const hasLeadingOrTrailingSpace = str.length > 0 && (str[0] === ' ' || str[str.length - 1] === ' ');
if (str.includes(',') ||
str.includes('"') ||
str.includes('\n') ||
str.includes('\r') ||
hasLeadingOrTrailingSpace) {
return `"${str.replace(/"/g, '""')}"`;
}
return str;
}
};
exports.DataTableService = DataTableService;
exports.DataTableService = DataTableService = __decorate([
(0, di_1.Service)(),
__metadata("design:paramtypes", [data_table_repository_1.DataTableRepository,
data_table_column_repository_1.DataTableColumnRepository,
data_table_rows_repository_1.DataTableRowsRepository,
backend_common_1.Logger,
data_table_size_validator_service_1.DataTableSizeValidator,
db_1.ProjectRelationRepository,
role_service_1.RoleService,
csv_parser_service_1.CsvParserService,
data_table_file_cleanup_service_1.DataTableFileCleanupService])
], DataTableService);
//# sourceMappingURL=data-table.service.js.map