n8n
Version:
n8n Workflow Automation Tool
661 lines • 30.2 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var ExecutionPersistence_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ExecutionPersistence = void 0;
const backend_common_1 = require("@n8n/backend-common");
const config_1 = require("@n8n/config");
const constants_1 = require("@n8n/constants");
const db_1 = require("@n8n/db");
const di_1 = require("@n8n/di");
const flatted_1 = require("flatted");
const n8n_core_1 = require("n8n-core");
const n8n_workflow_1 = require("n8n-workflow");
const corrupted_execution_data_error_1 = require("./execution-data/corrupted-execution-data.error");
const db_store_1 = require("./execution-data/db-store");
const execution_data_json_store_1 = require("./execution-data/execution-data-json-store");
const missing_execution_data_error_1 = require("./execution-data/missing-execution-data.error");
const sum_binary_data_bytes_1 = require("./sum-binary-data-bytes");
const duplicate_execution_error_1 = require("../errors/duplicate-execution.error");
const event_service_1 = require("../events/event.service");
let ExecutionPersistence = ExecutionPersistence_1 = class ExecutionPersistence {
constructor(executionRepository, binaryDataService, jsonStore, dbStore, storageConfig, executionsConfig, databaseConfig, errorReporter, eventService) {
this.executionRepository = executionRepository;
this.binaryDataService = binaryDataService;
this.jsonStore = jsonStore;
this.dbStore = dbStore;
this.storageConfig = storageConfig;
this.executionsConfig = executionsConfig;
this.databaseConfig = databaseConfig;
this.errorReporter = errorReporter;
this.eventService = eventService;
}
async create(payload) {
const { data: rawData, workflowData, ...rest } = payload;
const { connections, nodes, name, settings, id, nodeGroups } = workflowData;
const workflowSnapshot = {
connections,
nodes,
name,
settings,
id,
nodeGroups,
};
const storedAt = this.storageConfig.modeTag;
const workflowVersionId = workflowData.versionId ?? null;
const executionEntity = { ...rest, createdAt: new Date(), storedAt, workflowVersionId };
let reclaimedTombstone = null;
try {
const executionId = await this.executionRepository.manager.transaction(async (tx) => {
reclaimedTombstone = await this.reclaimTombstone(tx, executionEntity.deduplicationKey);
const { identifiers } = await tx.insert(db_1.ExecutionEntity, executionEntity);
const executionId = String(identifiers[0].id);
const ref = { workflowId: id, executionId };
const jsonSizeBytes = await this.trackWrite(storedAt, ref.workflowId, async () => {
const bundle = {
data: (0, flatted_1.stringify)(rawData),
workflowData: workflowSnapshot,
workflowVersionId,
};
return await this.writeData(storedAt, ref, bundle, tx);
});
const binaryDataSizeBytes = (0, sum_binary_data_bytes_1.sumBinaryDataBytes)(rawData);
await tx.update(db_1.ExecutionEntity, { id: executionId }, { jsonSizeBytes, binaryDataSizeBytes });
return executionId;
});
await this.deleteReclaimedTombstoneData(reclaimedTombstone);
return executionId;
}
catch (error) {
if (executionEntity.deduplicationKey && this.isDuplicateExecutionError(error)) {
throw new duplicate_execution_error_1.DuplicateExecutionError(executionEntity.deduplicationKey, error);
}
throw error;
}
}
async reclaimTombstone(tx, deduplicationKey) {
if (!deduplicationKey)
return null;
const tombstone = await tx.findOne(db_1.ExecutionEntity, {
where: { deduplicationKey, status: 'new' },
select: ['id', 'workflowId', 'storedAt'],
});
if (!tombstone)
return null;
const { affected } = await tx.delete(db_1.ExecutionEntity, { id: tombstone.id, status: 'new' });
if (!affected)
return null;
return {
workflowId: tombstone.workflowId,
executionId: tombstone.id,
storedAt: tombstone.storedAt,
};
}
async deleteReclaimedTombstoneData(target) {
if (!target)
return;
const blobRefs = this.toBlobRefs([target]);
if (blobRefs.length === 0)
return;
try {
await this.jsonStore.delete(blobRefs);
}
catch (error) {
this.errorReporter.error(error, {
extra: { executionId: target.executionId, storedAt: target.storedAt },
});
}
}
async updateExistingExecution(executionId, execution, conditions) {
const hasDataField = execution.data !== undefined || execution.workflowData !== undefined;
if (!hasDataField) {
return await this.updateEntityOnly(executionId, execution, conditions);
}
const entity = await this.executionRepository.findOne({
where: this.buildEntityWhereCondition(executionId, conditions),
select: ['id', 'workflowId', 'storedAt', 'workflowVersionId'],
});
if (!entity)
return false;
const ref = { workflowId: entity.workflowId, executionId };
return await this.applyDataUpdate(ref, entity.storedAt, entity.workflowVersionId, execution, conditions);
}
async findSingleExecution(id, options) {
if (!options?.includeData) {
return await this.executionRepository.findSingleExecution(id, options);
}
const entity = await this.executionRepository.findOne({
where: { id, ...options.where },
relations: {
metadata: true,
...(options.includeAnnotation ? { annotation: { tags: true } } : {}),
},
});
if (!entity)
return undefined;
const max = this.maxDisplayDataSize(options);
const ref = { workflowId: entity.workflowId, executionId: entity.id };
if (this.isKnownOversize(entity, max)) {
return (await this.assembleSkippedExecution(entity, ref, options));
}
if (max > 0 && entity.jsonSizeBytes === 0 && entity.storedAt === 'db') {
const size = await this.dbStore.getDataByteSize(ref);
if (size !== null && size > max) {
return (await this.assembleSkippedExecution(entity, ref, options));
}
}
const start = Date.now();
let success = false;
let unreadableBundles = 0;
try {
const bundle = await this.readData(entity.storedAt, ref);
if (!bundle) {
unreadableBundles = 1;
if (entity.storedAt === 'db') {
this.executionRepository.reportInvalidExecutions([entity]);
return undefined;
}
throw new missing_execution_data_error_1.MissingExecutionDataError(ref);
}
const assembled = await this.assembleReadExecution(entity, bundle, options, max);
success = true;
return assembled;
}
catch (error) {
if (error instanceof corrupted_execution_data_error_1.CorruptedExecutionDataError)
unreadableBundles = 1;
throw error;
}
finally {
this.eventService.emit('execution-data-read', {
mode: entity.storedAt,
durationMs: Date.now() - start,
success,
unreadableBundles,
});
}
}
async findMultipleExecutions(queryParams, options) {
if (!options?.includeData) {
return await this.executionRepository.findMultipleExecutions(queryParams, options);
}
queryParams.relations ??= [];
if (Array.isArray(queryParams.relations)) {
if (!queryParams.relations.includes('metadata'))
queryParams.relations.push('metadata');
}
else {
queryParams.relations.metadata = true;
}
const max = this.maxDisplayDataSize(options);
if (queryParams.select) {
const guardFields = max > 0 ? ['jsonSizeBytes', 'workflowVersionId'] : [];
if (Array.isArray(queryParams.select)) {
for (const field of ['id', 'workflowId', 'storedAt', ...guardFields]) {
if (!queryParams.select.includes(field))
queryParams.select.push(field);
}
}
else {
queryParams.select.id = true;
queryParams.select.workflowId = true;
queryParams.select.storedAt = true;
for (const field of guardFields)
queryParams.select[field] = true;
}
}
const entities = await this.executionRepository.find(queryParams);
if (entities.length === 0)
return [];
const assembledById = new Map();
const entitiesToRead = await this.skipOversizedEntities(entities, max, assembledById);
const entitiesByLocation = new Map();
for (const entity of entitiesToRead) {
const group = entitiesByLocation.get(entity.storedAt) ?? [];
group.push(entity);
entitiesByLocation.set(entity.storedAt, group);
}
await Promise.all([...entitiesByLocation].map(async ([location, group]) => {
const refs = group.map((e) => ({ workflowId: e.workflowId, executionId: e.id }));
const start = Date.now();
let success = false;
let unreadableBundles = 0;
try {
const bundles = location === 'db'
? await this.dbStore.readMany(refs)
: await this.jsonStore.readMany(refs.map((ref) => ({ ...ref, storedAt: location })));
const missing = group.filter((e) => !bundles.has(e.id));
if (missing.length > 0)
this.executionRepository.reportInvalidExecutions(missing);
unreadableBundles = missing.length;
const settled = await Promise.allSettled(group.map(async (entity) => {
const bundle = bundles.get(entity.id);
if (!bundle)
return;
assembledById.set(entity.id, await this.assembleReadExecution(entity, bundle, options, max));
}));
const corrupt = group.filter((_, i) => {
const outcome = settled[i];
return (outcome.status === 'rejected' && outcome.reason instanceof corrupted_execution_data_error_1.CorruptedExecutionDataError);
});
unreadableBundles += corrupt.length;
if (corrupt.length > 0)
this.executionRepository.reportInvalidExecutions(corrupt);
for (const outcome of settled) {
if (outcome.status === 'rejected' &&
!(outcome.reason instanceof corrupted_execution_data_error_1.CorruptedExecutionDataError)) {
throw outcome.reason;
}
}
success = true;
}
finally {
this.eventService.emit('execution-data-read', {
mode: location,
durationMs: Date.now() - start,
success,
unreadableBundles,
});
}
}));
return entities
.map((e) => assembledById.get(e.id))
.filter((e) => e !== undefined);
}
async findWithUnflattenedData(executionId, accessibleWorkflowIds) {
return await this.findSingleExecution(executionId, {
where: { workflowId: (0, db_1.In)(accessibleWorkflowIds) },
includeData: true,
unflattenData: true,
includeAnnotation: true,
});
}
async findIfSharedUnflatten(executionId, sharedWorkflowIds, maxDataSizeBytes) {
return await this.findSingleExecution(executionId, {
where: { workflowId: (0, db_1.In)(sharedWorkflowIds) },
includeData: true,
unflattenData: true,
includeAnnotation: true,
maxDataSizeBytes,
});
}
async getExecutionInWorkflowsForPublicApi(id, workflowIds, includeData, maxDataSizeBytes) {
return await this.findSingleExecution(id, {
where: { workflowId: (0, db_1.In)(workflowIds) },
includeData,
unflattenData: true,
maxDataSizeBytes,
});
}
async getExecutionsForPublicApi(params, maxDataSizeBytes) {
return await this.findMultipleExecutions({
select: [
'id',
'mode',
'retryOf',
'retrySuccessId',
'startedAt',
'stoppedAt',
'workflowId',
'waitTill',
'finished',
'status',
],
where: this.executionRepository.getFindExecutionsForPublicApiCondition(params),
order: { id: 'DESC' },
take: params.limit,
}, { includeData: params.includeData, unflattenData: true, maxDataSizeBytes });
}
async deleteInFlightExecution(target) {
if (this.executionsConfig.pruneData) {
const bufferMs = this.executionsConfig.pruneDataHardDeleteBuffer * constants_1.Time.hours.toMilliseconds;
const deletedAt = new Date(Date.now() - bufferMs);
await this.executionRepository.update(target.executionId, { deletedAt });
}
else {
await this.hardDelete(target);
}
}
async hardDelete(target) {
const targets = Array.isArray(target) ? target : [target];
if (targets.length === 0)
return;
await Promise.all([
this.executionRepository.deleteByIds(targets.map((t) => t.executionId)),
this.binaryDataService.deleteMany(targets.map((t) => ({ type: 'execution', ...t }))),
this.jsonStore.delete(this.toBlobRefs(targets)),
]);
}
async hardDeleteBy(criteria) {
const refs = await this.executionRepository.deleteExecutionsByFilter(criteria);
await this.jsonStore.delete(this.toBlobRefs(refs));
}
async hardDeleteByWorkflowId(workflowId) {
const maxBatches = await this.maxBulkDeletionBatches();
for (let batch = 0; batch < maxBatches; batch++) {
const executions = await this.executionRepository.find({
select: ['id', 'workflowId', 'storedAt'],
where: { workflowId },
take: ExecutionPersistence_1.bulkDeletionBatchSize,
withDeleted: true,
});
if (executions.length === 0)
return;
await this.hardDelete(executions.map((execution) => ({
executionId: execution.id,
workflowId: execution.workflowId,
storedAt: execution.storedAt,
})));
}
const remaining = await this.executionRepository.find({
select: ['id'],
where: { workflowId },
take: 1,
withDeleted: true,
});
if (remaining.length === 0)
return;
throw new n8n_workflow_1.UnexpectedError(`Failed to delete all executions of workflow ${workflowId}: executions keep being added while deleting them - is the workflow still active?`);
}
async maxBulkDeletionBatches() {
const { bulkDeletionBatchSize, maxBulkDeletionBatchesPerRun: fallback } = ExecutionPersistence_1;
const estimate = await this.estimateExecutionsTableSize();
if (estimate === null)
return fallback;
return Math.max(Math.ceil((estimate * 2) / bulkDeletionBatchSize), fallback);
}
async estimateExecutionsTableSize() {
if (this.databaseConfig.type !== 'postgresdb')
return null;
try {
const { schema, tableName } = this.executionRepository.metadata;
const table = schema ? `"${schema}"."${tableName}"` : `"${tableName}"`;
const rows = (await this.executionRepository.query('SELECT reltuples::bigint AS estimate FROM pg_class WHERE oid = to_regclass($1)', [table]));
const estimate = Number(rows[0]?.estimate);
return Number.isFinite(estimate) && estimate >= 0 ? estimate : null;
}
catch {
return null;
}
}
toBlobRefs(targets) {
return targets.filter((t) => t.storedAt !== 'db');
}
async updateEntityOnly(executionId, execution, conditions) {
const updatableColumns = this.pickUpdatableEntityColumns(execution);
if (Object.keys(updatableColumns).length === 0)
return true;
const whereCondition = this.buildEntityWhereCondition(executionId, conditions);
const result = await this.executionRepository.update(whereCondition, updatableColumns);
return (result.affected ?? 0) > 0;
}
async applyDataUpdate(ref, mode, workflowVersionId, execution, conditions) {
const { data, workflowData } = execution;
const updatableColumns = this.pickUpdatableEntityColumns(execution);
return await this.executionRepository.manager.transaction(async (tx) => {
const whereCondition = this.buildEntityWhereCondition(ref.executionId, conditions);
if (Object.keys(updatableColumns).length > 0) {
const result = await tx.update(db_1.ExecutionEntity, whereCondition, updatableColumns);
if ((result.affected ?? 0) === 0)
return false;
}
else if (conditions) {
const lock = this.databaseConfig.type === 'postgresdb'
? { mode: 'pessimistic_write' }
: undefined;
const matchingRow = await tx.findOne(db_1.ExecutionEntity, {
where: whereCondition,
select: ['id'],
lock,
});
if (!matchingRow)
return false;
}
if (data !== undefined &&
workflowData !== undefined &&
(workflowVersionId !== null || mode === 'db')) {
const binaryDataSizeBytes = (0, sum_binary_data_bytes_1.sumBinaryDataBytes)(data);
const jsonSizeBytes = await this.trackWrite(mode, ref.workflowId, async () => {
const bundle = {
data: (0, flatted_1.stringify)(data),
workflowData: this.toWorkflowSnapshot(workflowData),
workflowVersionId,
};
return mode === 'db'
? await this.dbStore.overwrite(ref, bundle, tx)
: await this.jsonStore.write(ref, bundle, mode);
});
await tx.update(db_1.ExecutionEntity, { id: ref.executionId }, { jsonSizeBytes, binaryDataSizeBytes });
return true;
}
const existing = await this.trackRead(mode, async () => await this.readData(mode, ref, tx));
if (!existing)
throw new missing_execution_data_error_1.MissingExecutionDataError(ref);
const jsonSizeBytes = await this.trackWrite(mode, ref.workflowId, async () => {
const bundle = {
data: data !== undefined ? (0, flatted_1.stringify)(data) : existing.data,
workflowData: workflowData
? this.toWorkflowSnapshot(workflowData)
: existing.workflowData,
workflowVersionId: existing.workflowVersionId,
};
return await this.writeData(mode, ref, bundle, tx);
});
const sizeColumns = data !== undefined
? { jsonSizeBytes, binaryDataSizeBytes: (0, sum_binary_data_bytes_1.sumBinaryDataBytes)(data) }
: { jsonSizeBytes };
await tx.update(db_1.ExecutionEntity, { id: ref.executionId }, sizeColumns);
return true;
});
}
pickUpdatableEntityColumns(execution) {
const { id: _id, data: _data, workflowId: _workflowId, workflowData: _workflowData, workflowVersionId: _workflowVersionId, createdAt: _createdAt, startedAt: _startedAt, customData: _customData, jsonSizeBytes: _jsonSizeBytes, binaryDataSizeBytes: _binaryDataSizeBytes, ...updatableColumns } = execution;
return updatableColumns;
}
buildEntityWhereCondition(executionId, conditions) {
if (conditions?.requireStatus && conditions?.requireNotCanceled) {
throw new n8n_workflow_1.UnexpectedError('`requireStatus` and `requireNotCanceled` cannot be combined');
}
const where = { id: executionId };
if (conditions?.requireStatus)
where.status = conditions.requireStatus;
if (conditions?.requireNotFinished)
where.finished = false;
if (conditions?.requireNotCanceled)
where.status = (0, db_1.Not)('canceled');
return where;
}
async trackRead(mode, op) {
const start = Date.now();
let success = false;
let unreadableBundles = 0;
try {
const result = await op();
success = result !== null && result !== undefined;
if (!success)
unreadableBundles = 1;
return result;
}
catch (error) {
if (error instanceof corrupted_execution_data_error_1.CorruptedExecutionDataError)
unreadableBundles = 1;
throw error;
}
finally {
this.eventService.emit('execution-data-read', {
mode,
durationMs: Date.now() - start,
success,
unreadableBundles,
});
}
}
async trackWrite(mode, workflowId, op) {
const start = Date.now();
let success = false;
let jsonSizeBytes = 0;
try {
jsonSizeBytes = await op();
success = true;
return jsonSizeBytes;
}
finally {
this.eventService.emit('execution-data-write', {
mode,
workflowId,
durationMs: Date.now() - start,
success,
jsonSizeBytes,
});
}
}
async writeData(mode, ref, payload, tx) {
return mode === 'db'
? await this.dbStore.write(ref, payload, tx)
: await this.jsonStore.write(ref, payload, mode);
}
async readData(mode, ref, tx) {
if (mode !== 'db')
return await this.jsonStore.read(ref, mode);
return tx ? await this.dbStore.read(ref, tx) : await this.dbStore.read(ref);
}
toWorkflowSnapshot(workflowData) {
const { id, name, nodes, connections, settings, nodeGroups } = workflowData;
return { id, name, nodes, connections, settings, nodeGroups };
}
async assembleExecution(entity, bundle, options) {
const { metadata, annotation, ...rest } = entity;
const ref = { workflowId: entity.workflowId, executionId: entity.id };
const data = await this.parseExecutionData(ref, bundle.data, options);
const serializedAnnotation = this.serializeAnnotation(annotation);
if (entity.status === 'success' && bundle.data === '[]') {
this.errorReporter.error('Found successful execution where data is empty stringified array', {
extra: { executionId: entity.id, workflowId: bundle.workflowData.id },
});
}
return {
...rest,
data,
workflowData: bundle.workflowData,
workflowVersionId: bundle.workflowVersionId ?? null,
customData: Object.fromEntries(metadata.map((m) => [m.key, m.value])),
...(options.includeAnnotation && serializedAnnotation
? { annotation: serializedAnnotation }
: {}),
};
}
maxDisplayDataSize(options) {
return options.unflattenData ? (options.maxDataSizeBytes ?? 0) : 0;
}
isKnownOversize(entity, max) {
return max > 0 && entity.jsonSizeBytes > 0 && entity.jsonSizeBytes > max;
}
async assembleSkippedExecution(entity, ref, options) {
const snapshot = entity.storedAt === 'db'
? ((await this.dbStore.readWorkflowData(ref)) ?? undefined)
: undefined;
return this.assembleOversizedExecution(entity, { includeAnnotation: options.includeAnnotation }, snapshot);
}
assembleOversizedExecution(entity, options, snapshot) {
const { metadata, annotation, ...rest } = entity;
const serializedAnnotation = this.serializeAnnotation(annotation);
const workflowData = snapshot?.workflowData ?? {
id: entity.workflowId,
name: '',
nodes: [],
connections: {},
settings: {},
};
return {
...rest,
data: (0, n8n_workflow_1.createEmptyRunExecutionData)(),
workflowData,
workflowVersionId: snapshot?.workflowVersionId ?? entity.workflowVersionId ?? null,
customData: Object.fromEntries(metadata.map((m) => [m.key, m.value])),
dataTooLargeToDisplay: true,
...(options.includeAnnotation && serializedAnnotation
? { annotation: serializedAnnotation }
: {}),
};
}
async skipOversizedEntities(entities, max, assembledById) {
if (max <= 0)
return entities;
const entitiesToRead = [];
const oversized = [];
for (const entity of entities) {
if (this.isKnownOversize(entity, max))
oversized.push(entity);
else
entitiesToRead.push(entity);
}
await Promise.all(oversized.map(async (entity) => {
const ref = { workflowId: entity.workflowId, executionId: entity.id };
assembledById.set(entity.id, await this.assembleSkippedExecution(entity, ref, {}));
}));
return entitiesToRead;
}
async assembleReadExecution(entity, bundle, options, max) {
if (max > 0 && entity.jsonSizeBytes === 0 && Buffer.byteLength(bundle.data, 'utf8') > max) {
return this.assembleOversizedExecution(entity, { includeAnnotation: options.includeAnnotation }, { workflowData: bundle.workflowData, workflowVersionId: bundle.workflowVersionId });
}
return await this.assembleExecution(entity, bundle, options);
}
async parseExecutionData(ref, data, options) {
if (!options.unflattenData)
return data;
try {
const deserialized = await (0, backend_common_1.parseFlatted)(data);
if (!deserialized)
return undefined;
return (0, n8n_workflow_1.migrateRunExecutionData)(deserialized);
}
catch (error) {
throw new corrupted_execution_data_error_1.CorruptedExecutionDataError(ref, error);
}
}
serializeAnnotation(annotation) {
if (!annotation)
return null;
const { id, vote, tags } = annotation;
return {
id,
vote,
tags: tags?.map(({ id, name }) => ({ id, name })) ?? [],
};
}
isDuplicateExecutionError(error) {
if (!(error instanceof Error) || !('driverError' in error))
return false;
const { driverError } = error;
if (typeof driverError !== 'object' || driverError === null || !('code' in driverError)) {
return false;
}
const { code } = driverError;
if (typeof code !== 'string')
return false;
if (!error.message.includes('deduplicationKey'))
return false;
if (this.databaseConfig.type === 'postgresdb') {
return code === '23505';
}
return (code === 'SQLITE_CONSTRAINT_UNIQUE' ||
(code === 'SQLITE_CONSTRAINT' && error.message.includes('UNIQUE constraint failed')));
}
};
exports.ExecutionPersistence = ExecutionPersistence;
ExecutionPersistence.bulkDeletionBatchSize = 500;
ExecutionPersistence.maxBulkDeletionBatchesPerRun = 20_000;
exports.ExecutionPersistence = ExecutionPersistence = ExecutionPersistence_1 = __decorate([
(0, di_1.Service)(),
__metadata("design:paramtypes", [db_1.ExecutionRepository, n8n_core_1.BinaryDataService, execution_data_json_store_1.ExecutionDataJsonStore, db_store_1.DbStore, n8n_core_1.StorageConfig, config_1.ExecutionsConfig, config_1.DatabaseConfig, n8n_core_1.ErrorReporter, event_service_1.EventService])
], ExecutionPersistence);
//# sourceMappingURL=execution-persistence.js.map