n8n
Version:
n8n Workflow Automation Tool
359 lines • 16.1 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.InsightsService = void 0;
const di_1 = require("@n8n/di");
const typeorm_1 = require("@n8n/typeorm");
const luxon_1 = require("luxon");
const n8n_core_1 = require("n8n-core");
const n8n_workflow_1 = require("n8n-workflow");
const shared_workflow_1 = require("../../databases/entities/shared-workflow");
const shared_workflow_repository_1 = require("../../databases/repositories/shared-workflow.repository");
const on_shutdown_1 = require("../../decorators/on-shutdown");
const insights_metadata_1 = require("../../modules/insights/database/entities/insights-metadata");
const insights_raw_1 = require("../../modules/insights/database/entities/insights-raw");
const insights_shared_1 = require("./database/entities/insights-shared");
const insights_by_period_repository_1 = require("./database/repositories/insights-by-period.repository");
const insights_raw_repository_1 = require("./database/repositories/insights-raw.repository");
const insights_config_1 = require("./insights.config");
const config = di_1.Container.get(insights_config_1.InsightsConfig);
const shouldSkipStatus = {
success: false,
crashed: false,
error: false,
canceled: true,
new: true,
running: true,
unknown: true,
waiting: true,
};
const shouldSkipMode = {
cli: false,
error: false,
retry: false,
trigger: false,
webhook: false,
evaluation: false,
integrated: true,
internal: true,
manual: true,
};
let InsightsService = class InsightsService {
constructor(sharedWorkflowRepository, insightsByPeriodRepository, insightsRawRepository, logger) {
this.sharedWorkflowRepository = sharedWorkflowRepository;
this.insightsByPeriodRepository = insightsByPeriodRepository;
this.insightsRawRepository = insightsRawRepository;
this.logger = logger;
this.cachedMetadata = new Map();
this.bufferedInsights = new Set();
this.isAsynchronouslySavingInsights = true;
this.flushesInProgress = new Set();
this.logger = this.logger.scoped('insights');
}
startBackgroundProcess() {
this.startCompactionScheduler();
this.startFlushingScheduler();
}
stopBackgroundProcess() {
this.stopCompactionScheduler();
this.stopFlushingScheduler();
}
startCompactionScheduler() {
this.stopCompactionScheduler();
const intervalMilliseconds = config.compactionIntervalMinutes * 60 * 1000;
this.compactInsightsTimer = setInterval(async () => await this.compactInsights(), intervalMilliseconds);
}
stopCompactionScheduler() {
if (this.compactInsightsTimer !== undefined) {
clearInterval(this.compactInsightsTimer);
this.compactInsightsTimer = undefined;
}
}
startFlushingScheduler() {
this.isAsynchronouslySavingInsights = true;
this.stopFlushingScheduler();
this.flushInsightsRawBufferTimer = setTimeout(async () => await this.flushEvents(), config.flushIntervalSeconds * 1000);
}
stopFlushingScheduler() {
if (this.flushInsightsRawBufferTimer !== undefined) {
clearTimeout(this.flushInsightsRawBufferTimer);
this.flushInsightsRawBufferTimer = undefined;
}
}
async shutdown() {
this.stopCompactionScheduler();
this.stopFlushingScheduler();
this.isAsynchronouslySavingInsights = false;
await Promise.all([...this.flushesInProgress, this.flushEvents()]);
}
async workflowExecuteAfterHandler(ctx, fullRunData) {
if (shouldSkipStatus[fullRunData.status] || shouldSkipMode[fullRunData.mode]) {
return;
}
const status = fullRunData.status === 'success' ? 'success' : 'failure';
const commonWorkflowData = {
workflowId: ctx.workflowData.id,
workflowName: ctx.workflowData.name,
timestamp: luxon_1.DateTime.utc().toJSDate(),
};
this.bufferedInsights.add({
...commonWorkflowData,
type: status,
value: 1,
});
if (fullRunData.stoppedAt) {
const value = fullRunData.stoppedAt.getTime() - fullRunData.startedAt.getTime();
this.bufferedInsights.add({
...commonWorkflowData,
type: 'runtime_ms',
value,
});
}
if (status === 'success' && ctx.workflowData.settings?.timeSavedPerExecution) {
this.bufferedInsights.add({
...commonWorkflowData,
type: 'time_saved_min',
value: ctx.workflowData.settings.timeSavedPerExecution,
});
}
if (!this.isAsynchronouslySavingInsights) {
await this.flushEvents();
}
if (this.bufferedInsights.size >= config.flushBatchSize) {
void this.flushEvents();
}
}
async saveInsightsMetadataAndRaw(insightsRawToInsertBuffer) {
const workflowIdNames = new Map();
for (const event of insightsRawToInsertBuffer) {
workflowIdNames.set(event.workflowId, event.workflowName);
}
await this.sharedWorkflowRepository.manager.transaction(async (trx) => {
const sharedWorkflows = await trx.find(shared_workflow_1.SharedWorkflow, {
where: { workflowId: (0, typeorm_1.In)([...workflowIdNames.keys()]), role: 'workflow:owner' },
relations: { project: true },
});
const metadataToUpsert = sharedWorkflows.reduce((acc, workflow) => {
const cachedMetadata = this.cachedMetadata.get(workflow.workflowId);
if (!cachedMetadata ||
cachedMetadata.projectId !== workflow.projectId ||
cachedMetadata.projectName !== workflow.project.name ||
cachedMetadata.workflowName !== workflowIdNames.get(workflow.workflowId)) {
const metadata = new insights_metadata_1.InsightsMetadata();
metadata.projectId = workflow.projectId;
metadata.projectName = workflow.project.name;
metadata.workflowId = workflow.workflowId;
metadata.workflowName = workflowIdNames.get(workflow.workflowId);
acc.push(metadata);
}
return acc;
}, []);
await trx.upsert(insights_metadata_1.InsightsMetadata, metadataToUpsert, ['workflowId']);
const upsertMetadata = await trx.findBy(insights_metadata_1.InsightsMetadata, {
workflowId: (0, typeorm_1.In)(metadataToUpsert.map((m) => m.workflowId)),
});
for (const metadata of upsertMetadata) {
this.cachedMetadata.set(metadata.workflowId, metadata);
}
const events = [];
for (const event of insightsRawToInsertBuffer) {
const insight = new insights_raw_1.InsightsRaw();
const metadata = this.cachedMetadata.get(event.workflowId);
if (!metadata) {
throw new n8n_workflow_1.UnexpectedError(`Could not find shared workflow for insight with workflowId ${event.workflowId}`);
}
insight.metaId = metadata.metaId;
insight.type = event.type;
insight.value = event.value;
insight.timestamp = event.timestamp;
events.push(insight);
}
await trx.insert(insights_raw_1.InsightsRaw, events);
});
}
async flushEvents() {
if (this.bufferedInsights.size === 0) {
this.startFlushingScheduler();
return;
}
this.stopFlushingScheduler();
const bufferedInsightsToFlush = new Set(this.bufferedInsights);
this.bufferedInsights.clear();
let flushPromise = undefined;
flushPromise = (async () => {
try {
await this.saveInsightsMetadataAndRaw(bufferedInsightsToFlush);
}
catch (e) {
this.logger.error('Error while saving insights metadata and raw data', { error: e });
for (const event of bufferedInsightsToFlush) {
this.bufferedInsights.add(event);
}
}
finally {
this.startFlushingScheduler();
this.flushesInProgress.delete(flushPromise);
}
})();
this.flushesInProgress.add(flushPromise);
await flushPromise;
}
async compactInsights() {
let numberOfCompactedRawData;
do {
numberOfCompactedRawData = await this.compactRawToHour();
} while (numberOfCompactedRawData > 0);
let numberOfCompactedHourData;
do {
numberOfCompactedHourData = await this.compactHourToDay();
} while (numberOfCompactedHourData > 0);
let numberOfCompactedDayData;
do {
numberOfCompactedDayData = await this.compactDayToWeek();
} while (numberOfCompactedDayData > 0);
}
async compactRawToHour() {
const batchQuery = this.insightsRawRepository.getRawInsightsBatchQuery(config.compactionBatchSize);
return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
sourceBatchQuery: batchQuery,
sourceTableName: this.insightsRawRepository.metadata.tableName,
periodUnitToCompactInto: 'hour',
});
}
async compactHourToDay() {
const batchQuery = this.insightsByPeriodRepository.getPeriodInsightsBatchQuery({
periodUnitToCompactFrom: 'hour',
compactionBatchSize: config.compactionBatchSize,
maxAgeInDays: config.compactionHourlyToDailyThresholdDays,
});
return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
sourceBatchQuery: batchQuery,
periodUnitToCompactInto: 'day',
});
}
async compactDayToWeek() {
const batchQuery = this.insightsByPeriodRepository.getPeriodInsightsBatchQuery({
periodUnitToCompactFrom: 'day',
compactionBatchSize: config.compactionBatchSize,
maxAgeInDays: config.compactionDailyToWeeklyThresholdDays,
});
return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
sourceBatchQuery: batchQuery,
periodUnitToCompactInto: 'week',
});
}
async getInsightsSummary({ periodLengthInDays, }) {
const rows = await this.insightsByPeriodRepository.getPreviousAndCurrentPeriodTypeAggregates({
periodLengthInDays,
});
const data = {
current: { byType: {} },
previous: { byType: {} },
};
rows.forEach((row) => {
const { period, type, total_value } = row;
if (!data[period])
return;
data[period].byType[insights_shared_1.NumberToType[type]] = total_value ? Number(total_value) : 0;
});
const getValueByType = (period, type) => data[period]?.byType[type] ?? 0;
const currentSuccesses = getValueByType('current', 'success');
const currentFailures = getValueByType('current', 'failure');
const previousSuccesses = getValueByType('previous', 'success');
const previousFailures = getValueByType('previous', 'failure');
const currentTotal = currentSuccesses + currentFailures;
const previousTotal = previousSuccesses + previousFailures;
const currentFailureRate = currentTotal > 0 ? Math.round((currentFailures / currentTotal) * 1000) / 1000 : 0;
const previousFailureRate = previousTotal > 0 ? Math.round((previousFailures / previousTotal) * 1000) / 1000 : 0;
const currentTotalRuntime = getValueByType('current', 'runtime_ms') ?? 0;
const previousTotalRuntime = getValueByType('previous', 'runtime_ms') ?? 0;
const currentAvgRuntime = currentTotal > 0 ? Math.round((currentTotalRuntime / currentTotal) * 100) / 100 : 0;
const previousAvgRuntime = previousTotal > 0 ? Math.round((previousTotalRuntime / previousTotal) * 100) / 100 : 0;
const currentTimeSaved = getValueByType('current', 'time_saved_min');
const previousTimeSaved = getValueByType('previous', 'time_saved_min');
const getDeviation = (current, previous) => previousTotal === 0 ? null : current - previous;
const result = {
averageRunTime: {
value: currentAvgRuntime,
unit: 'millisecond',
deviation: getDeviation(currentAvgRuntime, previousAvgRuntime),
},
failed: {
value: currentFailures,
unit: 'count',
deviation: getDeviation(currentFailures, previousFailures),
},
failureRate: {
value: currentFailureRate,
unit: 'ratio',
deviation: getDeviation(currentFailureRate, previousFailureRate),
},
timeSaved: {
value: currentTimeSaved,
unit: 'minute',
deviation: getDeviation(currentTimeSaved, previousTimeSaved),
},
total: {
value: currentTotal,
unit: 'count',
deviation: getDeviation(currentTotal, previousTotal),
},
};
return result;
}
async getInsightsByWorkflow({ maxAgeInDays, skip = 0, take = 10, sortBy = 'total:desc', }) {
const { count, rows } = await this.insightsByPeriodRepository.getInsightsByWorkflow({
maxAgeInDays,
skip,
take,
sortBy,
});
return {
count,
data: rows,
};
}
async getInsightsByTime({ maxAgeInDays, periodUnit, }) {
const rows = await this.insightsByPeriodRepository.getInsightsByTime({
maxAgeInDays,
periodUnit,
});
return rows.map((r) => {
const total = r.succeeded + r.failed;
return {
date: r.periodStart,
values: {
total,
succeeded: r.succeeded,
failed: r.failed,
failureRate: r.failed / total,
averageRunTime: r.runTime / total,
timeSaved: r.timeSaved,
},
};
});
}
};
exports.InsightsService = InsightsService;
__decorate([
(0, on_shutdown_1.OnShutdown)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], InsightsService.prototype, "shutdown", null);
exports.InsightsService = InsightsService = __decorate([
(0, di_1.Service)(),
__metadata("design:paramtypes", [shared_workflow_repository_1.SharedWorkflowRepository,
insights_by_period_repository_1.InsightsByPeriodRepository,
insights_raw_repository_1.InsightsRawRepository,
n8n_core_1.Logger])
], InsightsService);
//# sourceMappingURL=insights.service.js.map