n8n
Version:
n8n Workflow Automation Tool
173 lines • 8.69 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.InsightsCompactionService = void 0;
const backend_common_1 = require("@n8n/backend-common");
const constants_1 = require("@n8n/constants");
const di_1 = require("@n8n/di");
const n8n_workflow_1 = require("n8n-workflow");
const insights_by_period_repository_1 = require("./database/repositories/insights-by-period.repository");
const insights_raw_repository_1 = require("./database/repositories/insights-raw.repository");
const insights_config_1 = require("./insights.config");
let InsightsCompactionService = class InsightsCompactionService {
constructor(insightsByPeriodRepository, insightsRawRepository, insightsConfig, logger) {
this.insightsByPeriodRepository = insightsByPeriodRepository;
this.insightsRawRepository = insightsRawRepository;
this.insightsConfig = insightsConfig;
this.logger = logger;
this.isCompactionRunning = false;
this.logger = this.logger.scoped('insights');
}
startCompactionTimer() {
this.stopCompactionTimer();
this.compactInsightsTimer = setInterval(async () => await this.compactInsights(), this.insightsConfig.compactionIntervalMinutes * constants_1.Time.minutes.toMilliseconds);
this.logger.debug('Started compaction timer');
}
stopCompactionTimer() {
if (this.compactInsightsTimer !== undefined) {
clearInterval(this.compactInsightsTimer);
this.compactInsightsTimer = undefined;
this.logger.debug('Stopped compaction timer');
}
}
async compactInsights() {
if (this.isCompactionRunning) {
this.logger.debug('Skipping insights compaction because another compaction run is active');
return;
}
this.isCompactionRunning = true;
try {
const runState = {
startedAt: Date.now(),
batchesProcessed: 0,
rowsCompacted: 0,
};
const stoppedAfterRawToHour = await this.compactStage({
stageName: 'raw-to-hour',
beforeBatchMessage: 'Compacting raw data to hourly aggregates',
afterBatchMessage: (rowsCompacted) => `Compacted ${rowsCompacted} raw data to hourly aggregates`,
compactBatch: this.compactRawToHour.bind(this),
runState,
});
if (stoppedAfterRawToHour)
return;
const stoppedAfterHourToDay = await this.compactStage({
stageName: 'hour-to-day',
beforeBatchMessage: 'Compacting hourly data to daily aggregates',
afterBatchMessage: (rowsCompacted) => `Compacted ${rowsCompacted} hourly data to daily aggregates`,
compactBatch: this.compactHourToDay.bind(this),
runState,
});
if (stoppedAfterHourToDay)
return;
await this.compactStage({
stageName: 'day-to-week',
beforeBatchMessage: 'Compacting daily data to weekly aggregates',
afterBatchMessage: (rowsCompacted) => `Compacted ${rowsCompacted} daily data to weekly aggregates`,
compactBatch: this.compactDayToWeek.bind(this),
runState,
});
}
finally {
this.isCompactionRunning = false;
}
}
async compactStage({ stageName, beforeBatchMessage, afterBatchMessage, compactBatch, runState, }) {
let numberOfCompactedData;
do {
const stopReason = this.getCompactionRunStopReason(runState);
if (stopReason !== undefined) {
this.logCompactionRunLimitReached(stopReason, stageName, runState);
return true;
}
this.logger.debug(beforeBatchMessage);
numberOfCompactedData = await compactBatch();
this.logger.debug(afterBatchMessage(numberOfCompactedData));
runState.batchesProcessed++;
runState.rowsCompacted += numberOfCompactedData;
const stopReasonAfterBatch = this.getCompactionRunStopReason(runState);
if (stopReasonAfterBatch !== undefined) {
this.logCompactionRunLimitReached(stopReasonAfterBatch, stageName, runState);
return true;
}
await this.waitBeforeNextBatchIfFull(numberOfCompactedData);
} while (numberOfCompactedData === this.insightsConfig.compactionBatchSize);
return false;
}
getCompactionRunStopReason(runState) {
if (this.insightsConfig.compactionMaxBatchesPerRun > 0 &&
runState.batchesProcessed >= this.insightsConfig.compactionMaxBatchesPerRun) {
return 'max-batches';
}
if (this.insightsConfig.compactionMaxRuntimeSeconds > 0 &&
Date.now() - runState.startedAt >=
this.insightsConfig.compactionMaxRuntimeSeconds * constants_1.Time.seconds.toMilliseconds) {
return 'max-runtime';
}
return undefined;
}
logCompactionRunLimitReached(reason, stageName, runState) {
this.logger.warn('Stopping insights compaction because a per-run limit was reached', {
reason,
stageName,
batchesProcessed: runState.batchesProcessed,
rowsCompacted: runState.rowsCompacted,
compactionMaxBatchesPerRun: this.insightsConfig.compactionMaxBatchesPerRun,
compactionMaxRuntimeSeconds: this.insightsConfig.compactionMaxRuntimeSeconds,
});
}
async waitBeforeNextBatchIfFull(numberOfCompactedData) {
if (numberOfCompactedData !== this.insightsConfig.compactionBatchSize ||
this.insightsConfig.compactionBatchDelayMilliseconds <= 0) {
return;
}
await (0, n8n_workflow_1.sleep)(this.insightsConfig.compactionBatchDelayMilliseconds);
}
async compactRawToHour() {
const batchQuery = this.insightsRawRepository.getRawInsightsBatchQuery(this.insightsConfig.compactionBatchSize);
return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
sourceBatchQuery: batchQuery,
sourceTableName: this.insightsRawRepository.metadata.tableName,
periodUnitToCompactInto: 'hour',
});
}
async compactHourToDay() {
const batchQuery = this.insightsByPeriodRepository.getPeriodInsightsBatchQuery({
periodUnitToCompactFrom: 'hour',
compactionBatchSize: this.insightsConfig.compactionBatchSize,
maxAgeInDays: this.insightsConfig.compactionHourlyToDailyThresholdDays,
});
return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
sourceBatchQuery: batchQuery,
periodUnitToCompactInto: 'day',
});
}
async compactDayToWeek() {
const batchQuery = this.insightsByPeriodRepository.getPeriodInsightsBatchQuery({
periodUnitToCompactFrom: 'day',
compactionBatchSize: this.insightsConfig.compactionBatchSize,
maxAgeInDays: this.insightsConfig.compactionDailyToWeeklyThresholdDays,
});
return await this.insightsByPeriodRepository.compactSourceDataIntoInsightPeriod({
sourceBatchQuery: batchQuery,
periodUnitToCompactInto: 'week',
});
}
};
exports.InsightsCompactionService = InsightsCompactionService;
exports.InsightsCompactionService = InsightsCompactionService = __decorate([
(0, di_1.Service)(),
__metadata("design:paramtypes", [insights_by_period_repository_1.InsightsByPeriodRepository,
insights_raw_repository_1.InsightsRawRepository,
insights_config_1.InsightsConfig,
backend_common_1.Logger])
], InsightsCompactionService);
//# sourceMappingURL=insights-compaction.service.js.map