n8n
Version:
n8n Workflow Automation Tool
273 lines • 13.1 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ScalingService = void 0;
const typedi_1 = __importStar(require("typedi"));
const n8n_workflow_1 = require("n8n-workflow");
const ActiveExecutions_1 = require("../ActiveExecutions");
const config_1 = __importDefault(require("../config"));
const Logger_1 = require("../Logger");
const max_stalled_count_error_1 = require("../errors/max-stalled-count.error");
const constants_1 = require("../constants");
const OnShutdown_1 = require("../decorators/OnShutdown");
const constants_2 = require("./constants");
const job_processor_1 = require("./job-processor");
const config_2 = require("@n8n/config");
const execution_repository_1 = require("../databases/repositories/execution.repository");
const n8n_core_1 = require("n8n-core");
const orchestration_service_1 = require("../services/orchestration.service");
let ScalingService = class ScalingService {
constructor(logger, activeExecutions, jobProcessor, globalConfig, executionRepository, instanceSettings, orchestrationService) {
this.logger = logger;
this.activeExecutions = activeExecutions;
this.jobProcessor = jobProcessor;
this.globalConfig = globalConfig;
this.executionRepository = executionRepository;
this.instanceSettings = instanceSettings;
this.orchestrationService = orchestrationService;
this.instanceType = config_1.default.getEnv('generic.instanceType');
this.queueRecoveryContext = {
batchSize: config_1.default.getEnv('executions.queueRecovery.batchSize'),
waitMs: config_1.default.getEnv('executions.queueRecovery.interval') * 60 * 1000,
};
}
async setupQueue() {
const { default: BullQueue } = await Promise.resolve().then(() => __importStar(require('bull')));
const { RedisClientService } = await Promise.resolve().then(() => __importStar(require('../services/redis/redis-client.service')));
const service = typedi_1.default.get(RedisClientService);
const bullPrefix = this.globalConfig.queue.bull.prefix;
const prefix = service.toValidPrefix(bullPrefix);
this.queue = new BullQueue(constants_2.QUEUE_NAME, {
prefix,
settings: this.globalConfig.queue.bull.settings,
createClient: (type) => service.createClient({ type: `${type}(bull)` }),
});
this.registerListeners();
if (this.instanceSettings.isLeader)
this.scheduleQueueRecovery();
if (this.orchestrationService.isMultiMainSetupEnabled) {
this.orchestrationService.multiMainSetup
.on('leader-takeover', () => this.scheduleQueueRecovery())
.on('leader-stepdown', () => this.stopQueueRecovery());
}
this.logger.debug('[ScalingService] Queue setup completed');
}
setupWorker(concurrency) {
this.assertWorker();
void this.queue.process(constants_2.JOB_TYPE_NAME, concurrency, async (job) => await this.jobProcessor.processJob(job));
this.logger.debug('[ScalingService] Worker setup completed');
}
async stop() {
await this.queue.pause(true, true);
this.logger.debug('[ScalingService] Queue paused');
this.stopQueueRecovery();
this.logger.debug('[ScalingService] Queue recovery stopped');
let count = 0;
while (this.getRunningJobsCount() !== 0) {
if (count++ % 4 === 0) {
this.logger.info(`Waiting for ${this.getRunningJobsCount()} active executions to finish...`);
}
await (0, n8n_workflow_1.sleep)(500);
}
}
async pingQueue() {
await this.queue.client.ping();
}
async addJob(jobData, jobOptions) {
const { executionId } = jobData;
const job = await this.queue.add(constants_2.JOB_TYPE_NAME, jobData, jobOptions);
this.logger.info(`[ScalingService] Added job ${job.id} (execution ${executionId})`);
return job;
}
async getJob(jobId) {
return await this.queue.getJob(jobId);
}
async findJobsByStatus(statuses) {
const jobs = await this.queue.getJobs(statuses);
return jobs.filter((job) => job !== null);
}
async stopJob(job) {
const props = { jobId: job.id, executionId: job.data.executionId };
try {
if (await job.isActive()) {
await job.progress({ kind: 'abort-job' });
this.logger.debug('[ScalingService] Stopped active job', props);
return true;
}
await job.remove();
this.logger.debug('[ScalingService] Stopped inactive job', props);
return true;
}
catch (error) {
await job.progress({ kind: 'abort-job' });
this.logger.error('[ScalingService] Failed to stop job', { ...props, error });
return false;
}
}
getRunningJobsCount() {
return this.jobProcessor.getRunningJobIds().length;
}
registerListeners() {
this.queue.on('global:progress', (_jobId, msg) => {
if (msg.kind === 'respond-to-webhook') {
const { executionId, response } = msg;
this.activeExecutions.resolveResponsePromise(executionId, this.decodeWebhookResponse(response));
}
});
this.queue.on('global:progress', (jobId, msg) => {
if (msg.kind === 'abort-job') {
this.jobProcessor.stopJob(jobId);
}
});
let latestAttemptTs = 0;
let cumulativeTimeoutMs = 0;
const MAX_TIMEOUT_MS = this.globalConfig.queue.bull.redis.timeoutThreshold;
const RESET_LENGTH_MS = 30000;
this.queue.on('error', (error) => {
this.logger.error('[ScalingService] Queue errored', { error });
if (error.message.includes('ECONNREFUSED')) {
const nowTs = Date.now();
if (nowTs - latestAttemptTs > RESET_LENGTH_MS) {
latestAttemptTs = nowTs;
cumulativeTimeoutMs = 0;
}
else {
cumulativeTimeoutMs += nowTs - latestAttemptTs;
latestAttemptTs = nowTs;
if (cumulativeTimeoutMs > MAX_TIMEOUT_MS) {
this.logger.error('[ScalingService] Redis unavailable after max timeout');
this.logger.error('[ScalingService] Exiting process...');
process.exit(1);
}
}
this.logger.warn('[ScalingService] Redis unavailable - retrying to connect...');
return;
}
if (this.instanceType === 'worker' &&
error.message.includes('job stalled more than maxStalledCount')) {
throw new max_stalled_count_error_1.MaxStalledCountError(error);
}
if (this.instanceType === 'worker' &&
error.message.includes('Error initializing Lua scripts')) {
this.logger.error('[ScalingService] Fatal error initializing worker', { error });
this.logger.error('[ScalingService] Exiting process...');
process.exit(1);
}
throw error;
});
}
decodeWebhookResponse(response) {
if (typeof response === 'object' &&
typeof response.body === 'object' &&
response.body !== null &&
'__@N8nEncodedBuffer@__' in response.body &&
typeof response.body['__@N8nEncodedBuffer@__'] === 'string') {
response.body = Buffer.from(response.body['__@N8nEncodedBuffer@__'], n8n_workflow_1.BINARY_ENCODING);
}
return response;
}
assertWorker() {
if (this.instanceType === 'worker')
return;
throw new n8n_workflow_1.ApplicationError('This method must be called on a `worker` instance');
}
scheduleQueueRecovery(waitMs = this.queueRecoveryContext.waitMs) {
this.queueRecoveryContext.timeout = setTimeout(async () => {
try {
const nextWaitMs = await this.recoverFromQueue();
this.scheduleQueueRecovery(nextWaitMs);
}
catch (error) {
this.logger.error('[ScalingService] Failed to recover dangling executions from queue', {
msg: this.toErrorMsg(error),
});
this.logger.error('[ScalingService] Retrying...');
this.scheduleQueueRecovery();
}
}, waitMs);
const wait = [this.queueRecoveryContext.waitMs / constants_1.Time.minutes.toMilliseconds, 'min'].join(' ');
this.logger.debug(`[ScalingService] Scheduled queue recovery check for next ${wait}`);
}
stopQueueRecovery() {
clearTimeout(this.queueRecoveryContext.timeout);
}
async recoverFromQueue() {
const { waitMs, batchSize } = this.queueRecoveryContext;
const storedIds = await this.executionRepository.getInProgressExecutionIds(batchSize);
if (storedIds.length === 0) {
this.logger.debug('[ScalingService] Completed queue recovery check, no dangling executions');
return waitMs;
}
const runningJobs = await this.findJobsByStatus(['active', 'waiting']);
const queuedIds = new Set(runningJobs.map((job) => job.data.executionId));
if (queuedIds.size === 0) {
this.logger.debug('[ScalingService] Completed queue recovery check, no dangling executions');
return waitMs;
}
const danglingIds = storedIds.filter((id) => !queuedIds.has(id));
if (danglingIds.length === 0) {
this.logger.debug('[ScalingService] Completed queue recovery check, no dangling executions');
return waitMs;
}
await this.executionRepository.markAsCrashed(danglingIds);
this.logger.info('[ScalingService] Completed queue recovery check, recovered dangling executions', { danglingIds });
return storedIds.length >= this.queueRecoveryContext.batchSize ? waitMs / 2 : waitMs;
}
toErrorMsg(error) {
return error instanceof Error
? error.message
: (0, n8n_workflow_1.jsonStringify)(error, { replaceCircularRefs: true });
}
};
exports.ScalingService = ScalingService;
__decorate([
(0, OnShutdown_1.OnShutdown)(constants_1.HIGHEST_SHUTDOWN_PRIORITY),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], ScalingService.prototype, "stop", null);
exports.ScalingService = ScalingService = __decorate([
(0, typedi_1.Service)(),
__metadata("design:paramtypes", [Logger_1.Logger,
ActiveExecutions_1.ActiveExecutions,
job_processor_1.JobProcessor,
config_2.GlobalConfig,
execution_repository_1.ExecutionRepository,
n8n_core_1.InstanceSettings,
orchestration_service_1.OrchestrationService])
], ScalingService);
//# sourceMappingURL=scaling.service.js.map