n8n
Version:
n8n Workflow Automation Tool
308 lines • 12.9 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.InProcessEventBus = void 0;
const backend_common_1 = require("@n8n/backend-common");
const config_1 = require("@n8n/config");
const decorators_1 = require("@n8n/decorators");
const di_1 = require("@n8n/di");
const node_events_1 = require("node:events");
const n8n_core_1 = require("n8n-core");
const constants_1 = require("../../../scaling/constants");
const publisher_service_1 = require("../../../scaling/pubsub/publisher.service");
const durable_event_log_1 = require("./durable-event-log");
const MAX_EVENTS_PER_THREAD = 500;
const MAX_BYTES_PER_THREAD = 2 * 1024 * 1024;
const SEQ_KEY_TTL_SECONDS = 14 * 24 * 60 * 60;
let InProcessEventBus = class InProcessEventBus {
constructor(logger, instanceSettings, publisher, eventLog, globalConfig) {
this.logger = logger;
this.instanceSettings = instanceSettings;
this.publisher = publisher;
this.eventLog = eventLog;
this.emitter = new node_events_1.EventEmitter();
this.store = new Map();
this.sizeBytes = new Map();
this.lastLocalId = new Map();
this.pendingByThread = new Map();
this.inFlightByThread = new Map();
this.drainingThreads = new Set();
this.logger = this.logger.scoped('instance-ai');
this.seqKeyPrefix = `${globalConfig.redis.prefix}:instance-ai:event-seq:`;
this.durableLogEnabled = globalConfig.instanceAi.durableLog;
this.emitter.setMaxListeners(0);
}
publish(threadId, event) {
if (event.ts === undefined) {
event = { ...event, ts: Date.now() };
}
if (this.durableLogEnabled) {
this.eventLog.publish(threadId, event, (drained) => this.onDrained(threadId, drained));
return;
}
if (!this.instanceSettings.isMultiMain) {
const id = (this.lastLocalId.get(threadId) ?? 0) + 1;
this.lastLocalId.set(threadId, id);
this.storeAndEmit(threadId, { id, event });
return;
}
const pending = this.pendingByThread.get(threadId);
if (pending) {
pending.push(event);
}
else {
this.pendingByThread.set(threadId, [event]);
}
void this.drainQueue(threadId);
}
onDrained(threadId, drained) {
const sizeBytes = Buffer.byteLength(JSON.stringify(drained.event), 'utf8');
if (drained.id !== undefined) {
this.cacheSequencedEvent(threadId, { id: drained.id, event: drained.event }, sizeBytes);
}
if (drained.live) {
const stored = {
...(drained.id !== undefined ? { id: drained.id } : {}),
event: drained.event,
};
this.emitter.emit(threadId, stored);
this.relayToSiblings(threadId, stored, sizeBytes);
}
}
cacheSequencedEvent(threadId, sequenced, sizeBytes) {
const events = this.getOrCreateStore(threadId);
if (!this.insertById(events, sequenced))
return;
this.sizeBytes.set(threadId, (this.sizeBytes.get(threadId) ?? 0) + sizeBytes);
this.evictIfNeeded(threadId, events);
}
async drainQueue(threadId) {
if (this.drainingThreads.has(threadId))
return;
this.drainingThreads.add(threadId);
try {
let batch = this.takePending(threadId);
while (batch.length > 0) {
this.inFlightByThread.set(threadId, batch);
const firstId = await this.assignSequenceBlock(threadId, batch.length);
for (let i = 0; i < batch.length; i++) {
const stored = { id: firstId + i, event: batch[i] };
const sizeBytes = Buffer.byteLength(JSON.stringify(batch[i]), 'utf8');
this.storeAndEmit(threadId, stored, sizeBytes);
this.relayToSiblings(threadId, stored, sizeBytes);
}
this.inFlightByThread.delete(threadId);
batch = this.takePending(threadId);
}
}
finally {
this.inFlightByThread.delete(threadId);
this.drainingThreads.delete(threadId);
}
}
takePending(threadId) {
const pending = this.pendingByThread.get(threadId);
if (!pending)
return [];
this.pendingByThread.delete(threadId);
return pending;
}
async assignSequenceBlock(threadId, count) {
try {
const key = this.seqKey(threadId);
const results = await this.getRedisClient()
.multi()
.incrby(key, count)
.expire(key, SEQ_KEY_TTL_SECONDS)
.exec();
const [incrError, incrResult] = results?.[0] ?? [new Error('empty transaction result'), null];
if (incrError)
throw incrError;
const endId = Number(incrResult);
if (!Number.isFinite(endId)) {
throw new Error(`non-numeric INCRBY result: ${String(incrResult)}`);
}
this.bumpLocalHighWaterMark(threadId, endId);
return endId - count + 1;
}
catch (error) {
this.logger.error('Failed to assign Instance AI event sequence from Redis, falling back to local ids', { threadId, error });
const firstId = (this.lastLocalId.get(threadId) ?? 0) + 1;
this.lastLocalId.set(threadId, firstId + count - 1);
return firstId;
}
}
getRedisClient() {
return this.publisher.getClient();
}
seqKey(threadId) {
return `${this.seqKeyPrefix}${threadId}`;
}
bumpLocalHighWaterMark(threadId, id) {
if (id > (this.lastLocalId.get(threadId) ?? 0)) {
this.lastLocalId.set(threadId, id);
}
}
storeAndEmit(threadId, stored, eventSizeBytes) {
const size = eventSizeBytes ?? Buffer.byteLength(JSON.stringify(stored.event), 'utf8');
const events = this.getOrCreateStore(threadId);
if (!this.insertById(events, stored))
return;
this.sizeBytes.set(threadId, (this.sizeBytes.get(threadId) ?? 0) + size);
this.evictIfNeeded(threadId, events);
this.emitter.emit(threadId, stored);
}
insertById(events, stored) {
if (events.length === 0 || events[events.length - 1].id < stored.id) {
events.push(stored);
return true;
}
let i = events.length - 1;
while (i >= 0 && events[i].id > stored.id)
i--;
if (i >= 0 && events[i].id === stored.id)
return false;
events.splice(i + 1, 0, stored);
return true;
}
relayToSiblings(threadId, stored, sizeBytes) {
if (!this.instanceSettings.isMultiMain)
return;
if (sizeBytes > constants_1.MAX_PUBSUB_PAYLOAD_BYTES) {
this.logger.warn(`Skipping cross-main relay of "${stored.event.type}" event (${sizeBytes} bytes exceeds ${constants_1.MAX_PUBSUB_PAYLOAD_BYTES})`, { threadId, runId: stored.event.runId });
return;
}
void this.publisher
.publishCommand({
command: 'relay-instance-ai-event',
payload: { threadId, storedEvent: stored },
})
.catch((error) => this.logger.error('Failed to relay Instance AI event to sibling mains', {
threadId,
error,
}));
}
handleRelayInstanceAiEvent({ threadId, storedEvent, }) {
if (storedEvent.id !== undefined) {
this.bumpLocalHighWaterMark(threadId, storedEvent.id);
}
if (!this.hasSubscribers(threadId))
return;
if (storedEvent.id === undefined) {
this.emitter.emit(threadId, storedEvent);
return;
}
this.storeAndEmit(threadId, { id: storedEvent.id, event: storedEvent.event });
}
subscribe(threadId, handler) {
this.emitter.on(threadId, handler);
return () => this.emitter.off(threadId, handler);
}
hasSubscribers(threadId) {
return this.emitter.listenerCount(threadId) > 0;
}
getEventsAfter(threadId, afterId) {
const events = this.store.get(threadId);
if (!events)
return [];
return events.filter((e) => e.id > afterId);
}
getEventsForRun(threadId, runId) {
return this.getEventsForRuns(threadId, [runId]);
}
getEventsForRuns(threadId, runIds) {
if (runIds.length === 0)
return [];
const runIdSet = new Set(runIds);
const stored = (this.store.get(threadId) ?? [])
.filter((e) => runIdSet.has(e.event.runId))
.map((e) => e.event);
const unsequenced = [
...(this.inFlightByThread.get(threadId) ?? []),
...(this.pendingByThread.get(threadId) ?? []),
].filter((e) => runIdSet.has(e.runId));
return [...stored, ...unsequenced];
}
async getNextEventId(threadId) {
if (this.durableLogEnabled) {
const events = this.store.get(threadId);
const last = events?.length ? events[events.length - 1].id : undefined;
return (last ?? 0) + 1;
}
if (this.instanceSettings.isMultiMain) {
try {
const value = await this.getRedisClient().get(this.seqKey(threadId));
if (value !== null)
return Number(value) + 1;
}
catch (error) {
this.logger.warn('Failed to read Instance AI event sequence from Redis, falling back to local high-water mark', { threadId, error });
}
}
return (this.lastLocalId.get(threadId) ?? 0) + 1;
}
clearThread(threadId) {
this.store.delete(threadId);
this.sizeBytes.delete(threadId);
this.lastLocalId.delete(threadId);
this.pendingByThread.delete(threadId);
this.inFlightByThread.delete(threadId);
this.eventLog.clearThread(threadId);
this.emitter.removeAllListeners(threadId);
if (this.instanceSettings.isMultiMain) {
void this.getRedisClient()
.del(this.seqKey(threadId))
.catch((error) => this.logger.warn('Failed to delete Instance AI event sequence key', {
threadId,
error,
}));
}
}
clear() {
this.store.clear();
this.sizeBytes.clear();
this.lastLocalId.clear();
this.pendingByThread.clear();
this.inFlightByThread.clear();
this.eventLog.clear();
this.emitter.removeAllListeners();
}
evictIfNeeded(threadId, events) {
let totalSize = this.sizeBytes.get(threadId) ?? 0;
while (events.length > MAX_EVENTS_PER_THREAD || totalSize > MAX_BYTES_PER_THREAD) {
const evicted = events.shift();
if (!evicted)
break;
totalSize -= Buffer.byteLength(JSON.stringify(evicted.event), 'utf8');
}
this.sizeBytes.set(threadId, Math.max(0, totalSize));
}
getOrCreateStore(threadId) {
let events = this.store.get(threadId);
if (!events) {
events = [];
this.store.set(threadId, events);
}
return events;
}
};
exports.InProcessEventBus = InProcessEventBus;
__decorate([
(0, decorators_1.OnPubSubEvent)('relay-instance-ai-event', { instanceType: 'main' }),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Object]),
__metadata("design:returntype", void 0)
], InProcessEventBus.prototype, "handleRelayInstanceAiEvent", null);
exports.InProcessEventBus = InProcessEventBus = __decorate([
(0, di_1.Service)(),
__metadata("design:paramtypes", [backend_common_1.Logger, n8n_core_1.InstanceSettings, publisher_service_1.Publisher, durable_event_log_1.DurableEventLog, config_1.GlobalConfig])
], InProcessEventBus);
//# sourceMappingURL=in-process-event-bus.js.map