n8n
Version:
n8n Workflow Automation Tool
3,950 lines • 192 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.InstanceAiService = exports.QuotaExhaustedStreamError = exports.QUOTA_EXHAUSTED_USER_MESSAGE = void 0;
exports.getUserFacingErrorMessage = getUserFacingErrorMessage;
exports.isMaskedStreamFailure = isMaskedStreamFailure;
const agents_1 = require("@n8n/agents");
const api_types_1 = require("@n8n/api-types");
const backend_common_1 = require("@n8n/backend-common");
const backend_network_1 = require("@n8n/backend-network");
const config_1 = require("@n8n/config");
const db_1 = require("@n8n/db");
const decorators_1 = require("@n8n/decorators");
const di_1 = require("@n8n/di");
const instance_ai_1 = require("@n8n/instance-ai");
const lazy_import_1 = require("@n8n/utils/lazy-import");
const workflow_sdk_1 = require("@n8n/workflow-sdk");
const n8n_core_1 = require("n8n-core");
const n8n_workflow_1 = require("n8n-workflow");
const nanoid_1 = require("nanoid");
const constants_1 = require("../../constants");
const forbidden_error_1 = require("../../errors/response-errors/forbidden.error");
const event_service_1 = require("../../events/event.service");
const instance_ai_builder_delegate_adapter_1 = require("../../modules/agents/instance-ai-builder-delegate.adapter");
const check_access_1 = require("../../permissions.ee/check-access");
const source_control_preferences_service_ee_1 = require("../../modules/source-control.ee/source-control-preferences.service.ee");
const publisher_service_1 = require("../../scaling/pubsub/publisher.service");
const ai_service_1 = require("../../services/ai.service");
const proxy_token_manager_1 = require("../../services/proxy-token-manager");
const url_service_1 = require("../../services/url.service");
const telemetry_1 = require("../../telemetry");
const agent_preview_handoff_1 = require("./agent-preview-handoff");
const composite_local_mcp_server_1 = require("./browser/composite-local-mcp-server");
const instance_ai_browser_session_service_1 = require("./browser/instance-ai-browser-session.service");
const thread_credential_allowlist_service_1 = require("./eval/thread-credential-allowlist.service");
const durable_event_log_1 = require("./event-bus/durable-event-log");
const in_process_event_bus_1 = require("./event-bus/in-process-event-bus");
const interrupted_run_sweeper_1 = require("./event-bus/interrupted-run-sweeper");
const instance_ai_credit_service_1 = require("./instance-ai-credit.service");
const instance_ai_error_reporter_service_1 = require("./instance-ai-error-reporter.service");
const instance_ai_gateway_service_1 = require("./instance-ai-gateway.service");
const instance_ai_memory_service_1 = require("./instance-ai-memory.service");
const instance_ai_model_service_1 = require("./instance-ai-model.service");
const instance_ai_run_probe_1 = require("./instance-ai-run-probe");
const instance_ai_settings_service_1 = require("./instance-ai-settings.service");
const instance_ai_temporary_workflow_service_1 = require("./instance-ai-temporary-workflow.service");
const instance_ai_terminal_outcome_service_1 = require("./instance-ai-terminal-outcome.service");
const instance_ai_adapter_service_1 = require("./instance-ai.adapter.service");
const internal_messages_1 = require("./internal-messages");
const liveness_1 = require("./liveness");
const mcp_1 = require("./mcp");
const observability_1 = require("./observability");
const output_redaction_config_1 = require("./output-redaction-config");
const planned_task_action_runner_1 = require("./planned-task-action-runner");
const instance_ai_pending_confirmation_repository_1 = require("./repositories/instance-ai-pending-confirmation.repository");
const instance_ai_thread_grant_repository_1 = require("./repositories/instance-ai-thread-grant.repository");
const sandbox_1 = require("./sandbox");
const db_iteration_log_storage_1 = require("./storage/db-iteration-log-storage");
const db_snapshot_storage_1 = require("./storage/db-snapshot-storage");
const typeorm_agent_checkpoint_store_1 = require("./storage/typeorm-agent-checkpoint-store");
const typeorm_agent_memory_1 = require("./storage/typeorm-agent-memory");
const suspended_run_restorer_service_1 = require("./suspended-run-restorer.service");
const suspended_thread_persistence_service_1 = require("./suspended-thread-persistence.service");
const tracing_1 = require("./tracing");
const workflow_verification_obligation_service_1 = require("./workflow-verification-obligation-service");
const workflow_verification_task_projector_1 = require("./workflow-verification-task-projector");
const agent_execution_service_1 = require("../agents/agent-execution.service");
const format_preview_context_1 = require("../agents/builder/format-preview-context");
function getErrorMessage(error) {
return error instanceof Error ? error.message : String(error);
}
function buildSuspensionTraceOutputs(runId, suspension) {
const rawMessage = suspension?.suspendPayload.message;
const message = typeof rawMessage === 'string' && rawMessage ? rawMessage : undefined;
return {
status: 'suspended',
runId,
...(suspension?.requestId ? { requestId: suspension.requestId } : {}),
...(suspension?.toolCallId ? { pendingToolCallId: suspension.toolCallId } : {}),
...(suspension?.toolName ? { toolName: suspension.toolName } : {}),
...(message ? { message } : {}),
};
}
function buildContextResourcesBlock(contextAttachments) {
if (contextAttachments.length === 0)
return '';
const lines = contextAttachments.map((attachment) => {
const name = attachment.name ? ` "${attachment.name}"` : '';
if (attachment.type === 'agent') {
return `- Agent${name} (id: \`${attachment.id}\`, in project \`${attachment.projectId}\`).`;
}
const execution = attachment.executionId
? `, currently viewing its execution \`${attachment.executionId}\``
: '';
return `- Workflow${name} (id: \`${attachment.id}\`)${execution}.`;
});
const header = contextAttachments.some((attachment) => attachment.type === 'agent')
? 'The user opened this conversation from the agent editor, where they are looking at:'
: 'The user opened this conversation from the workflow editor, where they are looking at:';
const prose = [
header,
...lines,
"Treat this purely as context. Until the user tells you what they need, don't read, inspect, run, or otherwise call tools on these resources, and don't make claims about their contents — just briefly acknowledge what they're working on and ask how you can help.",
].join('\n');
return `${internal_messages_1.EDITOR_CONTEXT_OPEN_TAG}\n${JSON.stringify(contextAttachments)}\n\n${prose}\n${internal_messages_1.EDITOR_CONTEXT_CLOSE_TAG}`;
}
function buildHandoffContextBlock(context) {
if (!context || context.source !== 'credential-modal')
return '';
const { credential } = context;
const lines = [
`- Credential type: \`${credential.credentialType}\` (${credential.displayName}).`,
credential.id ? `- Existing credential id: \`${credential.id}\`.` : '',
credential.nodeName ? `- Node name: "${credential.nodeName}".` : '',
credential.nodeType ? `- Node type: \`${credential.nodeType}\`.` : '',
credential.documentationUrl ? `- n8n documentation URL: ${credential.documentationUrl}` : '',
credential.oauthRedirectUrl
? `- OAuth redirect/callback URL shown in the modal: ${credential.oauthRedirectUrl}`
: '',
].filter(Boolean);
const prose = [
'The user opened this conversation from the credential setup modal and is asking for setup guidance.',
...lines,
'Use this metadata only as setup context. Never ask the user to paste credential secrets into chat. For credential setup docs, load `n8n-docs-assistant` and use `n8n-docs` with `intent: "credential-setup"`.',
].join('\n');
return `${internal_messages_1.CREDENTIAL_CONTEXT_OPEN_TAG}\n${JSON.stringify(context)}\n\n${prose}\n${internal_messages_1.CREDENTIAL_CONTEXT_CLOSE_TAG}`;
}
function isTelemetryConfigurableAgent(agent) {
return (typeof agent === 'object' &&
agent !== null &&
typeof Reflect.get(agent, 'telemetry') === 'function');
}
const INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS = 30 * 1000;
const WORKFLOW_SETUP_ROUTING_CLAIM_TTL_MS = 15 * 60 * 1000;
const CONFIRMATION_EXPIRED_MESSAGE = 'This confirmation has expired. Send a new message to continue.';
const INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS = 5 * 1000;
function isTextMessagePart(part) {
return (typeof part === 'object' &&
part !== null &&
'type' in part &&
part.type === 'text' &&
'text' in part &&
typeof part.text === 'string');
}
function isSandboxEndpointNotAllowedError(error) {
return getErrorMessage(error).toLowerCase().includes('endpoint not allowed');
}
exports.QUOTA_EXHAUSTED_USER_MESSAGE = "You've run out of AI credits. Upgrade your plan to continue using the AI assistant.";
function getUserFacingErrorCode(error) {
return (0, instance_ai_1.isQuotaExhaustedError)(error) ? 'quota_exhausted' : undefined;
}
function getUserFacingErrorMessage(error) {
if ((0, instance_ai_1.isQuotaExhaustedError)(error)) {
return exports.QUOTA_EXHAUSTED_USER_MESSAGE;
}
if (error instanceof n8n_workflow_1.UserError) {
return error.message;
}
if (isSandboxEndpointNotAllowedError(error)) {
return "I couldn't finish preparing the workspace sandbox. Please try again in a moment.";
}
if (error instanceof n8n_workflow_1.OperationalError) {
return 'I hit an operational error before I could finish that response. Please try again.';
}
if (error instanceof n8n_workflow_1.UnexpectedError) {
return 'Something went wrong before I could finish that response. Please try again.';
}
return 'Something went wrong before I could finish that response. Please try again.';
}
function isMaskedStreamFailure(error) {
if (!(error instanceof Error))
return false;
if (error.name === 'AI_NoOutputGeneratedError')
return true;
return error.name === 'TypeError' && error.message === 'terminated';
}
class QuotaExhaustedStreamError extends n8n_workflow_1.UserError {
constructor(maskedError) {
super(`AI credits exhausted (${maskedError.name}: ${maskedError.message})`, {
cause: maskedError,
});
this.errorCode = 'quota_exhausted';
}
}
exports.QuotaExhaustedStreamError = QuotaExhaustedStreamError;
function createInertAbortSignal() {
return new AbortController().signal;
}
function getAbortReason(signal) {
const reason = signal.reason;
if (typeof reason === 'object' &&
reason !== null &&
'name' in reason &&
reason.name === 'AbortError') {
return 'user_cancelled';
}
if (reason instanceof Error)
return reason.message;
return typeof reason === 'string' ? reason : 'user_cancelled';
}
const MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD = 5;
const MAX_CONSECUTIVE_FAILED_INTERNAL_FOLLOW_UPS = 3;
const TITLE_REFINE_HISTORY_LIMIT = 50;
function toConfirmationData(request) {
switch (request.kind) {
case 'approval':
return { approved: request.approved, userInput: request.userInput, scope: request.scope };
case 'domainAccessApprove':
return { approved: true, domainAccessAction: request.domainAccessAction };
case 'domainAccessDeny':
return { approved: false };
case 'planDeny':
return { approved: false, denied: true };
case 'questions':
return { approved: true, answers: request.answers };
case 'credentialSelection':
return { approved: true, credentials: request.credentials };
case 'credentialAutoSetup':
return { approved: true, autoSetup: { credentialType: request.credentialType } };
case 'resourceDecision':
return { approved: true, resourceDecision: request.resourceDecision };
case 'setupWorkflowApply':
return {
approved: true,
action: 'apply',
nodeCredentials: request.nodeCredentials,
nodeParameters: request.nodeParameters,
};
case 'setupWorkflowTestTrigger':
return {
approved: true,
action: 'test-trigger',
testTriggerNode: request.testTriggerNode,
nodeCredentials: request.nodeCredentials,
nodeParameters: request.nodeParameters,
};
}
}
let InstanceAiService = class InstanceAiService {
get mcpClientManager() {
if (!this._mcpClientManager) {
this._mcpClientManager = new instance_ai_1.McpClientManager(this._ssrfProtectionConfig.enabled ? this._ssrfProtectionService : undefined, { onToolCallSettled: (event) => this.trackMcpToolCall(event) });
}
return this._mcpClientManager;
}
constructor(logger, globalConfig, instanceSettings, adapterService, eventBus, eventLog, interruptedRunSweeper, settingsService, gatewayService, browserSessionService, memoryService, agentMemory, checkpointStore, aiService, threadGrantRepo, pendingConfirmationRepo, urlService, dbSnapshotStorage, dbIterationLogStorage, sourceControlPreferencesService, telemetry, mcpRegistryService, userRepository, temporaryWorkflowService, errorReporter, ssrfProtectionConfig, ssrfProtectionService, eventService, evalCredentialAllowlists, runProbe, modelService, creditService, publisher, instanceAiErrorReporter) {
this.instanceSettings = instanceSettings;
this.adapterService = adapterService;
this.eventBus = eventBus;
this.eventLog = eventLog;
this.interruptedRunSweeper = interruptedRunSweeper;
this.settingsService = settingsService;
this.gatewayService = gatewayService;
this.browserSessionService = browserSessionService;
this.memoryService = memoryService;
this.agentMemory = agentMemory;
this.checkpointStore = checkpointStore;
this.aiService = aiService;
this.threadGrantRepo = threadGrantRepo;
this.pendingConfirmationRepo = pendingConfirmationRepo;
this.urlService = urlService;
this.dbSnapshotStorage = dbSnapshotStorage;
this.dbIterationLogStorage = dbIterationLogStorage;
this.sourceControlPreferencesService = sourceControlPreferencesService;
this.telemetry = telemetry;
this.mcpRegistryService = mcpRegistryService;
this.userRepository = userRepository;
this.temporaryWorkflowService = temporaryWorkflowService;
this.errorReporter = errorReporter;
this.eventService = eventService;
this.evalCredentialAllowlists = evalCredentialAllowlists;
this.modelService = modelService;
this.creditService = creditService;
this.publisher = publisher;
this.instanceAiErrorReporter = instanceAiErrorReporter;
this.runState = new instance_ai_1.RunStateRegistry();
this.backgroundTasks = new instance_ai_1.BackgroundTaskManager(MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD);
this.memoryTaskRegistry = new instance_ai_1.MemoryTaskRegistry();
this.domainAccessTrackersByThread = new Map();
this.threadPushRef = new Map();
this.planRequestsByThread = new Map();
this.schedulerLocks = new Map();
this.failedInternalFollowUpStreaks = new Map();
this.pendingCheckpointReentries = new Map();
this.runDebugBuffer = new instance_ai_1.RunDebugBuffer();
this.checkpointPruningStopped = true;
this.inFlightExecutions = new Set();
this.preserveHitlOnShutdown = new Set();
this.logger = logger.scoped('instance-ai');
runProbe.registerActiveRunCountProvider(() => this.runState.activeRunCount());
this.workflowObligations = new workflow_verification_obligation_service_1.WorkflowVerificationObligationService(this.agentMemory);
this.taskProjector = new workflow_verification_task_projector_1.WorkflowVerificationTaskProjector(this.agentMemory, this.eventBus, this.logger, this.workflowObligations);
this.instanceAiConfig = globalConfig.instanceAi;
this.suspendedThreads = new suspended_thread_persistence_service_1.SuspendedThreadPersistenceService({
logger: this.logger,
config: this.instanceAiConfig,
pendingConfirmationRepo: this.pendingConfirmationRepo,
});
this.suspendedRunRestorer = new suspended_run_restorer_service_1.SuspendedRunRestorer({
logger: this.logger,
pendingConfirmationRepo: this.pendingConfirmationRepo,
runState: this.runState,
dbSnapshotStorage: this.dbSnapshotStorage,
eventBus: this.eventBus,
rebuilder: {
rebuildSuspendedRun: this.rebuildSuspendedRunFromCheckpoint.bind(this),
resumeSuspendedRun: this.resumeSuspendedRun.bind(this),
},
});
const livenessPolicyConfig = (0, instance_ai_1.createInstanceAiLivenessPolicyConfig)({
confirmationTimeoutMs: this.instanceAiConfig.confirmationTimeout,
});
this.liveness = new liveness_1.InstanceAiLivenessService({
policy: new instance_ai_1.InstanceAiLivenessPolicy(livenessPolicyConfig),
backgroundTaskIdleTimeoutMs: livenessPolicyConfig.backgroundTaskIdleTimeoutMs,
runState: this.runState,
backgroundTasks: this.backgroundTasks,
eventBus: this.eventBus,
logger: this.logger,
finalizeCancelledSuspendedRun: (suspended, reason) => {
void this.finalizeCancelledSuspendedRun(suspended, reason);
},
onPendingConfirmationRejected: (requestId) => {
void this.suspendedThreads.dropPendingConfirmation(requestId);
},
});
this.tracing = new tracing_1.InstanceAiTracingService({
logger: this.logger,
eventBus: this.eventBus,
runState: this.runState,
dbSnapshotStorage: this.dbSnapshotStorage,
aiService: this.aiService,
});
this.sandboxService = new sandbox_1.InstanceAiSandboxService({
config: this.instanceAiConfig,
logger: this.logger,
errorReporter: this.errorReporter,
runState: this.runState,
backgroundTasks: this.backgroundTasks,
settingsService: this.settingsService,
aiService: this.aiService,
});
this.terminalOutcome = new instance_ai_terminal_outcome_service_1.InstanceAiTerminalOutcomeService({
durableLog: globalConfig.instanceAi.durableLog,
eventBus: {
publish: (threadId, event) => this.eventBus.publish(threadId, event),
getEventsForRun: async (threadId, runId) => this.instanceAiConfig.durableLog
? await this.readDurableEventsForRuns(threadId, [runId])
: this.eventBus.getEventsForRun(threadId, runId),
getEventsForRuns: async (threadId, runIds) => this.instanceAiConfig.durableLog
? await this.readDurableEventsForRuns(threadId, runIds)
: this.eventBus.getEventsForRuns(threadId, runIds),
},
dbSnapshotStorage: this.dbSnapshotStorage,
agentMemory: this.agentMemory,
telemetry: this.telemetry,
logger: this.logger,
runState: this.runState,
suspendedThreads: this.suspendedThreads,
tracing: this.tracing,
publishRunFinish: (threadId, runId, status, reason) => {
this.publishRunFinish(threadId, runId, status, reason);
},
saveAgentTreeSnapshot: async (threadId, runId, snapshotStorage) => await this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage),
});
this.defaultTimeZone = globalConfig.generic.timezone;
const restEndpoint = globalConfig.endpoints.rest;
this.oauth2CallbackUrl = `${this.urlService.getInstanceBaseUrl()}/${restEndpoint}/oauth2-credential/callback`;
this.webhookBaseUrl = `${this.urlService.getWebhookBaseUrl()}${globalConfig.endpoints.webhook}`;
this.formBaseUrl = `${this.urlService.getWebhookBaseUrl()}${globalConfig.endpoints.form}`;
this._ssrfProtectionConfig = ssrfProtectionConfig;
this._ssrfProtectionService = ssrfProtectionService;
this.eventService.on('instance-ai-settings-updated', ({ mcpSettingsChanged }) => {
this.sandboxService.invalidateCachedWorkspaces();
if (!mcpSettingsChanged)
return;
if (!this._mcpClientManager)
return;
this._mcpClientManager.disconnect().catch((error) => {
this.logger.warn('Failed to disconnect MCP clients after settings change', {
error: getErrorMessage(error),
});
});
});
this.liveness.start();
if (this.instanceSettings.isLeader)
this.startCheckpointPruning();
}
async createProxyRunConfig(user) {
if (!this.aiService.isProxyEnabled())
return {};
const client = await this.aiService.getClient();
const proxyBaseUrl = client.getApiProxyBaseUrl();
const tokenManager = new proxy_token_manager_1.ProxyTokenManager(async () => {
return await client.getInstanceAiApiProxyToken({ id: user.id }, { userMessageId: (0, nanoid_1.nanoid)() });
});
const featureHeaders = (0, api_types_1.buildProxyHeaders)({
feature: 'instance-ai',
n8nVersion: constants_1.N8N_VERSION,
});
return {
proxyBaseUrl,
tokenManager,
searchProxyConfig: {
apiUrl: proxyBaseUrl + '/brave-search',
getAuthHeaders: async () => ({
...(await tokenManager.getAuthHeaders()),
...featureHeaders,
}),
},
tracingProxyConfig: {
apiUrl: proxyBaseUrl + '/langsmith',
getAuthHeaders: async () => ({
...(await tokenManager.getAuthHeaders()),
...featureHeaders,
}),
},
};
}
async resolveAgentModelConfig(user) {
return await this.modelService.resolveAgentModelConfig(user);
}
async loadThreadSessionGrants(threadId, userId) {
try {
return await this.threadGrantRepo.findKeys(threadId, userId);
}
catch (error) {
this.logger.warn('Failed to load Instance AI session grants', {
threadId,
error: getErrorMessage(error),
});
return new Set();
}
}
async persistThreadSessionGrant(threadId, userId, key) {
try {
await this.threadGrantRepo.grant(threadId, userId, key);
}
catch (error) {
this.logger.warn('Failed to persist Instance AI session grant', {
threadId,
key,
error: getErrorMessage(error),
});
}
}
isProxyEnabled() {
return this.modelService.isProxyEnabled();
}
async getCredits(user) {
return await this.modelService.getCredits(user);
}
async reclassifyMaskedStreamFailure(error, user, context) {
if (!isMaskedStreamFailure(error))
return error;
try {
const { creditsQuota, creditsClaimed } = await this.modelService.getCredits(user);
if (creditsQuota < 0 || creditsClaimed < creditsQuota)
return error;
}
catch (creditsError) {
this.logger.debug('Masked stream failure credit re-check failed; keeping original error', {
error: getErrorMessage(creditsError),
...context,
});
return error;
}
this.logger.info('Reclassified masked stream failure as quota-exhausted', {
maskedError: getErrorMessage(error),
...context,
});
return new QuotaExhaustedStreamError(error);
}
isEnabled() {
return this.settingsService.isAgentEnabled() && !!this.instanceAiConfig.model;
}
hasActiveRun(threadId) {
return this.runState.hasLiveRun(threadId);
}
isRunLive(threadId, runId) {
return (this.runState.getActiveRunId(threadId) === runId ||
this.runState.getSuspendedRun(threadId)?.runId === runId);
}
getThreadStatus(threadId) {
const status = this.runState.getThreadStatus(threadId, this.backgroundTasks.getTaskSnapshots(threadId));
const memoryTasks = this.memoryTaskRegistry.getTasks(threadId);
return { ...status, memoryTasks };
}
memoryTaskObserverFor(threadId, tracing) {
return (event) => {
tracing?.onMemoryTaskEvent?.(event);
this.memoryTaskRegistry.handleEvent(threadId, event);
const pendingTasks = this.memoryTaskRegistry.getTasks(threadId);
const logContext = {
threadId,
taskId: event.task.id,
taskKind: event.task.taskKind,
pendingCount: pendingTasks.length,
...(event.type === 'skipped' ? { reason: event.reason } : {}),
...(event.type === 'failed' ? { error: getErrorMessage(event.error) } : {}),
...(event.type === 'completed' &&
event.value &&
typeof event.value === 'object' &&
'status' in event.value &&
typeof event.value.status === 'string'
? { outcome: event.value.status }
: {}),
};
this.logger.info(`Observational memory task ${event.type}`, logContext);
};
}
subscribeToAgentErrors(agent, threadId, runId) {
agent.on("error", (event) => {
if (event.type !== "error" || !event.source)
return;
this.instanceAiErrorReporter.report(event.error, {
component: `instance-ai-${event.source}`,
threadId,
runId,
});
});
}
isRunDebugEnabled() {
return this.instanceAiConfig.runDebugEnabled;
}
buildOrchestratorAgentStreamOptions(user, threadId, runId, signal) {
if (this.isRunDebugEnabled()) {
this.runDebugBuffer.ensure(runId, threadId);
}
return {
maxIterations: instance_ai_1.MAX_STEPS.ORCHESTRATOR,
abortSignal: signal,
recoverUsageOnAbort: true,
persistence: {
resourceId: user.id,
threadId,
hostRunId: runId,
},
providerOptions: {
anthropic: { cacheControl: { type: 'ephemeral' } },
},
...(this.isRunDebugEnabled()
? (0, instance_ai_1.createRunDebugStepHooks)(this.runDebugBuffer, { runId, threadId })
: {}),
};
}
buildOrchestratorResumeAgentOptions(user, threadId, runId, agentRunId, toolCallId) {
if (this.isRunDebugEnabled()) {
this.runDebugBuffer.ensure(runId, threadId);
}
return {
runId: agentRunId,
toolCallId,
recoverUsageOnAbort: true,
persistence: { resourceId: user.id, threadId, hostRunId: runId },
providerOptions: {
anthropic: { cacheControl: { type: 'ephemeral' } },
},
...(this.isRunDebugEnabled()
? (0, instance_ai_1.createRunDebugStepHooks)(this.runDebugBuffer, { runId, threadId })
: {}),
};
}
getRunDebug(runId) {
return this.runDebugBuffer.get(runId);
}
listThreadDebugRuns(threadId) {
return this.runDebugBuffer.listByThread(threadId).map((record) => ({
runId: record.runId,
threadId: record.threadId,
startedAt: record.startedAt,
stepCount: record.steps.length,
workflowCodeCount: record.workflowCode.length,
label: record.label,
}));
}
clearTraceContextsForTest() {
this.tracing.clearTraceContextsForTest();
}
async submitLangsmithFeedback(user, threadId, responseId, payload) {
await this.tracing.submitLangsmithFeedback(user, threadId, responseId, payload);
}
startRun(user, threadId, message, attachments, context, timeZone, pushRef) {
this.liveness.clearThreadState(threadId);
const { runId, abortController, messageGroupId } = this.runState.startRun({
threadId,
user,
});
if (timeZone) {
this.runState.setTimeZone(threadId, timeZone);
}
if (pushRef !== undefined) {
this.threadPushRef.set(threadId, pushRef);
}
this.startExecuteRun(user, threadId, runId, message, abortController, attachments, context, messageGroupId, timeZone);
return runId;
}
getMessageGroupId(threadId) {
return this.runState.getMessageGroupId(threadId);
}
getLiveMessageGroupId(threadId) {
return this.runState.getLiveMessageGroupId(threadId, this.backgroundTasks.getTaskSnapshots(threadId));
}
getRunIdsForMessageGroup(messageGroupId) {
return this.runState.getRunIdsForMessageGroup(messageGroupId);
}
getActiveRunId(threadId) {
return this.runState.getActiveRunId(threadId);
}
cancelRun(threadId, reason = 'user_cancelled') {
const cancelledTasks = this.backgroundTasks.cancelThread(threadId);
const user = this.runState.getThreadUser(threadId);
for (const task of cancelledTasks) {
void this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
this.eventBus.publish(threadId, {
type: 'agent-completed',
runId: task.runId,
agentId: task.agentId,
payload: {
role: task.role,
result: '',
error: reason === liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON ? 'Timed out' : 'Cancelled by user',
},
});
void this.terminalOutcome.recordBackgroundTerminalOutcome(task).finally(() => {
void this.saveAgentTreeSnapshot(threadId, task.runId, this.dbSnapshotStorage, true, task.messageGroupId);
});
if (user) {
void this.handlePlannedTaskSettlement(user, task, 'cancelled');
}
}
void this.cancelAwaitingApprovalPlan(threadId);
const { active, suspended } = this.runState.cancelThread(threadId);
if (active) {
if (reason === liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON)
this.liveness.markRunTimedOut(active.runId);
active.abortController.abort();
void this.suspendedThreads.dropPendingConfirmationsForThread(threadId);
return;
}
if (suspended) {
if (reason === liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON)
this.liveness.markRunTimedOut(suspended.runId);
suspended.abortController.abort();
void this.finalizeCancelledSuspendedRun(suspended, reason);
}
void this.suspendedThreads.dropPendingConfirmationsForThread(threadId);
}
sendCorrectionToTask(threadId, taskId, correction) {
return this.backgroundTasks.queueCorrection(threadId, taskId, correction);
}
cancelBackgroundTask(threadId, taskId) {
const task = this.backgroundTasks.cancelTask(threadId, taskId);
if (!task)
return;
void this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
this.eventBus.publish(threadId, {
type: 'agent-completed',
runId: task.runId,
agentId: task.agentId,
payload: { role: task.role, result: '', error: 'Cancelled by user' },
});
void this.terminalOutcome.recordBackgroundTerminalOutcome(task).finally(() => {
void this.saveAgentTreeSnapshot(threadId, task.runId, this.dbSnapshotStorage, true, task.messageGroupId);
});
const user = this.runState.getThreadUser(threadId);
if (user) {
void this.handlePlannedTaskSettlement(user, task, 'cancelled');
}
}
broadcastTaskControl(payload) {
if (!this.instanceSettings.isMultiMain)
return;
void this.publisher
.publishCommand({ command: 'relay-instance-ai-task-control', payload })
.catch((error) => this.logger.error('Failed to relay Instance AI task-control to sibling mains', {
threadId: payload.threadId,
action: payload.action,
error,
}));
}
async applyTaskControlLocally({ threadId, taskId, action, correction, }) {
switch (action) {
case 'correct':
if (!taskId || correction === undefined)
return true;
return this.sendCorrectionToTask(threadId, taskId, correction) !== 'task-not-found';
case 'cancel-task': {
if (!taskId)
return true;
const isLocal = this.backgroundTasks
.getTaskSnapshots(threadId)
.some((task) => task.taskId === taskId);
this.cancelBackgroundTask(threadId, taskId);
return isLocal;
}
case 'cancel-thread':
this.cancelRun(threadId);
return false;
case 'clear-thread':
await this.clearThreadState(threadId);
return false;
}
}
async routeTaskControl(payload) {
const foundLocally = await this.applyTaskControlLocally(payload);
if (!foundLocally)
this.broadcastTaskControl(payload);
}
async routeCorrectionToTask(threadId, taskId, correction) {
await this.routeTaskControl({ threadId, taskId, action: 'correct', correction });
}
async routeCancelBackgroundTask(threadId, taskId) {
await this.routeTaskControl({ threadId, taskId, action: 'cancel-task' });
}
async routeCancelRun(threadId) {
const hadLiveLocalRun = this.runState.hasLiveRun(threadId);
await this.routeTaskControl({ threadId, action: 'cancel-thread' });
if (!hadLiveLocalRun) {
await this.eventLog.flush(threadId);
await this.interruptedRunSweeper.cancelUnfinishedRuns(threadId);
}
}
async routeClearThreadState(threadId) {
await this.routeTaskControl({ threadId, action: 'clear-thread' });
}
async handleRelayTaskControl(payload) {
try {
await this.applyTaskControlLocally(payload);
}
catch (error) {
this.logger.error('Failed to apply relayed Instance AI task-control', {
threadId: payload.threadId,
taskId: payload.taskId,
action: payload.action,
error,
});
}
}
cancelAllBackgroundTasks() {
const cancelled = this.backgroundTasks.cancelAll();
for (const task of cancelled) {
void this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
}
return cancelled.length;
}
async startStuckBackgroundTaskForTest(user, threadId) {
const messageId = `msg_${(0, nanoid_1.nanoid)()}`;
const messageText = 'I started a background workflow-builder task.';
const { runId, messageGroupId } = this.runState.startRun({ threadId, user });
if (!messageGroupId) {
throw new n8n_workflow_1.UnexpectedError('Failed to create message group for timeout simulation');
}
const taskId = `task_${(0, nanoid_1.nanoid)()}`;
const agentId = `agent_${(0, nanoid_1.nanoid)()}`;
this.eventBus.publish(threadId, {
type: 'run-start',
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
userId: user.id,
payload: { messageId, messageGroupId },
});
this.eventBus.publish(threadId, {
type: 'text-delta',
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
responseId: `test-background-start:${runId}`,
payload: { text: messageText },
});
this.eventBus.publish(threadId, {
type: 'agent-spawned',
runId,
agentId,
payload: {
parentId: (0, instance_ai_1.orchestratorAgentId)(runId),
role: 'workflow-builder',
tools: [],
taskId,
kind: 'builder',
title: 'Building workflow',
subtitle: 'Timeout simulation',
goal: 'Simulate a stuck background task timeout',
},
});
await this.agentMemory.saveMessages({
threadId,
resourceId: user.id,
messages: [
{
id: messageId,
createdAt: new Date(),
type: 'llm',
role: 'assistant',
content: [{ type: 'text', text: messageText }],
},
],
});
const outcome = this.backgroundTasks.spawn({
taskId,
threadId,
runId,
role: 'workflow-builder',
agentId,
messageGroupId,
run: async (signal) => await new Promise((resolve) => {
signal.addEventListener('abort', () => resolve('aborted'), { once: true });
}),
onFailed: (task) => {
this.eventBus.publish(threadId, {
type: 'agent-completed',
runId,
agentId,
payload: {
role: task.role,
result: '',
error: task.error ?? 'Unknown error',
},
});
},
onSettled: async (task) => {
await this.terminalOutcome.recordBackgroundTerminalOutcome(task);
await this.saveAgentTreeSnapshot(threadId, runId, this.dbSnapshotStorage, true, messageGroupId);
},
});
if (outcome.status !== 'started') {
throw new n8n_workflow_1.UnexpectedError('Failed to start stuck background task simulation');
}
this.runState.clearActiveRun(threadId);
this.eventBus.publish(threadId, {
type: 'run-finish',
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
userId: user.id,
payload: { status: 'completed' },
});
return {
threadId,
runId,
messageGroupId,
taskId,
agentId,
timeoutAt: outcome.task.lastActivityAt + this.liveness.backgroundTaskIdleTimeoutMs + 1,
};
}
async runLivenessSweepForTest(now) {
await this.liveness.sweepTimedOutWork(now);
}
loadTraceEvents(slug, events) {
this.tracing.loadTraceEvents(slug, events);
}
getTraceEvents(slug) {
return this.tracing.getTraceEvents(slug);
}
hasRunningWorkForTest() {
const threadIds = new Set(this.tracing.getTrackedThreadIds());
for (const threadId of threadIds) {
if (this.runState.getActiveRunId(threadId))
return true;
if (this.backgroundTasks.getRunningTasks(threadId).length > 0)
return true;
}
return false;
}
activateTraceSlug(slug) {
this.tracing.activateTraceSlug(slug);
}
clearTraceEvents(slug) {
this.tracing.clearTraceEvents(slug);
}
async clearThreadState(threadId) {
this.liveness.clearThreadState(threadId);
const { active, suspended } = this.runState.clearThread(threadId);
if (active) {
active.abortController.abort();
await this.tracing.finalizeRunTracing(active.runId, active.tracing, {
status: 'cancelled',
reason: 'thread_cleared',
});
}
if (suspended) {
suspended.abortController.abort();
await this.tracing.finalizeRunTracing(suspended.runId, suspended.tracing, {
status: 'cancelled',
reason: 'thread_cleared',
});
}
for (const task of this.backgroundTasks.cancelThread(threadId)) {
task.abortController.abort();
await this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
}
await this.tracing.finalizeRemainingMessageTraceRoots(threadId, {
status: 'cancelled',
reason: 'thread_cleared',
metadata: { completion_source: 'service_cleanup' },
});
this.schedulerLocks.delete(threadId);
this.failedInternalFollowUpStreaks.delete(threadId);
this.domainAccessTrackersByThread.delete(threadId);
this.evalCredentialAllowlists.clearThread(threadId);
this.threadPushRef.delete(threadId);
this.planRequestsByThread.delete(threadId);
this.memoryTaskRegistry.clearThread(threadId);
this.tracing.deleteTraceContextsForThread(threadId);
await this.deleteAgentBuilderSessions(threadId);
await this.sandboxService.destroySandbox(threadId);
await this.temporaryWorkflowService.reapForThreadCleanup(threadId);
await this.suspendedThreads.dropPendingConfirmationsForThread(threadId);
this.eventBus.clearThread(threadId);
}
async deleteAgentBuilderSessions(threadId) {
if (!di_1.Container.get(backend_common_1.ModuleRegistry).isActive('agents'))
return;
try {
await di_1.Container.get(instance_ai_builder_delegate_adapter_1.InstanceAiBuilderDelegateAdapterService).deleteBuilderSessions(threadId);
}
catch (error) {
this.logger.warn('Failed to clean up agent-builder sessions for thread', {
threadId,
error: error instanceof Error ? error.message : String(error),
});
}
}
async shutdown() {
this.stopCheckpointPruning();
this.liveness.shutdown();
const { activeRuns, suspendedRuns, pendingThreadIds } = this.runState.shutdown();
const threadsWithPendingHitl = new Set(pendingThreadIds);
for (const run of activeRuns) {
if (threadsWithPendingHitl.has(run.threadId)) {
await this.tracing.finalizeRunTracing(run.runId, run.tracing, {
status: 'cancelled',
reason: 'service_shutdown',
});
this.preserveHitlOnShutdown.add(run.runId);
run.abortController.abort();
continue;
}
this.publishRunFinish(run.threadId, run.runId, 'cancelled', 'service_shutdown');
await this.persistShutdownSnapshot(run.threadId, run.runId, run.messageGroupId);
await this.tracing.finalizeRunTracing(run.runId, run.tracing, {
status: 'cancelled',
reason: 'service_shutdown',
});
run.abortController.abort();
}
for (const run of suspendedRuns) {
await this.tracing.finalizeRunTracing(run.runId, run.tracing, {
status: 'cancelled',
reason: 'service_shutdown',
});
run.abortController.abort();
}
for (const task of this.backgroundTasks.cancelAll()) {
task.abortController.abort();
await this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
}
await this.drainInFlightExecutions(INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS);
this.instanceAiErrorReporter.endAllRuns();
const threadsWithTraces = new Set(this.tracing.getTrackedThreadIds());
for (const threadId of threadsWithTraces) {
await this.tracing.finalizeRemainingMessageTraceRoots(threadId, {
status: 'cancelled',
reason: 'service_shutdown',
metadata: { completion_source: 'service_cleanup' },
});
}
this.gatewayService.disconnectAll();
await this.browserSessionService.shutdown();
this.sandboxService.stopSandboxExpiryTimers();
this.domainAccessTrackersByThread.clear();
this.tracing.clear();
await this.eventLog.flushAll();
this.eventBus.clear();
await this._mcpClientManager?.disconnect();
await (0, instance_ai_1.shutdownProductTelemetryProviders)();
this.logger.debug('Instance AI service shut down');
}
startCheckpointPruning() {
if (this.checkpointPruneTimer || this.instanceAiConfig.pruneInterval <= 0)
return;
this.checkpointPruningStopped = false;
this.scheduleCheckpointPrune(0);
}
stopCheckpointPruning() {
this.checkpointPruningStopped = true;
clearTimeout(this.checkpointPruneTimer);
this.checkpointPruneTimer = undefined;
}
scheduleCheckpointPrune(delayMs = this.instanceAiConfig.pruneInterval) {
if (this.checkpointPruningStopped)
return;
this.checkpointPruneTimer = setTimeout(() => {
void this.runScheduledPrune();
}, delayMs);
this.checkpointPruneTimer.unref();
}
trackInFlightExecution(promise) {
const tracked = promise.finally(() => {
this.inFlightExecutions.delete(tracked);
});
this.inFlightExecutions.add(tracked);
}
startExecuteRun(...args) {
this.trackInFlightExecution(this.executeRun(...args));
}
startProcessResumedStream(...args) {
this.trackInFlightExecution(this.processResumedStream(...args));
}
shouldPreserveHitlOnShutdown(runId) {
return this.preserveHitlOnShutdown.has(runId);
}
async drainInFlightExecutions(timeoutMs) {
if (this.inFlightExecutions.size === 0)
return;
const drain = Promise.allSettled([...this.inFlightExecutions]);
const timeout = new Promise((resolve) => {
setTimeout(() => resolve('timeout'), timeoutMs).unref();
});
const outcome = await Promise.race([drain.then(() => 'drained'), timeout]);
if (outcome === 'timeout') {
this.logger.warn('Timed out waiting for in-flight Instance AI runs to drain', {
timeoutMs,
stillInFlight: this.inFlightExecutions.size,
});
}
}
async runScheduledPrune(now = Date.now()) {
const olderThan = new Date(now - this.instanceAiConfig.snapshotRetention);
try {
const count = await this.checkpointStore.markExpiredOlderThan(olderThan);
if (count > 0) {
this.logger.info('Expired stale Instance AI checkpoints', { count });
}
else {
this.logger.debug('No stale Instance AI checkpoints to expire');
}
await this.hardDeleteExpiredCheckpoints(now);
await this.suspendedThreads.pruneStalePendingConfirmations(now);
await this.pruneExpiredThreads();
this.scheduleCheckpointPrune();
}
catch (error) {
this.logger.warn('Failed to expire stale Instance AI checkpoints', {
error: getErrorMessage(error),
});
this.scheduleCheckpointPrune(INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS);
}
}
async hardDeleteExpiredCheckpoints(now) {
const retention = this.instanceAiConfig.checkpointGcRetention;
if (retention <= 0)
return;
try {
const olderThan = new Date(now - retention);
const count = await this.checkpointStore.hardDeleteExpiredOlderThan(olderThan);
if (count > 0) {
this.logger.info('Hard-deleted expired Instance AI checkpoint tombstones', { count });
}
}
catch (error) {
this.logger.warn('Failed to hard-delete expired Instance AI checkpoint tombstones', {
error: getErrorMessage(error),
});
}
}
async pruneExpiredThreads() {
try {
await this.memoryService.cleanupExpiredThreads(async (threadId) => await this.clearThreadState(threadId));
}
catch (error) {
this.logger.warn('Failed to clean up expired Instance AI conversation threads', {
error: getErrorMessage(error),
});
}
}
async persistShutdownSnapshot(threadId, runId, messageGroupId) {
try {
await this.saveAgentTreeSnapshot(threadId, runId, this.dbSnapshotStorage, true, messageGroupId);
}
catch (error) {
this.logger.warn('Failed to persist shutdown snapshot', {
threadId,
runId,
error: getErrorMessage(error),
});
}
}
createAgentMemoryOptions(user, threadId, runId) {
return {
observationalMemory: {
observerThresholdTokens: this.instanceAiConfig.observerMessageTokens,
reflectorThresholdTokens: this.instanceAiConfig.reflectorObservationTokens,
onTaskUsage: async (report) => {
try {
const items = (0, instance_ai_1.tokenUsageToBuilderUsageItems)(report.model, report.usage);
if (items.length === 0)
return;
await this.creditService.claimRunUsage(user, threadId, `${runId}:memory:${report.task}:${report.reportId}`, items, 'completed');
}
catch (error) {
this.logger.warn('Failed to claim observational-memory usage', {
threadId,
runId,
task: report.task,
error: getErrorMessage(error),
});
}
},
},
};
}
createWorkflowTaskServiceWithUiSync(threadId, runId, workflowTasks) {
const sync = async () => await this.taskProjector.syncFromWorkflowLoop(threadId, runId);
return {
reportBuildOutcome: async (outcome) => {
const action = await workflowTasks.reportBuildOutcome(outcome);
await sync();
return action;
},
reportVerificationVerdict: async (verdict) => {
const action = await workflowTasks.reportVerificationVerdict(verdict);
await sync();
return action;
},
updateBuildOutcome: async (workItemId, update) => {
await workflowTasks.updateBuildOutcome(workItemId, update);
await sync();
},
getBuildOutcome: async (workItemId) => await workflowTasks.getBuildOutcome(workItemId),
getLatestBuildOutcomeForWorkflow: async (workflowId) => await workflowTasks.getLatestBuildOutcomeForWorkflow(workflowId),
getWorkflowLoopState: async (workItemId) => await workflowTasks.getWorkflowLoopState(workItemId),
};
}
trackWorkflowVerificationObligation(obligation, event, extra = {}) {
try {
this.telemetry?.track('instance_ai_workflow_verification_obligation', {
event,
thread_id: obligation.threadId,
run_id: obligation.runId,
task_id: obligation.taskId,
planned_task_id: obligation.plannedTaskId,
work_item_id: obligation.workItemId,
workflow_id: obligation.workflowId,
source: obligation.source,
policy: obligation.policy,
status: obligation.status,
readiness_status: obligation.readiness?.status,
setup_status: obligation.setupRequirement?.status,
has_evidence: obligation.evidence?.attempted === true,
evidence_success: obligation.evidence?.success,
blocking_reason: obligation.blockingReason,
...extra,
});
}
catch (error) {
this.logger.warn('Failed to track workflow verification obligation telemetry', {
threadId: obligation.threadId,
workItemId: obligation.workItemId,
error: getErrorMessage(error),
});
}
}
buildPlannedTaskFollowUpMessage(type, graph, options = {}) {
const payload = {
tasks: graph.tasks.map((task) => ({
id: task.id,
title: task.title,
kind: task.kind,
status: task.status,
result: task.result,
error: task.error,
outcome: task.outcome,
})),
};
if (options.failedTask) {
payload.failedTask = {
id: options.failedTask.id,
title: options.failedTask.title,
kind: options.failedTask.kind,
error: options.failedTask.error,
result: options.failedTask.result,
};
}
if (options.checkpoint) {
const depOutcomes = graph.tasks
.filter((t) => options.checkpoint.deps.includes(t.id))
.map((t) => ({
id: t.id,
title: t.title,
kind: t.kind,
status: t.status,
result: t.result,
outcome: t.outcome,
}));
payload.checkpoint = {
id: options.checkpoint.id,
title: options.checkpoint.title,
instructions: options.checkpoint.spec,
dependsOn: depOutcomes,
};
}
if (options.buildTask) {
payload.buildTask = {
id: options.buildTask.id,
title: options.buildTask.title,
kind: options.buildTask.kind,
spec: options.buildTask.spec,
workflowId: options.buildTask.workflowId,
isSupportingWorkflow: options.buildTask.isSupportingWorkflow,
deps: options.buildTask.deps,
};
}
return `<planned-task-follow-up type="${type}">\n${JSON.stringify(payload, null, 2)}\n</planned-task-follow-up>\n\n${internal_messages_1.AUTO_FOLLOW_UP_MESSAGE}`;
}
buildWorkflowVerificationFollowUpMessage(input) {
const payload = {
obligation: input.obligation,
outcome: input.outcome,
sourceTask: input.sourceTask,
};
return `<workflow-verification-follow-up>\n${JSON.stringify(payload, null, 2)}\n</workflow-verification-follow-up>\n\n${internal_messages_1.AUTO_FOLLOW_UP_MESSAGE}`;
}
async createPlannedTaskState() {
const memory = this.agentMemory;
const taskStorage = new instance_ai_1.ThreadTaskStorage(memory);
const plannedTaskStorage = new instance_ai_1.PlannedTaskStorage(memory);
const plannedTaskService = new instance_ai_1.PlannedTaskCoordinator(plannedTaskStorage);
return { memory, taskStorage, plannedTaskService };
}
async replayUndeliveredTerminalOutcomes(threadId, options = {}) {
await this.terminalOutcome.replayUndeliveredTerminalOutcomes(threadId, options);
}
async syncPlannedTasksToUi(threadId, graph) {
const { taskStorage } = await this.createPlannedTaskState();
const tasks = await this.taskProjector.projectPlannedTaskList(threadId, graph);
await taskStorage.save(threadId, tasks);
this.eventBus.publish(threadId, {
type: 'tasks-update',
runId: graph.planRunId,
agentId: (0, instance_ai_1.orchestratorAgentId)(graph.planRunId),
payload: { tasks },
});
}
async cancelAwaitingApprovalPlan(threadId) {
try {
const { plannedTaskService, taskStorage } = await this.createPlannedTaskState();
const graph = await plannedTaskService.getGraph(threadId);
if (!graph || graph.status !== 'awaiting_approval')
return;
await plannedTaskService.clear(threadId);
await taskStorage.save(threadId, { tasks: [] });
this.eventBus.publish(threadId, {
type: 'tasks-update',
runId: graph.planRunId,
agentId: (0, instance_ai_1.orchestratorAgentId)(graph.planRunId),
payload: { tasks: { tasks: [] }, planItems: [] },
});
}
catch (error) {
this.logger.warn('Failed to clean up awaiting_approval plan on cancel', {
threadId,
error: error instanceof Error ? error.message : String(error),
});
}
}
async createExecutionEnvironment(user, threadId, runId, abortSignal, messageGroupId, pushRef, proxyRunConfig) {
const memory = this.agentMemory;
const boundProjectId = await memory.getThreadProjectId(threadId);
if (!boundProjectId) {
throw new n8n_workflow_1.UnexpectedError(`Instance AI thread "${threadId}" has no bound project; it must be created via POST /instance-ai/threads before a run can start`);
}
const adminSettings = await this.settingsService.getAdminSettings();
const localGatewayDisabledGlobally = adminSettings.localGatewayDisabled;
const browserUseEnabledGlobally = adminSettings.browserUseEnabled;
const localGatewayDisabledForUser = await this.settingsService.isLocalGatewayDisabledForUser(user.id);
const userGateway = this.gatewayService.findGateway(user.id);
const { searchProxyConfig, tracingProxyConfig, tokenManager, proxyBaseUrl } = proxyRunConfig ?? (await this.createProxyRunConfig(user));
const modelId = proxyBaseUrl && tokenManager
? await this.modelService.resolveProxyModel(user, proxyBaseUrl, tokenManager)
: await this.modelService.resolveAgentModelConfig(user);
const configEvalsEnabled = await this.adapterService.isConfigEvalsEnabled(user);
const context = this.adapterService.createContext(user, {
searchProxyConfig,
pushRef,
threadId,
projectId: boundProjectId,
credentialIdAllowlist: this.evalCredentialAllowlists.get(threadId),
configEvalsEnabled,
modelId,
});
this.gatewayService.applyToolPolicy(user.id);
const gatewayMcpServer = !localGatewayDisabledForUser && userGateway?.isConnected ? userGateway : undefined;
const browserMcpServer = browserUseEnabledGlobally
? this.browserSessionService.findMcpServer(user.id)
: undefined;
const localMcpServer = (0, composite_local_mcp_server_1.composeLocalMcpServers)(gatewayMcpServer, browserMcpServer);
if (localMcpServer) {
context.localMcpServer = localMcpServer;
}
context.permissions = this.settingsService.getPermissions();
if (this.sourceControlPreferencesService.getPreferences().branchReadOnly) {
context.permissions = (0, api_types_1.applyBranchReadOnlyOverrides)(context.permissions);
context.branchReadOnly = true;
}
context.runId = runId;
const sessionGrants = await this.loadThreadSessionGrants(threadId, user.id);
context.sessionApprovedToolKeys = sessionGrants;
const grantSessionToolApproval = async (key) => {
await this.persistThreadSessionGrant(threadId, user.id, key);
sessionGrants.add(key);
};
context.grantSessionToolApproval = grantSessionToolApproval;
const domainTracker = (0, instance_ai_1.createDomainAccessTracker)({
grantedKeys: sessionGrants,
persistGrant: grantSessionToolApproval,
});
this.domainAccessTrackersByThread.set(threadId, domainTracker);
context.domainAccessTracker = domainTracker;
if (this.isRunDebugEnabled()) {
context.recordWorkflowCodeSnapshot = (snapshot) => {
this.runDebugBuffer.ensure(runId, threadId);
this.runDebugBuffer.recordWorkflowCode(runId, snapshot);
};
}
browserMcpServer?.setDomainGate({
tracker: domainTracker,
runId,
permissionMode: context.permissions?.fetchUrl,
});
if (gatewayMcpServer || browserMcpServer) {
const capabilities = new Set();
if (gatewayMcpServer) {
for (const { name, enabled } of gatewayMcpServer.getStatus().toolCategories) {
if (enabled) {
capabilities.add(name);
}
}
}
if (browserMcpServer) {
capabilities.add(instance_ai_gateway_service_1.BROWSER_TOOL_CATEGORY);
}
context.localGatewayStatus = {
status: 'connected',
capabilities: [...capabilities],
};
}
else if (localGatewayDisabledGlobally && !browserUseEnabledGlobally) {
context.localGatewayStatus = { status: 'disabledGlobally' };
}
else {
context.localGatewayStatus = {
status: localGatewayDisabledForUser ? 'disabled' : 'disconnected',
};
}
const taskStorage = new instance_ai_1.ThreadTaskStorage(memory);
const iterationLog = this.dbIterationLogStorage;
const snapshotStorage = this.dbSnapshotStorage;
const workflowLoopStorage = new instance_ai_1.WorkflowLoopStorage(memory);
const workflowTasks = this.createWorkflowTaskServiceWithUiSync(threadId, runId, new instance_ai_1.WorkflowTaskCoordinator(threadId, workflowLoopStorage));
const plannedTaskStorage = new instance_ai_1.PlannedTaskStorage(memory);
const plannedTaskService = new instance_ai_1.PlannedTaskCoordinator(plannedTaskStorage);
const nodeDefDirs = this.adapterService.getNodeDefinitionDirs();
if (nodeDefDirs.length > 0) {
(0, workflow_sdk_1.setSchemaBaseDirs)(nodeDefDirs);
}
const flagDisabledSkillIds = (0, instance_ai_1.disabledInstanceAiSkillIds)({ configEvalsEnabled });
const allRuntimeSkills = flagDisabledSkillIds.length > 0
? (0, agents_1.filterRuntimeSkillSource)((0, instance_ai_1.loadInstanceAiRuntimeSkillSource)(), flagDisabledSkillIds)
: (0, instance_ai_1.loadInstanceAiRuntimeSkillSource)();
let runtimeSkills = allRuntimeSkills;
let runtimeWorkspace;
let workspaceRoot;
const sandboxStatus = this.settingsService.getSandboxStatus();
if (sandboxStatus.workflowBuilderAvailable) {
const sandboxConfig = await this.instanceAiErrorReporter.withBoundary('instance-ai-sandbox-setup', { threadId, runId, userId: user.id, messageGroupId }, async () => await this.sandboxService.resolveSandboxConfig(user));
if (sandboxConfig.enabled) {
workspaceRoot = (0, instance_ai_1.getPromptWorkspaceRoot)(sandboxConfig.provider);
let sandboxEntryPromise;
const getSandboxEntry = async () => {
sandboxEntryPromise ??= this.sandboxService
.getOrCreateWorkspaceEntry(threadId, user)
.catch((error) => {
sandboxEntryPromise = undefined;
throw error;
});
return await sandboxEntryPromise;
};
const getSetupSandboxEntry = async () => {
return await this.sandboxService.getOrCreateWorkspace(threadId, user, context);
};
const scopeWorkspaceForAgent = async (workspace) => {
if (!workspace)
return undefined;
const root = await (0, instance_ai_1.getWorkspaceRoot)(workspace);
return (0, instance_ai_1.createScopedWorkspace)(workspace, root);
};
runtimeWorkspace = (0, instance_ai_1.createLazyRuntimeWorkspace)({
sandboxInstructions: '',
filesystemInstructions: '',
ensureWorkspace: async () => await scopeWorkspaceForAgent((await getSetupSandboxEntry())?.workspace),
});
const runtimeSkillWorkspace = (0, instance_ai_1.createLazyRuntimeWorkspace)({
id: 'instance-ai-runtime-skill-workspace',
name: 'Instance AI runtime skill workspace',
ensureWorkspace: async () => await scopeWorkspaceForAgent((await getSandboxEntry())?.workspace),
});
runtimeSkills = (0, instance_ai_1.createLazyWorkspaceRuntimeSkillSource)({
source: allRuntimeSkills,
workspace: runtimeSkillWorkspace,
logger: this.logger,
});
}
}
context.workspace = runtimeWorkspace;
context.threadId = threadId;
context.threadMemory = memory;
context.trackTelemetry = (eventName, properties) => {
this.telemetry.track(eventName, properties);
};
const domainTools = (0, instance_ai_1.createAllTools)(context);
const orchestrationContext = {
threadId,
runId,
messageGroupId,
userId: user.id,
projectId: boundProjectId,
orchestratorAgentId: (0, instance_ai_1.orchestratorAgentId)(runId),
modelId,
checkpointStore: this.checkpointStore,
eventBus: this.eventBus,
logger: this.logger,
outputRedaction: (0, output_redaction_config_1.resolveOutputRedaction)(this.instanceAiConfig),
trackTelemetry: (eventName, properties) => {
this.telemetry.track(eventName, properties);
},
claimSubAgentUsage: async (dedupeId, usage, status) => {
try {
await this.creditService.claimRunUsage(user, threadId, dedupeId, usage, status);
}
catch (error) {
this.instanceAiErrorReporter.report(error, {
component: 'instance-ai-agent-builder-usage',
threadId,
runId,
userId: user.id,
...(boundProjectId ? { projectId: boundProjectId } : {}),
...(messageGroupId ? { messageGroupId } : {}),
});
this.logger.warn('Failed to claim agent-builder usage', {
threadId,
runId,
dedupeId,
error: getErrorMessage(error),
});
}
},
domainTools,
abortSignal,
taskStorage,
timeZone: this.defaultTimeZone,
localMcpServer: context.localMcpServer,
runtimeSkills,
runtimeSkillCatalog: allRuntimeSkills,
oauth2CallbackUrl: this.oauth2CallbackUrl,
webhookBaseUrl: this.webhookBaseUrl,
formBaseUrl: this.formBaseUrl,
waitForConfirmation: async (requestId) => {
this.runState.touchActiveRun(threadId);
return await new Promise((resolve) => {
this.runState.registerPendingConfirmation(requestId, {
resolve,
threadId,
userId: user.id,
createdAt: Date.now(),
});
void this.suspendedThreads.persistPendingConfirmation({
requestId,
threadId,
userId: user.id,
runId,
messageGroupId,
kind: 'inline',
});
queueMicrotask(() => {
void this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage);
});
});
},
cancelBackgroundTask: async (taskId) => this.cancelBackgroundTask(threadId, taskId),
spawnBackgroundTask: (opts) => this.spawnBackgroundTask(runId, opts, snapshotStorage, messageGroupId),
touchRun: () => this.runState.touchActiveRun(threadId),
touchBackgroundTask: (taskId) => this.backgroundTasks.touchTask(threadId, taskId),
plannedTaskService,
schedulePlannedTasks: async () => await this.schedulePlannedTasks(user, threadId),
iterationLog,
sendCorrectionToTask: (taskId, correction) => this.sendCorrectionToTask(threadId, taskId, correction),
workflowTaskService: workflowTasks,
workspace: runtimeWorkspace,
workspaceRoot,
nodeDefinitionDirs: nodeDefDirs.length > 0 ? nodeDefDirs : undefined,
domainContext: context,
tracingProxyConfig,
memory,
};
return {
context,
memory,
taskStorage,
iterationLog,
snapshotStorage,
workflowTasks,
plannedTaskService,
modelId,
orchestrationContext,
};
}
getAgentExecutionService() {
if (!di_1.Container.get(backend_common_1.ModuleRegistry).isActive('agents'))
return null;
try {
return di_1.Container.get(agent_execution_service_1.AgentExecutionService);
}
catch {
return null;
}
}
async bindAgentPreviewSession(context, user) {
await (0, instance_ai_1.resolveAgentPreviewSession)(context);
const projectId = context.projectId;
if (!context.agentPreviewSession || !projectId)
return;
if (!(await this.canAccessAgentPreviewHandoff(user, projectId))) {
context.agentPreviewSession = undefined;
return;
}
context.resolvePreviewSession = async (ref) => {
const service = this.getAgentExecutionService();
if (!service)
return null;
const detail = await service.getThreadDetail(ref.threadId, projectId, ref.agentId);
if (!detail)
return null;
const transcript = (0, format_preview_context_1.formatPreviewSessionContext)(detail.thread, detail.executions, ref.executionId);
if (transcript === null)
return null;
return {
title: detail.thread.title?.trim() || `Session #${detail.thread.sessionNumber}`,
sessionNumber: detail.thread.sessionNumber,
transcript,
};
};
}
async dispatchPlannedTask(task, context, _graph) {
if (task.kind === 'build-workflow' || task.kind === 'checkpoint') {
this.logger.warn('dispatchPlannedTask called for a runtime planned-task kind', {
threadId: context.threadId,
taskId: task.id,
kind: task.kind,
});
return;
}
await context.plannedTaskService?.markFailed(context.threadId, task.id, {
error: `Planned task kind "${task.kind}" is no longer supported`,
});
const nextGraph = await context.plannedTaskService?.getGraph(context.threadId);
if (nextGraph) {
await this.syncPlannedTasksToUi(context.threadId, nextGraph);
}
}
collectWorkflowIds(value, workflowIds) {
if (value === null || value === undefined || typeof value !== 'object')
return;
if (Array.isArray(value)) {
for (const item of value) {
this.collectWorkflowIds(item, workflowIds);
}
return;
}
for (const [key, child] of Object.entries(value)) {
if (key === 'workflowId' && typeof child === 'string' && child.length > 0) {
workflowIds.add(child);
continue;
}
if (key === 'supportingWorkflowIds' && Array.isArray(child)) {
for (const workflowId of child) {
if (typeof workflowId === 'string' && workflowId.length > 0) {
workflowIds.add(workflowId);
}
}
continue;
}
this.collectWorkflowIds(child, workflowIds);
}
}
getBuildTaskWorkflowName(task) {
if (task.kind !== 'build-workflow')
return undefined;
const titleMatch = task.title.match(/^Build '(.+)' workflow$/) ?? task.title.match(/^Build "(.+)" workflow$/);
return titleMatch?.[1];
}
checkpointRequiresRunApproval(graph, _checkpoint) {
return graph.postBuildRunApprovalRequired === true;
}
async getCheckpointRunPolicy(threadId, checkpointTaskId) {
try {
const { plannedTaskService } = await this.createPlannedTaskState();
const graph = await plannedTaskService.getGraph(threadId);
const checkpoint = graph?.tasks.find((t) => t.id === checkpointTaskId);
if (!graph || !checkpoint) {
return {
allowedWorkflowIds: new Set(),
allowedWorkflowNames: new Set(),
requireApproval: false,
};
}
const deps = new Set(checkpoint.deps);
const ids = new Set();
const names = new Set();
for (const task of graph.tasks) {
if (!deps.has(task.id))
continue;
const workflowName = this.getBuildTaskWorkflowName(task);
if (workflowName) {
names.add(workflowName);
}
if (task.workflowId) {
ids.add(task.workflowId);
}
this.collectWorkflowIds(task.outcome, ids);
}
const tracing = this.tracing.getTraceContextForContinuation(threadId);
for (const workflowId of [...ids]) {
const remappedWorkflowId = tracing?.idRemapper?.remapOutput(workflowId);
if (typeof remappedWorkflowId === 'string' && remappedWorkflowId.length > 0) {
ids.add(remappedWorkflowId);
}
}
return {
allowedWorkflowIds: ids,
allowedWorkflowNames: names,
requireApproval: this.checkpointRequiresRunApproval(graph, checkpoint),
};
}
catch (error) {
this.logger.warn('Failed to resolve checkpoint allowed workflow IDs', {
threadId,
checkpointTaskId,
error: error instanceof Error ? error.message : String(error),
});
return {
allowedWorkflowIds: new Set(),
allowedWorkflowNames: new Set(),
requireApproval: false,
};
}
}
async handlePlannedTaskSettlement(user, task, status) {
if (!task.plannedTaskId)
return;
const { plannedTaskService } = await this.createPlannedTaskState();
let graph = null;
if (status === 'succeeded') {
graph = await plannedTaskService.markSucceeded(task.threadId, task.plannedTaskId, {
result: task.result,
outcome: task.outcome,
});
}
else if (status === 'failed') {
graph = await plannedTaskService.markFailed(task.threadId, task.plannedTaskId, {
error: task.error,
});
}
else {
graph = await plannedTaskService.markCancelled(task.threadId, task.plannedTaskId, {
error: task.error,
});
}
if (graph) {
await this.syncPlannedTasksToUi(task.threadId, graph);
}
await this.schedulePlannedTasks(user, task.threadId);
}
async maybeStartWorkflowVerificationFollowUp(user, task) {
if (task.role !== 'workflow-builder' || !task.workItemId)
return false;
const obligation = await this.workflowObligations.getObligation(task.threadId, task.workItemId, {
source: task.plannedTaskId ? 'planned' : 'direct',
plannedTaskId: task.plannedTaskId,
});
if (!obligation)
return false;
this.trackWorkflowVerificationObligation(obligation, 'background_task_settled');
if (obligation.status !== 'ready_to_verify' && obligation.status !== 'verifying') {
return false;
}
const outcome = (0, workflow_verification_obligation_service_1.parseWorkflowBuildOutcome)(task.outcome);
const startedRunId = await this.startInternalFollowUpRun(user, task.threadId, this.buildWorkflowVerificationFollowUpMessage({
obligation,
outcome,
sourceTask: {
taskId: task.taskId,
role: task.role,
status: task.status,
result: task.result,
error: task.error,
plannedTaskId: task.plannedTaskId,
workItemId: task.workItemId,
},
}), task.messageGroupId, false, undefined, 'workflow_verification');
this.trackWorkflowVerificationObligation(obligation, 'follow_up_start_attempted', {
follow_up_started: startedRunId.length > 0,
});
return startedRunId.length > 0;
}
buildWorkflowSetupFollowUpMessage(obligation) {
const payload = {
workflowId: obligation.workflowId,
workItemId: obligation.workItemId,
setupRequirement: obligation.setupRequirement,
verificationReadiness: obligation.readiness,
};
return `<workflow-setup-required>\n${JSON.stringify(payload, null, 2)}\n</workflow-setup-required>\n\n${internal_messages_1.AUTO_FOLLOW_UP_MESSAGE}`;
}
getWorkflowSetupSuspensionWorkflowId(toolName, suspendPayload) {
if (toolName !== 'workflows' || !suspendPayload)
return undefined;
if (!Array.isArray(suspendPayload.setupRequests))
return undefined;
return typeof suspendPayload.workflowId === 'string' ? suspendPayload.workflowId : undefined;
}
async markWorkflowSetupHandled(threadId, workflowId, runId) {
const records = await this.listWorkflowLoopRecords(threadId);
if (records.length === 0)
return false;
const candidates = [];
for (const record of records) {
if (record.state.setupRoutedAt)
continue;
if (this.workflowObligations.isPlannedRecord(record))
continue;
const obligation = this.workflowObligations.obligationFromRecord(threadId, record, {
source: 'direct',
});
if (obligation.workflowId !== workflowId)
continue;
if (obligation.setupRequirement?.status !== 'required')
continue;
candidates.push({ record, obligation });
}
const sameRunCandidates = runId
? candidates.filter(({ obligation, record }) => obligation.runId === runId || record.state.runId === runId)
: [];
const fallbackCandidates = candidates.filter((candidate) => !sameRunCandidates.includes(candidate));
for (const { record, obligation } of [...sameRunCandidates, ...fallbackCandidates]) {
const claim = await this.claimWorkItemSetupRouting(threadId, record);
if (!claim)
continue;
const marked = await this.markWorkItemSetupRouted(threadId, record.state.workItemId, claim.claimId);
if (!marked) {
await this.releaseWorkItemSetupRoutingClaim(threadId, record.state.workItemId, claim.claimId);
this.logger.warn('Workflow setup completed but routing marker was not saved', {
threadId,
workItemId: record.state.workItemId,
workflowId,
});
continue;
}
this.trackWorkflowVerificationObligation(obligation, 'setup_completed_by_tool');
return true;
}
return false;
}
async maybeStartWorkflowSetupFollowUp(user, threadId) {
const records = await this.listWorkflowLoopRecords(threadId);
if (records.length === 0)
return false;
for (const record of records) {
if (record.state.setupRoutedAt)
continue;
if (this.workflowObligations.isPlannedRecord(record))
continue;
const obligation = this.workflowObligations.obligationFromRecord(threadId, record, {
source: 'direct',
});
const verificationConcluded = obligation.status === 'verified' ||
obligation.status === 'needs_setup' ||
obligation.status === 'not_verifiable';
if (!verificationConcluded)
continue;
if (obligation.setupRequirement?.status !== 'required' || !obligation.workflowId)
continue;
const claim = await this.claimWorkItemSetupRouting(threadId, record);
if (!claim)
continue;
const startedRunId = await this.startInternalFollowUpRun(user, threadId, this.buildWorkflowSetupFollowUpMessage(obligation), this.runState.getMessageGroupId(threadId), false, undefined, 'workflow_setup');
if (startedRunId.length === 0) {
await this.releaseWorkItemSetupRoutingClaim(threadId, record.state.workItemId, claim.claimId);
return false;
}
const marked = await this.markWorkItemSetupRouted(threadId, record.state.workItemId, claim.claimId);
if (!marked) {
this.logger.warn('Workflow setup follow-up started but routing marker was not saved', {
threadId,
workItemId: record.state.workItemId,
});
}
this.trackWorkflowVerificationObligation(obligation, 'setup_follow_up_started');
return true;
}
return false;
}
async listWorkflowLoopRecords(threadId) {
return await new instance_ai_1.WorkflowLoopStorage(this.agentMemory).listWorkItems(threadId);
}
createWorkflowSetupRoutingClaim() {
const claimedAt = new Date();
const expiresAt = new Date(claimedAt.getTime() + WORKFLOW_SETUP_ROUTING_CLAIM_TTL_MS);
return {
claimId: `setup:${(0, nanoid_1.nanoid)()}`,
claimedAt: claimedAt.toISOString(),
expiresAt: expiresAt.toISOString(),
};
}
async claimWorkItemSetupRouting(threadId, record) {
const claim = this.createWorkflowSetupRoutingClaim();
const claimed = await new instance_ai_1.WorkflowLoopStorage(this.agentMemory).claimSetupRouting(threadId, record.state.workItemId, claim);
return claimed ? claim : null;
}
async markWorkItemSetupRouted(threadId, workItemId, claimId) {
return await new instance_ai_1.WorkflowLoopStorage(this.agentMemory).markSetupRouted(threadId, workItemId, claimId, new Date().toISOString());
}
async releaseWorkItemSetupRoutingClaim(threadId, workItemId, claimId) {
await new instance_ai_1.WorkflowLoopStorage(this.agentMemory).releaseSetupRoutingClaim(threadId, workItemId, claimId);
}
updateInternalFollowUpFailureStreak(threadId, status, isInternalFollowUp) {
if (status === 'completed' || status === 'suspended') {
this.failedInternalFollowUpStreaks.delete(threadId);
return;
}
if (status === 'error' && isInternalFollowUp) {
this.failedInternalFollowUpStreaks.set(threadId, (this.failedInternalFollowUpStreaks.get(threadId) ?? 0) + 1);
}
}
async startInternalFollowUpRun(user, threadId, message, messageGroupId, isReplanFollowUp = false, checkpoint, resumeReasonOverride, plannedBuild) {
if (this.runState.hasLiveRun(threadId)) {
this.logger.warn('Skipping internal follow-up: active run exists', { threadId });
return '';
}
const failedStreak = this.failedInternalFollowUpStreaks.get(threadId) ?? 0;
if (failedStreak >= MAX_CONSECUTIVE_FAILED_INTERNAL_FOLLOW_UPS) {
this.logger.warn('Skipping internal follow-up: consecutive follow-up runs keep failing', {
threadId,
failedStreak,
resumeReason: resumeReasonOverride,
});
return '';
}
const { runId, abortController } = this.runState.startRun({
threadId,
user,
messageGroupId,
});
const timeZone = this.runState.getTimeZone(threadId) ?? this.defaultTimeZone;
const resumeReason = resumeReasonOverride ??
(checkpoint
? 'planned_checkpoint'
: isReplanFollowUp
? 'replan'
: 'background_task_completed');
this.startExecuteRun(user, threadId, runId, message, abortController, undefined, undefined, messageGroupId, timeZone, isReplanFollowUp, checkpoint, resumeReason, plannedBuild);
return runId;
}
async schedulePlannedTasks(user, threadId) {
const prev = this.schedulerLocks.get(threadId) ?? Promise.resolve();
const current = prev.then(() => this.doSchedulePlannedTasks(user, threadId)).catch(() => { });
this.schedulerLocks.set(threadId, current);
await current;
}
createPlannedTaskActionRunner(activeUser, threadId, plannedTaskService) {
const scope = { user: activeUser, threadId };
return new planned_task_action_runner_1.PlannedTaskActionRunner({
scope,
plannedTaskService,
logger: this.logger,
view: this.createPlannedTaskView(),
runGate: this.createPlannedTaskRunGate(),
dispatcher: this.createPlannedTaskDispatcher(),
followUps: this.createPlannedTaskFollowUps(),
workflowVerificationGate: this.createPlannedWorkflowVerificationGate(threadId),
workflowVerificationTracker: this.createPlannedWorkflowVerificationTracker(),
});
}
createPlannedTaskView() {
return {
sync: async (scope, graph) => await this.syncPlannedTasksToUi(scope.threadId, graph),
};
}
createPlannedTaskRunGate() {
return {
hasLiveRun: (threadId) => this.runState.hasLiveRun(threadId),
};
}
createPlannedTaskDispatcher() {
return {
dispatch: async ({ scope, graph, tasks }) => {
const context = await this.createPlannedTaskDispatchContext(scope.user, scope.threadId, graph);
for (const task of tasks) {
await this.dispatchPlannedTask(task, context, graph);
}
},
};
}
createPlannedTaskFollowUps() {
return {
startReplan: async ({ scope, graph, failedTask }) => await this.startInternalFollowUpRun(scope.user, scope.threadId, this.buildPlannedTaskFollowUpMessage('replan', graph, { failedTask }), graph.messageGroupId, true, undefined, undefined, undefined),
startWorkflowVerification: async ({ scope, graph, verification }) => await this.startInternalFollowUpRun(scope.user, scope.threadId, this.buildWorkflowVerificationFollowUpMessage({
obligation: verification.obligation,
outcome: verification.outcome,
sourceTask: this.toWorkflowVerificationSourceTask(verification),
}), graph.messageGroupId, false, undefined, 'workflow_verification', undefined),
startSynthesis: async ({ scope, graph }) => await this.startInternalFollowUpRun(scope.user, scope.threadId, this.buildPlannedTaskFollowUpMessage('synthesize', graph), graph.messageGroupId, false, undefined, 'synthesize', undefined),
startWorkflowBuild: async ({ scope, graph, task, workItemId }) => {
const plannedBuild = {
isPlannedBuildFollowUp: true,
buildTaskId: task.id,
workItemId,
isSupportingWorkflowTask: task.isSupportingWorkflow === true,
};
return await this.startInternalFollowUpRun(scope.user, scope.threadId, this.buildPlannedTaskFollowUpMessage('build-workflow', graph, { buildTask: task }), graph.messageGroupId, false, undefined, undefined, plannedBuild);
},
startCheckpoint: async ({ scope, graph, task }) => await this.startInternalFollowUpRun(scope.user, scope.threadId, this.buildPlannedTaskFollowUpMessage('checkpoint', graph, { checkpoint: task }), graph.messageGroupId, false, { isCheckpointFollowUp: true, checkpointTaskId: task.id }, undefined, undefined),
};
}
createPlannedWorkflowVerificationGate(threadId) {
return {
revalidate: async (verification) => await this.workflowObligations.revalidatePlannedWorkflowVerification(threadId, verification),
};
}
toWorkflowVerificationSourceTask(verification) {
return {
taskId: verification.task.backgroundTaskId ?? verification.task.id,
role: 'workflow-builder',
status: 'completed',
result: verification.task.result,
error: verification.task.error,
plannedTaskId: verification.task.id,
workItemId: verification.obligation.workItemId,
};
}
createPlannedWorkflowVerificationTracker() {
return {
scheduled: ({ obligation }) => this.trackWorkflowVerificationObligation(obligation, 'planned_verification_scheduled'),
followUpStartAttempted: ({ obligation }, started) => this.trackWorkflowVerificationObligation(obligation, 'follow_up_start_attempted', {
follow_up_started: started,
}),
};
}
async createPlannedTaskDispatchContext(user, threadId, graph) {
const environment = await this.createExecutionEnvironment(user, threadId, graph.planRunId, createInertAbortSignal(), graph.messageGroupId, this.threadPushRef.get(threadId));
environment.orchestrationContext.tracing = this.tracing.getTraceContext(graph.planRunId);
return environment.orchestrationContext;
}
async doSchedulePlannedTasks(user, threadId) {
const revalidated = await this.revalidateActiveUser(user.id);
if (!revalidated) {
this.logger.warn('Cancelling run: user no longer authorized for AI Assistant', {
userId: user.id,
threadId,
});
this.cancelRun(threadId);
return;
}
const activeUser = revalidated;
const { plannedTaskService } = await this.createPlannedTaskState();
const actionRunner = this.createPlannedTaskActionRunner(activeUser, threadId, plannedTaskService);
while (true) {
const graph = await plannedTaskService.getGraph(threadId);
if (!graph)
return;
await this.syncPlannedTasksToUi(threadId, graph);
const availableSlots = Math.max(0, MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD -
this.backgroundTasks.getRunningTasks(threadId).length);
const pendingWorkflowVerification = await this.workflowObligations.findPendingPlannedWorkflowVerification(threadId, graph);
const action = await plannedTaskService.tick(threadId, {
availableSlots,
pendingWorkflowVerification,
});
if (action.type === 'none')
return;
const result = await actionRunner.run(action);
if (result.type !== 'continue-scheduling')
return;
}
}
async persistInterruptedUserMessage(threadId, userId, message, createdAt) {
if (!message)
return;
try {
await this.agentMemory.saveMessages({
threadId,
resourceId: userId,
messages: [
{
id: (0, nanoid_1.nanoid)(),
createdAt,
type: 'llm',
role: 'user',
content: [{ type: 'text', text: message }],
},
],
});
}
catch (error) {
this.logger.warn('Failed to persist user message on cancel', {
threadId,
error: error instanceof Error ? error.message : String(error),
});
}
}
async executeRun(user, threadId, runId, message, abortController, attachments, handoffContext, messageGroupId, timeZone, isReplanFollowUp = false, checkpoint, resumeReason, plannedBuild) {
const fileAttachments = (attachments ?? []).filter((attachment) => attachment.type === 'file');
const workflowAttachments = (attachments ?? []).filter((attachment) => attachment.type === 'workflow');
const agentAttachments = (attachments ?? []).filter((attachment) => attachment.type === 'agent');
const contextAttachments = [
...workflowAttachments,
...agentAttachments,
];
const signal = abortController.signal;
let tracing;
let messageTraceFinalization;
let aiCreatedWorkflowIds;
let activeSnapshotStorage;
let messageId = '';
let streamReached = false;
const turnStartedAt = new Date();
try {
this.instanceAiErrorReporter.beginRun(runId);
messageId = (0, nanoid_1.nanoid)();
const traceInput = {
message,
...(fileAttachments.length
? {
attachments: fileAttachments.map((attachment) => ({
mimeType: attachment.mimeType,
size: attachment.data.length,
})),
}
: {}),
...(messageGroupId ? { messageGroupId } : {}),
};
const proxyRunConfig = await this.createProxyRunConfig(user);
if (resumeReason) {
tracing = await this.tracing.createOrchestratorResumeTraceContext({
threadId,
messageId,
messageGroupId,
runId,
userId: user.id,
input: traceInput,
resumeReason,
metadata: {
...(checkpoint?.isCheckpointFollowUp
? { checkpoint_task_id: checkpoint.checkpointTaskId }
: {}),
...(plannedBuild?.isPlannedBuildFollowUp
? { build_task_id: plannedBuild.buildTaskId }
: {}),
},
});
}
else {
tracing = await (0, instance_ai_1.createInstanceAiTraceContext)({
threadId,
messageId,
messageGroupId,
runId,
userId: user.id,
input: traceInput,
proxyConfig: proxyRunConfig.tracingProxyConfig,
n8nVersion: constants_1.N8N_VERSION,
workflowSdkVersion: constants_1.WORKFLOW_SDK_VERSION,
});
}
if (this.isRunDebugEnabled()) {
this.runDebugBuffer.ensure(runId, threadId, (0, instance_ai_1.buildRunDebugLabel)({ message, resumeReason }));
}
const traceId = tracing?.rootRun.otelTraceId;
this.eventBus.publish(threadId, {
type: 'run-start',
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
userId: user.id,
payload: { messageId, messageGroupId, ...(traceId ? { traceId } : {}) },
});
if (signal.aborted) {
await this.persistInterruptedUserMessage(threadId, user.id, message, turnStartedAt);
await this.terminalOutcome.evaluateTerminalResponse(threadId, runId, 'cancelled', {
messageGroupId,
correlationId: messageId,
});
this.eventBus.publish(threadId, {
type: 'run-finish',
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
payload: { status: 'cancelled', reason: 'user_cancelled' },
});
return;
}
const executionPushRef = this.threadPushRef.get(threadId);
const environment = await this.createExecutionEnvironment(user, threadId, runId, signal, messageGroupId, executionPushRef, proxyRunConfig);
activeSnapshotStorage = environment.snapshotStorage;
const { context, memory, taskStorage, snapshotStorage, workflowTasks, plannedTaskService, modelId, orchestrationContext, } = environment;
aiCreatedWorkflowIds = context.aiCreatedWorkflowIds ??= new Set();
const isPostPlanFollowUp = isReplanFollowUp || checkpoint?.isCheckpointFollowUp === true;
orchestrationContext.currentUserMessage = message;
orchestrationContext.isReplanFollowUp = isReplanFollowUp;
orchestrationContext.timeZone = timeZone ?? this.defaultTimeZone;
if (checkpoint?.isCheckpointFollowUp) {
orchestrationContext.isCheckpointFollowUp = true;
orchestrationContext.checkpointTaskId = checkpoint.checkpointTaskId;
context.permissions = {
...context.permissions,
...(instance_ai_1.PLANNED_TASK_PERMISSION_OVERRIDES.checkpoint ?? {}),
};
const runPolicy = await this.getCheckpointRunPolicy(threadId, checkpoint.checkpointTaskId);
context.allowedRunWorkflowIds = runPolicy.allowedWorkflowIds;
context.allowedRunWorkflowNames = runPolicy.allowedWorkflowNames;
context.requireRunWorkflowApproval = runPolicy.requireApproval;
}
if (plannedBuild?.isPlannedBuildFollowUp) {
context.permissions = {
...context.permissions,
...(instance_ai_1.PLANNED_TASK_PERMISSION_OVERRIDES['build-workflow'] ?? {}),
};
context.workflowBuildContext = {
threadId,
runId,
taskId: plannedBuild.buildTaskId,
workItemId: plannedBuild.workItemId,
allowPostPlanWorkflowCreate: true,
isSupportingWorkflowTask: plannedBuild.isSupportingWorkflowTask,
plannedTaskService,
workflowTaskService: workflowTasks,
onBuildOutcome: (outcome) => {
plannedBuild.savedOutcome = outcome;
},
};
}
else {
context.workflowBuildContext = {
threadId,
runId,
taskId: `build-${runId}`,
workItemId: `wi_${(0, nanoid_1.nanoid)(8)}`,
allowPostPlanWorkflowCreate: isPostPlanFollowUp,
workflowTaskService: workflowTasks,
};
}
if (fileAttachments.length > 0) {
context.currentUserAttachments = fileAttachments;
}
if (!tracing && process.env.E2E_TESTS === 'true') {
const { createTraceReplayOnlyContext } = await (0, lazy_import_1.lazyImport)(async () => await import('@n8n/instance-ai'));
tracing = createTraceReplayOnlyContext();
}
if (tracing) {
orchestrationContext.tracing = tracing;
if (this.tracing.getTraceContext(runId) !== tracing) {
await this.tracing.configureTraceReplayMode(tracing);
this.runState.attachTracing(threadId, tracing);
this.tracing.storeTraceContext(runId, threadId, tracing, messageGroupId);
}
}
const enrichedMessage = await this.buildMessageWithRunningTasks(threadId, message);
const contextResourcesBlock = buildContextResourcesBlock(contextAttachments);
let handoffContextBlock = '';
let agentPreviewTitleFallback;
if (handoffContext?.source === 'agent-preview') {
const projectId = context.projectId;
if (!projectId) {
throw new n8n_workflow_1.UnexpectedError(`Instance AI thread "${threadId}" has no bound project; agent-preview handoff requires a project`);
}
await this.assertAgentPreviewHandoffScopes(user, projectId);
const agentExecutionService = this.getAgentExecutionService();
if (!agentExecutionService) {
throw new n8n_workflow_1.UserError('Agent preview handoff is not available');
}
const resolved = await (0, agent_preview_handoff_1.resolveAgentPreviewHandoff)(handoffContext, {
projectId,
getThreadDetail: agentExecutionService.getThreadDetail.bind(agentExecutionService),
});
handoffContextBlock = resolved.block;
agentPreviewTitleFallback = resolved.titleFallback;
context.agentBuilderTarget = resolved.target;
context.agentPreviewSession = {
agentId: handoffContext.agentId,
threadId: handoffContext.threadId,
...(handoffContext.executionId ? { executionId: handoffContext.executionId } : {}),
};
await (0, instance_ai_1.saveAgentBuilderTarget)(context, resolved.target, {
previewSession: context.agentPreviewSession,
});
}
else {
handoffContextBlock = buildHandoffContextBlock(handoffContext);
}
const thread = await memory.getThread(threadId);
if (thread && !thread.title) {
const handoffTitle = contextAttachments.find((attachment) => attachment.name)?.name ??
agentPreviewTitleFallback;
await (0, instance_ai_1.patchThread)(memory, {
threadId,
update: ({ metadata }) => handoffTitle
? {
title: (0, instance_ai_1.truncateToTitle)(handoffTitle),
metadata: { ...metadata, titleRefined: true },
}
: { title: (0, instance_ai_1.truncateToTitle)(message) },
});
}
const existingTasks = await taskStorage.get(threadId);
if (existingTasks) {
this.eventBus.publish(threadId, {
type: 'tasks-update',
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
payload: { tasks: existingTasks },
});
}
let nonStructuredAttachments = [];
let attachmentManifest = '';
let hasParseableAttachment = false;
if (fileAttachments.length > 0) {
const classifiedAttachments = (0, instance_ai_1.classifyAttachments)(fileAttachments);
nonStructuredAttachments = fileAttachments.filter((attachment) => !(0, instance_ai_1.isParseableAttachment)(attachment));
hasParseableAttachment = classifiedAttachments.some((attachment) => attachment.parseable);
attachmentManifest = (0, instance_ai_1.buildAttachmentManifest)(classifiedAttachments);
}
const messageBody = !message && hasParseableAttachment
? `The user attached file(s) without a message. Inspect the first parseable attachment with parse-file and provide a concise summary.\n\n${attachmentManifest}`
: attachmentManifest
? `${enrichedMessage}\n\n${attachmentManifest}`
: enrichedMessage;
const messageWithContext = [contextResourcesBlock, handoffContextBlock, messageBody]
.filter(Boolean)
.join('\n\n');
const fullMessage = (0, internal_messages_1.withCurrentDateTime)(messageWithContext, (0, instance_ai_1.getDateTimeSection)(timeZone ?? this.defaultTimeZone));
const promptBuildRun = tracing
? await tracing.startChildRun(tracing.messageRun, {
name: 'prepare: prompt',
canonicalName: 'instance-ai.prompt_build',
tags: ['prompt'],
metadata: { agent_role: 'prompt_build' },
inputs: {
message,
attachmentCount: attachments?.length ?? 0,
},
})
: undefined;
let streamInput;
try {
if (nonStructuredAttachments.length > 0) {
const baseContent = [
{ type: 'text', text: fullMessage },
...nonStructuredAttachments.map((attachment) => ({
type: 'file',
data: attachment.data,
mediaType: attachment.mimeType,
})),
];
streamInput = [
{
role: 'user',
content: baseContent,
},
];
}
else {
streamInput = fullMessage;
}
if (promptBuildRun && tracing) {
const traceOutput = typeof streamInput === 'string'
? { fullMessage: streamInput }
: {
fullMessage,
attachmentCount: attachments?.length ?? 0,
nonStructuredAttachmentCount: nonStructuredAttachments.length,
};
await tracing.finishRun(promptBuildRun, {
outputs: traceOutput,
metadata: { final_status: 'completed' },
});
}
}
catch (error) {
if (promptBuildRun && tracing) {
await tracing.failRun(promptBuildRun, error, {
final_status: 'error',
});
}
throw error;
}
if (tracing && tracing.actorRun.id === tracing.rootRun.id) {
const actorRun = await tracing.startChildRun(tracing.rootRun, {
name: 'agent: orchestrator',
canonicalName: 'instance-ai.agent.orchestrator',
tags: ['orchestrator'],
metadata: {
agent_role: 'orchestrator',
agent_id: (0, instance_ai_1.orchestratorAgentId)(runId),
execution_mode: 'foreground',
trace_kind: tracing.traceKind,
},
inputs: traceInput,
});
tracing.actorRun = actorRun;
tracing.orchestratorRun = actorRun;
}
const runControl = (0, instance_ai_1.createOrchestratorRunControl)(orchestrationContext);
const stopSignal = () => runControl.getStopSignal();
const agent = await this.createAgentFromEnvironment(environment, threadId, runId, user, tracing);
const streamOptions = this.buildOrchestratorAgentStreamOptions(user, threadId, runId, signal);
streamReached = true;
const result = tracing
? await tracing.withActiveSpan(tracing.actorRun, async () => {
return await (0, instance_ai_1.streamAgentRun)(agent, streamInput, streamOptions, {
threadId,
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
signal,
eventBus: this.eventBus,
logger: this.logger,
onActivity: () => this.runState.touchActiveRun(threadId),
stopSignal,
outputRedaction: (0, output_redaction_config_1.resolveOutputRedaction)(this.instanceAiConfig),
});
})
: await (0, instance_ai_1.streamAgentRun)(agent, streamInput, streamOptions, {
threadId,
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
signal,
eventBus: this.eventBus,
logger: this.logger,
onActivity: () => this.runState.touchActiveRun(threadId),
stopSignal,
outputRedaction: (0, output_redaction_config_1.resolveOutputRedaction)(this.instanceAiConfig),
});
if (result.status === 'suspended') {
this.emitRunMetrics(threadId, 'suspended', {
modelId,
workSummary: result.workSummary,
usage: result.usage,
});
if (result.suspension) {
this.runState.suspendRun(threadId, {
runId,
agentRunId: result.agentRunId,
agent,
orchestrationContext,
threadId,
user,
toolCallId: result.suspension.toolCallId,
...(result.suspension.toolName ? { toolName: result.suspension.toolName } : {}),
...(result.suspension.suspendPayload
? { suspendPayload: result.suspension.suspendPayload }
: {}),
requestId: result.suspension.requestId,
abortController,
messageGroupId,
createdAt: Date.now(),
tracing,
modelId,
checkpoint,
plannedBuild,
runHandoff: runControl.state,
});
void this.suspendedThreads.persistPendingConfirmation({
requestId: result.suspension.requestId,
threadId,
userId: user.id,
runId,
messageGroupId,
kind: 'suspended',
toolCallId: result.suspension.toolCallId,
checkpointKey: result.agentRunId,
checkpointTaskId: checkpoint?.checkpointTaskId,
});
void this.creditService.claimRunUsage(user, threadId, `${result.agentRunId || runId}:${result.suspension.requestId}`, result.usage?.usage ?? [], 'suspended');
}
const intermediateText = await (result.text ?? Promise.resolve(''));
if (intermediateText) {
this.telemetry.track('Builder sent message', {
thread_id: threadId,
message: intermediateText,
is_intermediate: true,
});
}
const waitingDecision = await this.terminalOutcome.evaluateWaitingResponse(threadId, runId, result.confirmationEvent, {
messageGroupId,
correlationId: messageId,
});
if (waitingDecision?.reason === 'confirmation-invalid') {
messageTraceFinalization = await this.terminalOutcome.finishInvalidConfirmationRun({
threadId,
runId,
abortController,
snapshotStorage,
tracing,
});
return;
}
if (result.confirmationEvent) {
this.trackConfirmationRequest(threadId, result.confirmationEvent);
this.eventBus.publish(threadId, result.confirmationEvent);
}
await this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage);
const suspensionOutputs = buildSuspensionTraceOutputs(runId, result.suspension);
await this.tracing.finalizeRunTracing(runId, tracing, {
status: 'suspended',
outputs: suspensionOutputs,
metadata: {
completion_source: 'orchestrator',
...(result.suspension?.requestId ? { request_id: result.suspension.requestId } : {}),
...(result.suspension?.toolCallId
? { pending_tool_call_id: result.suspension.toolCallId }
: {}),
...(result.suspension?.toolName
? { pending_tool_name: result.suspension.toolName }
: {}),
},
});
messageTraceFinalization = {
status: 'suspended',
outputs: suspensionOutputs,
metadata: {
completion_source: 'orchestrator',
...(result.suspension?.requestId ? { request_id: result.suspension.requestId } : {}),
...(result.suspension?.toolCallId
? { pending_tool_call_id: result.suspension.toolCallId }
: {}),
...(result.suspension?.toolName
? { pending_tool_name: result.suspension.toolName }
: {}),
},
};
return;
}
if (result.status === 'cancelled' && this.shouldPreserveHitlOnShutdown(runId)) {
return;
}
const outputText = await (result.text ?? Promise.resolve(''));
const terminalError = result.status === 'errored'
? await this.reclassifyMaskedStreamFailure(result.error, user, { threadId, runId })
: undefined;
if (result.status === 'errored') {
this.instanceAiErrorReporter.report(terminalError ?? new Error('Instance AI stream errored'), {
component: 'instance-ai-stream',
threadId,
runId,
tracing,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
userId: user.id,
messageGroupId,
messageId,
});
}
const userFacingErrorMessage = result.status === 'errored' ? getUserFacingErrorMessage(terminalError) : undefined;
const userFacingErrorCode = result.status === 'errored' ? getUserFacingErrorCode(terminalError) : undefined;
if (runControl.shouldEmitTerminalOutcome(result.stopReason)) {
await this.terminalOutcome.evaluateTerminalResponse(threadId, runId, result.status, {
messageGroupId,
correlationId: messageId,
workSummary: result.workSummary,
errorMessage: userFacingErrorMessage,
errorCode: userFacingErrorCode,
suppressCompletedFallback: checkpoint?.isCheckpointFollowUp === true ||
plannedBuild?.isPlannedBuildFollowUp === true,
});
}
const finalStatus = result.status === 'errored' ? 'error' : result.status;
await this.tracing.finalizeRunTracing(runId, tracing, {
status: finalStatus,
outputText,
modelId,
});
messageTraceFinalization = {
status: finalStatus,
outputText,
modelId,
metadata: this.tracing.buildMessageTraceMetadata(threadId, runId, { status: finalStatus }),
};
const archivedWorkflowIds = await this.temporaryWorkflowService.reapForRun(threadId, user, aiCreatedWorkflowIds, this.backgroundTasks.getRunningTasks(threadId).length);
await this.finalizeRun(threadId, runId, result.status, snapshotStorage, {
userId: user.id,
modelId,
archivedWorkflowIds,
workSummary: result.workSummary,
usage: result.usage,
errorReason: userFacingErrorMessage,
...(result.status === 'errored'
? {
errorInfo: {
errorMessage: terminalError
? getErrorMessage(terminalError)
: 'Instance AI stream errored',
errorSource: 'stream',
},
}
: {}),
});
await this.creditService.claimRunUsage(user, threadId, result.agentRunId || runId, result.usage?.usage ?? [], result.status);
if (result.status === 'completed') {
this.telemetry.track('Builder sent message', {
thread_id: threadId,
message: outputText,
});
this.telemetry.track('Builder satisfied user intent', {
thread_id: threadId,
});
}
}
catch (error) {
if (signal.aborted) {
if (this.shouldPreserveHitlOnShutdown(runId)) {
return;
}
if (!streamReached) {
await this.persistInterruptedUserMessage(threadId, user.id, message, turnStartedAt);
}
const runTimeout = this.liveness.consumeRunTimeout(runId);
const cancellationReason = runTimeout.timedOut
? liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON
: getAbortReason(signal);
if (cancellationReason === liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON) {
this.liveness.publishRunTimeoutNotice(threadId, runId);
}
await this.terminalOutcome.evaluateTerminalResponse(threadId, runId, 'cancelled', {
messageGroupId,
correlationId: messageId,
});
await this.tracing.finalizeRunTracing(runId, tracing, {
status: 'cancelled',
reason: cancellationReason,
});
messageTraceFinalization = {
status: 'cancelled',
reason: cancellationReason,
metadata: this.tracing.buildMessageTraceMetadata(threadId, runId, {
status: 'cancelled',
cancellationReason,
runTimeout,
}),
};
const archivedWorkflowIds = await this.temporaryWorkflowService.reapForRun(threadId, user, aiCreatedWorkflowIds, this.backgroundTasks.getRunningTasks(threadId).length);
this.publishRunFinish(threadId, runId, 'cancelled', cancellationReason, archivedWorkflowIds, user.id);
if (activeSnapshotStorage) {
await this.saveAgentTreeSnapshot(threadId, runId, activeSnapshotStorage);
}
return;
}
const terminalError = await this.reclassifyMaskedStreamFailure(error, user, {
threadId,
runId,
});
const errorMessage = getErrorMessage(terminalError);
const userFacingErrorMessage = getUserFacingErrorMessage(terminalError);
const userFacingErrorCode = getUserFacingErrorCode(terminalError);
const errCtx = {
threadId,
runId,
tracing,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
userId: user.id,
messageGroupId,
messageId,
};
this.logger.error(`Instance AI run error: ${errorMessage}`, {
error: errorMessage,
...(0, observability_1.buildInstanceAiObservabilityContext)(errCtx),
});
this.instanceAiErrorReporter.report(terminalError, {
component: 'instance-ai-run',
...errCtx,
});
await this.terminalOutcome.evaluateTerminalResponse(threadId, runId, 'errored', {
messageGroupId,
correlationId: messageId,
errorMessage: userFacingErrorMessage,
errorCode: userFacingErrorCode,
});
await this.tracing.finalizeRunTracing(runId, tracing, {
status: 'error',
reason: errorMessage,
});
messageTraceFinalization = {
status: 'error',
reason: errorMessage,
metadata: this.tracing.buildMessageTraceMetadata(threadId, runId, { status: 'error' }),
};
const archivedWorkflowIds = await this.temporaryWorkflowService.reapForRun(threadId, user, aiCreatedWorkflowIds, this.backgroundTasks.getRunningTasks(threadId).length);
this.publishRunFinish(threadId, runId, 'errored', userFacingErrorMessage, archivedWorkflowIds, user.id, { errorMessage, errorSource: 'exception' });
if (activeSnapshotStorage) {
await this.saveAgentTreeSnapshot(threadId, runId, activeSnapshotStorage);
}
}
finally {
this.runState.clearActiveRun(threadId);
this.domainAccessTrackersByThread.get(threadId)?.clearRun(runId);
if (messageTraceFinalization) {
await this.tracing.maybeFinalizeRunTraceRoot(runId, messageTraceFinalization);
if (messageTraceFinalization.status !== 'cancelled') {
this.liveness.consumeRunTimeout(runId);
}
}
this.updateInternalFollowUpFailureStreak(threadId, messageTraceFinalization?.status, resumeReason !== undefined);
if (!this.runState.hasSuspendedRun(threadId)) {
if (checkpoint?.isCheckpointFollowUp) {
await this.finalizeCheckpointFollowUp(user, threadId, checkpoint.checkpointTaskId);
}
else if (plannedBuild?.isPlannedBuildFollowUp) {
await this.finalizePlannedBuildFollowUp(user, threadId, plannedBuild);
}
else {
await this.schedulePlannedTasks(user, threadId);
}
await this.drainPendingCheckpointReentries(user, threadId);
await this.taskProjector.syncFromWorkflowLoop(threadId, runId);
await this.maybeStartWorkflowSetupFollowUp(user, threadId);
}
this.instanceAiErrorReporter.endRun(runId);
}
}
queuePendingCheckpointReentry(threadId, checkpointTaskId) {
let set = this.pendingCheckpointReentries.get(threadId);
if (!set) {
set = new Set();
this.pendingCheckpointReentries.set(threadId, set);
}
set.add(checkpointTaskId);
}
async drainPendingCheckpointReentries(user, threadId) {
const set = this.pendingCheckpointReentries.get(threadId);
if (!set || set.size === 0)
return;
const snapshot = [...set];
for (const checkpointTaskId of snapshot) {
if (this.runState.getActiveRunId(threadId) || this.runState.hasSuspendedRun(threadId)) {
return;
}
const siblings = this.backgroundTasks.getRunningTasksByParentCheckpoint(threadId, checkpointTaskId);
if (siblings.length > 0)
continue;
set.delete(checkpointTaskId);
await this.reenterCheckpointById(user, threadId, checkpointTaskId);
}
if (set.size === 0)
this.pendingCheckpointReentries.delete(threadId);
}
async reenterCheckpointById(user, threadId, checkpointTaskId, messageGroupId) {
try {
const { plannedTaskService } = await this.createPlannedTaskState();
const graph = await plannedTaskService.getGraph(threadId);
const checkpoint = graph?.tasks.find((t) => t.id === checkpointTaskId);
if (!graph || !checkpoint || checkpoint.kind !== 'checkpoint')
return false;
if (checkpoint.status !== 'running')
return false;
const startedRunId = await this.startInternalFollowUpRun(user, threadId, this.buildPlannedTaskFollowUpMessage('checkpoint', graph, { checkpoint }), messageGroupId, false, { isCheckpointFollowUp: true, checkpointTaskId });
if (!startedRunId)
return false;
this.logger.debug('Re-entered checkpoint follow-up', {
threadId,
checkpointTaskId,
messageGroupId,
});
return true;
}
catch (error) {
this.logger.error('Failed to re-enter checkpoint follow-up', {
threadId,
checkpointTaskId,
error: error instanceof Error ? error.message : String(error),
});
return false;
}
}
async maybeReenterParentCheckpoint(user, threadId, task) {
const parentCheckpointId = task.parentCheckpointId;
if (!parentCheckpointId)
return false;
const siblings = this.backgroundTasks
.getRunningTasksByParentCheckpoint(threadId, parentCheckpointId)
.filter((t) => t.taskId !== task.taskId);
if (siblings.length > 0)
return false;
if (this.runState.getActiveRunId(threadId) || this.runState.hasSuspendedRun(threadId)) {
return false;
}
return await this.reenterCheckpointById(user, threadId, parentCheckpointId, task.messageGroupId);
}
async finalizeCheckpointFollowUp(user, threadId, checkpointTaskId) {
try {
const { plannedTaskService } = await this.createPlannedTaskState();
const graph = await plannedTaskService.getGraph(threadId);
const task = graph?.tasks.find((t) => t.id === checkpointTaskId);
if (task && task.status === 'running') {
const inflightChildren = this.backgroundTasks.getRunningTasksByParentCheckpoint(threadId, checkpointTaskId);
if (inflightChildren.length > 0) {
this.logger.debug('Checkpoint run ended with in-flight child tasks — deferring finalization', {
threadId,
checkpointTaskId,
inflightTaskIds: inflightChildren.map((t) => t.taskId),
});
}
else {
this.logger.warn('Checkpoint run ended without reporting completion — marking failed', {
threadId,
checkpointTaskId,
});
await plannedTaskService.markCheckpointFailed(threadId, checkpointTaskId, {
error: 'Checkpoint run ended without reporting completion',
});
const nextGraph = await plannedTaskService.getGraph(threadId);
if (nextGraph) {
await this.syncPlannedTasksToUi(threadId, nextGraph);
}
}
}
}
catch (error) {
this.logger.error('Checkpoint finalization failed', {
threadId,
checkpointTaskId,
error: error instanceof Error ? error.message : String(error),
});
}
await this.schedulePlannedTasks(user, threadId);
}
async finalizePlannedBuildFollowUp(user, threadId, plannedBuild) {
try {
const { plannedTaskService } = await this.createPlannedTaskState();
const graph = await plannedTaskService.getGraph(threadId);
const task = graph?.tasks.find((t) => t.id === plannedBuild.buildTaskId);
if (task && task.status === 'running') {
if (plannedBuild.savedOutcome?.submitted === true) {
await plannedTaskService.markSucceeded(threadId, plannedBuild.buildTaskId, {
result: plannedBuild.savedOutcome.summary,
outcome: plannedBuild.savedOutcome,
});
}
else {
this.logger.warn('Build workflow follow-up ended without saving — marking failed', {
threadId,
buildTaskId: plannedBuild.buildTaskId,
});
await plannedTaskService.markFailed(threadId, plannedBuild.buildTaskId, {
error: 'Workflow build run ended without saving a workflow',
});
}
const nextGraph = await plannedTaskService.getGraph(threadId);
if (nextGraph) {
await this.syncPlannedTasksToUi(threadId, nextGraph);
}
}
}
catch (error) {
this.logger.error('Build workflow finalization failed', {
threadId,
buildTaskId: plannedBuild.buildTaskId,
error: error instanceof Error ? error.message : String(error),
});
}
await this.schedulePlannedTasks(user, threadId);
}
async resolveConfirmation(requestingUserId, requestId, request) {
const data = toConfirmationData(request);
const freshUser = await this.revalidateActiveUser(requestingUserId);
if (!freshUser) {
this.runState.rejectPendingConfirmation(requestId);
const suspended = this.runState.findSuspendedByRequestId(requestId);
if (suspended?.user.id === requestingUserId) {
this.cancelRun(suspended.threadId);
}
this.logger.warn('Rejecting confirmation: user no longer authorized for AI Assistant', {
userId: requestingUserId,
requestId,
});
return null;
}
if (await this.pendingConfirmationRepo.isPastExpiry(requestId, freshUser.id, new Date())) {
this.logger.debug('Rejecting expired confirmation', { requestId });
throw new n8n_workflow_1.UserError(CONFIRMATION_EXPIRED_MESSAGE);
}
const pending = this.runState.getPendingConfirmation(requestId);
if (pending &&
pending.userId === freshUser.id &&
this.runState.resolvePendingConfirmation(freshUser.id, requestId, data)) {
void this.suspendedThreads.dropPendingConfirmation(requestId);
this.logger.debug('Resolved pending confirmation (sub-agent HITL)', {
requestId,
approved: data.approved,
});
const runId = this.runState.getActiveRunId(pending.threadId);
return {
ok: true,
...(runId ? { runId } : {}),
};
}
this.logger.debug('Pending confirmation not found, trying suspended run resume', {
requestId,
approved: data.approved,
});
const resumed = await this.resumeSuspendedRun(requestingUserId, requestId, data);
if (resumed) {
return resumed;
}
return await this.suspendedRunRestorer.resolveOrphanedConfirmation(requestingUserId, requestId, data);
}
async buildMcpServers(user, threadId, runId, tracing, messageGroupId) {
const staticMcpServers = this.parseMcpServers(this.instanceAiConfig.mcpServers);
const registryMcpServers = this.settingsService.isMcpAccessEnabled()
? await this.instanceAiErrorReporter.withBoundary('instance-ai-mcp-setup', {
threadId,
runId,
tracing,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
userId: user.id,
messageGroupId,
}, async () => await this.mcpRegistryService.getRegistryMcpServers(user))
: [];
return [...staticMcpServers, ...registryMcpServers];
}
async createAgentFromEnvironment(environment, threadId, runId, user, tracing) {
if (tracing) {
environment.orchestrationContext.tracing = tracing;
}
await this.bindAgentPreviewSession(environment.context, user);
const mcpServers = await this.buildMcpServers(user, threadId, runId, tracing, environment.orchestrationContext.messageGroupId);
const { agent, mcpConnectionFailures } = await (0, instance_ai_1.createInstanceAgent)({
modelId: environment.modelId,
context: environment.context,
orchestrationContext: environment.orchestrationContext,
mcpServers,
mcpManager: this.mcpClientManager,
memoryConfig: this.createAgentMemoryOptions(user, threadId, runId),
memory: environment.memory,
checkpointStore: this.checkpointStore,
onMemoryTaskEvent: this.memoryTaskObserverFor(threadId, tracing),
thinkingEnabled: this.instanceAiConfig.thinkingEnabled,
});
if (mcpConnectionFailures.length > 0) {
const names = mcpConnectionFailures.map((f) => f.server).join(', ');
for (const failure of mcpConnectionFailures) {
this.errorReporter.error(new Error(`MCP server "${failure.server}" failed to connect: ${failure.error}`), {
level: 'warning',
tags: { component: 'instance-ai-mcp', server: failure.server },
extra: { runId, threadId, server: failure.server, error: failure.error },
shouldIsolate: true,
});
}
this.eventBus.publish(threadId, {
type: 'status',
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
payload: {
message: `Couldn't reach MCP server${mcpConnectionFailures.length > 1 ? 's' : ''} ${names}; continuing without their tools.`,
},
});
}
this.subscribeToAgentErrors(agent, threadId, runId);
return agent;
}
async buildFreshInstanceAgent(user, threadId, runId, abortSignal, tracing, messageGroupId, pushRef) {
const environment = await this.createExecutionEnvironment(user, threadId, runId, abortSignal, messageGroupId, pushRef);
const agent = await this.createAgentFromEnvironment(environment, threadId, runId, user, tracing);
return {
agent,
modelId: environment.modelId,
orchestrationContext: environment.orchestrationContext,
};
}
async rebuildSuspendedRunFromCheckpoint(orphan) {
const user = await this.revalidateActiveUser(orphan.userId);
if (!user)
return { kind: 'no-user' };
try {
const state = await this.checkpointStore.load(orphan.checkpointKey);
if (!state)
return { kind: 'no-checkpoint' };
}
catch (error) {
return { kind: 'no-checkpoint', error };
}
const abortController = new AbortController();
let environment;
try {
environment = await this.createExecutionEnvironment(user, orphan.threadId, orphan.runId, abortController.signal, orphan.messageGroupId ?? undefined, this.threadPushRef.get(orphan.threadId));
}
catch (error) {
return { kind: 'env-failure', error };
}
const runControl = (0, instance_ai_1.createOrchestratorRunControl)(environment.orchestrationContext);
let agent;
try {
agent = await this.createAgentFromEnvironment(environment, orphan.threadId, orphan.runId, user, undefined);
}
catch (error) {
return { kind: 'agent-failure', error };
}
return {
kind: 'ready',
state: {
runId: orphan.runId,
agentRunId: orphan.checkpointKey,
agent,
orchestrationContext: environment.orchestrationContext,
threadId: orphan.threadId,
user,
toolCallId: orphan.toolCallId,
requestId: orphan.requestId,
abortController,
messageGroupId: orphan.messageGroupId ?? undefined,
createdAt: Date.now(),
tracing: undefined,
modelId: environment.modelId,
checkpoint: orphan.checkpointTaskId
? { isCheckpointFollowUp: true, checkpointTaskId: orphan.checkpointTaskId }
: undefined,
runHandoff: runControl.state,
},
};
}
async revalidateActiveUser(userId) {
try {
const user = await this.userRepository.findOne({
where: { id: userId },
relations: ['role'],
});
if (!user || user.disabled)
return null;
const hasInstanceAiMessageScope = user.role?.scopes?.some((scope) => scope.slug === 'instanceAi:message') ?? false;
return hasInstanceAiMessageScope ? user : null;
}
catch (error) {
this.logger.warn('Failed to revalidate user', {
userId,
error: getErrorMessage(error),
});
return null;
}
}
async canAccessAgentPreviewHandoff(user, projectId) {
const requiredScopes = ['agent:read', 'agent:update'];
return await (0, check_access_1.userHasScopes)(user, requiredScopes, false, { projectId });
}
async assertAgentPreviewHandoffScopes(user, projectId) {
if (!(await this.canAccessAgentPreviewHandoff(user, projectId))) {
throw new forbidden_error_1.ForbiddenError('You do not have permission to load or edit agent previews in this project.');
}
}
async rebuildAgentForAutoSetupResume(user, threadId, runId, abortController, tracing, runHandoff, messageGroupId) {
try {
const rebuilt = await this.buildFreshInstanceAgent(user, threadId, runId, abortController.signal, tracing, messageGroupId, this.threadPushRef.get(threadId));
(0, instance_ai_1.createOrchestratorRunControl)(rebuilt.orchestrationContext, runHandoff ?? {});
return {
agent: rebuilt.agent,
modelId: rebuilt.modelId,
orchestrationContext: rebuilt.orchestrationContext,
};
}
catch (error) {
this.logger.warn('Failed to rebuild agent for credential auto-setup resume', {
threadId,
runId,
error: getErrorMessage(error),
});
return undefined;
}
}
async resumeSuspendedRun(requestingUserId, requestId, data) {
const suspended = this.runState.findSuspendedByRequestId(requestId);
if (!suspended) {
this.logger.warn('Confirmation target not found: no pending confirmation or suspended run', {
requestId,
approved: data.approved,
});
return null;
}
const { agent, runId, agentRunId, threadId, user, toolCallId, toolName, suspendPayload, abortController, tracing, modelId, messageGroupId, checkpoint, plannedBuild, runHandoff, orchestrationContext, } = suspended;
if (user.id !== requestingUserId)
return null;
const activeUser = await this.revalidateActiveUser(user.id);
if (!activeUser) {
this.logger.warn('Cancelling suspended run: user no longer authorized for AI Assistant', {
userId: user.id,
threadId,
requestId,
});
this.cancelRun(threadId);
return null;
}
this.runState.activateSuspendedRun(threadId);
void this.suspendedThreads.dropPendingConfirmation(requestId);
const credentialsPayload = data.nodeCredentials ?? data.credentials;
const resumeData = {
approved: data.approved,
...(credentialsPayload ? { credentials: credentialsPayload } : {}),
...(data.userInput !== undefined ? { userInput: data.userInput } : {}),
...(data.domainAccessAction ? { domainAccessAction: data.domainAccessAction } : {}),
...(data.action ? { action: data.action } : {}),
...(data.nodeParameters ? { nodeParameters: data.nodeParameters } : {}),
...(data.testTriggerNode ? { testTriggerNode: data.testTriggerNode } : {}),
...(data.answers ? { answers: data.answers } : {}),
...(data.resourceDecision ? { resourceDecision: data.resourceDecision } : {}),
...(data.scope ? { scope: data.scope } : {}),
...(data.autoSetup ? { autoSetup: data.autoSetup } : {}),
};
const resumeTracing = await this.tracing.createOrchestratorResumeTraceContext({
baseTracing: tracing,
threadId,
messageId: (0, nanoid_1.nanoid)(),
messageGroupId,
runId,
userId: activeUser.id,
modelId,
input: {
requestId,
toolCallId,
approved: data.approved,
resumeFields: Object.keys(resumeData),
...(data.userInput ? { userInput: data.userInput } : {}),
...(data.action ? { action: data.action } : {}),
...(data.resourceDecision ? { resourceDecision: data.resourceDecision } : {}),
...(data.answers ? { answers: data.answers } : {}),
},
resumeReason: 'approval',
metadata: {
request_id: requestId,
pending_tool_call_id: toolCallId,
approved: data.approved,
...(checkpoint?.isCheckpointFollowUp
? { checkpoint_task_id: checkpoint.checkpointTaskId }
: {}),
...(plannedBuild?.isPlannedBuildFollowUp
? { build_task_id: plannedBuild.buildTaskId }
: {}),
},
});
const effectiveTracing = resumeTracing ?? tracing;
if (orchestrationContext && effectiveTracing) {
orchestrationContext.tracing = effectiveTracing;
}
let resumeAgent = agent;
let resumeModelId = modelId;
let resumeOrchestrationContext = orchestrationContext;
if (data.autoSetup) {
const rebuilt = await this.rebuildAgentForAutoSetupResume(activeUser, threadId, runId, abortController, effectiveTracing, runHandoff, messageGroupId);
if (!rebuilt) {
this.cancelRun(threadId, 'agent_rebuild_failed');
return null;
}
resumeAgent = rebuilt.agent;
resumeModelId = rebuilt.modelId;
resumeOrchestrationContext = rebuilt.orchestrationContext;
}
this.startProcessResumedStream(resumeAgent, resumeData, {
runId,
agentRunId,
threadId,
user: activeUser,
toolCallId,
toolName,
suspendPayload,
signal: abortController.signal,
abortController,
snapshotStorage: this.dbSnapshotStorage,
tracing: effectiveTracing,
orchestrationContext: resumeOrchestrationContext,
modelId: resumeModelId,
checkpoint,
plannedBuild,
runHandoff,
});
return { ok: true, runId };
}
async processResumedStream(agent, resumeData, opts) {
let messageTraceFinalization;
let completedSetupWorkflowId;
try {
this.instanceAiErrorReporter.beginRun(opts.runId);
if (opts.tracing?.getTelemetry && isTelemetryConfigurableAgent(agent)) {
try {
agent.telemetry(opts.tracing.getTelemetry({
agentRole: 'orchestrator',
functionId: 'instance-ai.orchestrator',
executionMode: opts.tracing.traceKind === 'orchestrator_resume' ? 'resume' : 'foreground',
}));
}
catch (error) {
this.logger.warn('Failed to configure Instance AI resume tracing', {
error: getErrorMessage(error),
threadId: opts.threadId,
runId: opts.runId,
});
}
}
const resumeOptions = this.buildOrchestratorResumeAgentOptions(opts.user, opts.threadId, opts.runId, opts.agentRunId, opts.toolCallId);
const runControl = (0, instance_ai_1.createOrchestratorRunControlForState)(opts.runHandoff);
const stopSignal = () => runControl.getStopSignal();
const result = opts.tracing
? await opts.tracing.withActiveSpan(opts.tracing.actorRun, async () => {
return await (0, instance_ai_1.resumeAgentRun)(agent, resumeData, resumeOptions, {
threadId: opts.threadId,
runId: opts.runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(opts.runId),
signal: opts.signal,
eventBus: this.eventBus,
logger: this.logger,
agentRunId: opts.agentRunId,
onActivity: () => this.runState.touchActiveRun(opts.threadId),
stopSignal,
outputRedaction: (0, output_redaction_config_1.resolveOutputRedaction)(this.instanceAiConfig),
});
})
: await (0, instance_ai_1.resumeAgentRun)(agent, resumeData, resumeOptions, {
threadId: opts.threadId,
runId: opts.runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(opts.runId),
signal: opts.signal,
eventBus: this.eventBus,
logger: this.logger,
agentRunId: opts.agentRunId,
onActivity: () => this.runState.touchActiveRun(opts.threadId),
stopSignal,
outputRedaction: (0, output_redaction_config_1.resolveOutputRedaction)(this.instanceAiConfig),
});
if (result.status === 'suspended') {
this.emitRunMetrics(opts.threadId, 'suspended', {
modelId: opts.modelId,
workSummary: result.workSummary,
usage: result.usage,
});
if (result.suspension) {
const resumeMessageGroupId = this.tracing.getMessageGroupId(opts.runId);
this.runState.suspendRun(opts.threadId, {
runId: opts.runId,
agentRunId: result.agentRunId,
agent,
orchestrationContext: opts.orchestrationContext,
threadId: opts.threadId,
user: opts.user,
toolCallId: result.suspension.toolCallId,
...(result.suspension.toolName ? { toolName: result.suspension.toolName } : {}),
...(result.suspension.suspendPayload
? { suspendPayload: result.suspension.suspendPayload }
: {}),
requestId: result.suspension.requestId,
abortController: opts.abortController,
messageGroupId: resumeMessageGroupId,
createdAt: Date.now(),
tracing: opts.tracing,
...(opts.modelId !== undefined ? { modelId: opts.modelId } : {}),
checkpoint: opts.checkpoint,
plannedBuild: opts.plannedBuild,
runHandoff: runControl.state,
});
void this.suspendedThreads.persistPendingConfirmation({
requestId: result.suspension.requestId,
threadId: opts.threadId,
userId: opts.user.id,
runId: opts.runId,
messageGroupId: resumeMessageGroupId,
kind: 'suspended',
toolCallId: result.suspension.toolCallId,
checkpointKey: result.agentRunId,
checkpointTaskId: opts.checkpoint?.checkpointTaskId,
});
void this.creditService.claimRunUsage(opts.user, opts.threadId, `${result.agentRunId || opts.runId}:${result.suspension.requestId}`, result.usage?.usage ?? [], 'suspended');
}
const intermediateText = await (result.text ?? Promise.resolve(''));
if (intermediateText) {
this.telemetry.track('Builder sent message', {
thread_id: opts.threadId,
message: intermediateText,
is_intermediate: true,
});
}
const messageGroupId = this.tracing.getMessageGroupId(opts.runId);
const waitingDecision = await this.terminalOutcome.evaluateWaitingResponse(opts.threadId, opts.runId, result.confirmationEvent, { messageGroupId });
if (waitingDecision?.reason === 'confirmation-invalid') {
messageTraceFinalization = await this.terminalOutcome.finishInvalidConfirmationRun({
threadId: opts.threadId,
runId: opts.runId,
abortController: opts.abortController,
snapshotStorage: opts.snapshotStorage,
tracing: opts.tracing,
});
return;
}
if (result.confirmationEvent) {
this.trackConfirmationRequest(opts.threadId, result.confirmationEvent);
this.eventBus.publish(opts.threadId, result.confirmationEvent);
}
await this.saveAgentTreeSnapshot(opts.threadId, opts.runId, opts.snapshotStorage);
const suspensionOutputs = buildSuspensionTraceOutputs(opts.runId, result.suspension);
await this.tracing.finalizeRunTracing(opts.runId, opts.tracing, {
status: 'suspended',
outputs: suspensionOutputs,
metadata: {
completion_source: 'orchestrator',
...(result.suspension?.requestId ? { request_id: result.suspension.requestId } : {}),
...(result.suspension?.toolCallId
? { pending_tool_call_id: result.suspension.toolCallId }
: {}),
...(result.suspension?.toolName
? { pending_tool_name: result.suspension.toolName }
: {}),
},
});
messageTraceFinalization = {
status: 'suspended',
outputs: suspensionOutputs,
metadata: {
completion_source: 'orchestrator',
...(result.suspension?.requestId ? { request_id: result.suspension.requestId } : {}),
...(result.suspension?.toolCallId
? { pending_tool_call_id: result.suspension.toolCallId }
: {}),
...(result.suspension?.toolName
? { pending_tool_name: result.suspension.toolName }
: {}),
},
};
return;
}
if (result.status === 'cancelled' && this.shouldPreserveHitlOnShutdown(opts.runId)) {
return;
}
const outputText = await (result.text ?? Promise.resolve(''));
const messageGroupId = this.tracing.getMessageGroupId(opts.runId);
const terminalError = result.status === 'errored'
? await this.reclassifyMaskedStreamFailure(result.error, opts.user, {
threadId: opts.threadId,
runId: opts.runId,
})
: undefined;
if (result.status === 'errored') {
this.instanceAiErrorReporter.report(terminalError ?? new Error('Instance AI resumed stream errored'), {
component: 'instance-ai-stream',
threadId: opts.threadId,
runId: opts.runId,
tracing: opts.tracing,
agentId: (0, instance_ai_1.orchestratorAgentId)(opts.runId),
userId: opts.user.id,
messageGroupId,
});
}
const userFacingErrorMessage = result.status === 'errored' ? getUserFacingErrorMessage(terminalError) : undefined;
const userFacingErrorCode = result.status === 'errored' ? getUserFacingErrorCode(terminalError) : undefined;
if (runControl.shouldEmitTerminalOutcome(result.stopReason)) {
await this.terminalOutcome.evaluateTerminalResponse(opts.threadId, opts.runId, result.status, {
messageGroupId,
workSummary: result.workSummary,
errorMessage: userFacingErrorMessage,
errorCode: userFacingErrorCode,
suppressCompletedFallback: opts.checkpoint?.isCheckpointFollowUp === true ||
opts.plannedBuild?.isPlannedBuildFollowUp === true,
});
}
const finalStatus = result.status === 'errored' ? 'error' : result.status;
await this.tracing.finalizeRunTracing(opts.runId, opts.tracing, {
status: finalStatus,
outputText,
});
messageTraceFinalization = {
status: finalStatus,
outputText,
metadata: this.tracing.buildMessageTraceMetadata(opts.threadId, opts.runId, {
status: finalStatus,
}),
};
const archivedWorkflowIds = await this.temporaryWorkflowService.reapForRun(opts.threadId, opts.user, undefined, this.backgroundTasks.getRunningTasks(opts.threadId).length);
await this.finalizeRun(opts.threadId, opts.runId, result.status, opts.snapshotStorage, {
userId: opts.user.id,
...(opts.modelId !== undefined ? { modelId: opts.modelId } : {}),
archivedWorkflowIds,
workSummary: result.workSummary,
usage: result.usage,
errorReason: userFacingErrorMessage,
...(result.status === 'errored'
? {
errorInfo: {
errorMessage: terminalError
? getErrorMessage(terminalError)
: 'Instance AI resumed stream errored',
errorSource: 'stream',
},
}
: {}),
});
await this.creditService.claimRunUsage(opts.user, opts.threadId, result.agentRunId || opts.runId, result.usage?.usage ?? [], result.status);
if (result.status === 'completed') {
completedSetupWorkflowId = this.getWorkflowSetupSuspensionWorkflowId(opts.toolName, opts.suspendPayload);
this.telemetry.track('Builder sent message', {
thread_id: opts.threadId,
message: outputText,
});
this.telemetry.track('Builder satisfied user intent', {
thread_id: opts.threadId,
});
}
}
catch (error) {
if (opts.signal.aborted) {
if (this.shouldPreserveHitlOnShutdown(opts.runId)) {
return;
}
const messageGroupId = this.tracing.getMessageGroupId(opts.runId);
const runTimeout = this.liveness.consumeRunTimeout(opts.runId);
const cancellationReason = runTimeout.timedOut
? liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON
: getAbortReason(opts.signal);
if (cancellationReason === liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON) {
this.liveness.publishRunTimeoutNotice(opts.threadId, opts.runId);
}
await this.terminalOutcome.evaluateTerminalResponse(opts.threadId, opts.runId, 'cancelled', {
messageGroupId,
});
await this.tracing.finalizeRunTracing(opts.runId, opts.tracing, {
status: 'cancelled',
reason: cancellationReason,
});
messageTraceFinalization = {
status: 'cancelled',
reason: cancellationReason,
metadata: this.tracing.buildMessageTraceMetadata(opts.threadId, opts.runId, {
status: 'cancelled',
cancellationReason,
runTimeout,
}),
};
const archivedWorkflowIds = await this.temporaryWorkflowService.reapForRun(opts.threadId, opts.user, undefined, this.backgroundTasks.getRunningTasks(opts.threadId).length);
this.publishRunFinish(opts.threadId, opts.runId, 'cancelled', cancellationReason, archivedWorkflowIds, opts.user.id);
await this.saveAgentTreeSnapshot(opts.threadId, opts.runId, opts.snapshotStorage);
return;
}
const terminalError = await this.reclassifyMaskedStreamFailure(error, opts.user, {
threadId: opts.threadId,
runId: opts.runId,
});
const errorMessage = getErrorMessage(terminalError);
const userFacingErrorMessage = getUserFacingErrorMessage(terminalError);
const userFacingErrorCode = getUserFacingErrorCode(terminalError);
const messageGroupId = this.tracing.getMessageGroupId(opts.runId);
const errCtx = {
threadId: opts.threadId,
runId: opts.runId,
tracing: opts.tracing,
agentId: (0, instance_ai_1.orchestratorAgentId)(opts.runId),
userId: opts.user.id,
messageGroupId,
};
this.logger.error(`Instance AI resumed run error: ${errorMessage}`, {
error: errorMessage,
...(0, observability_1.buildInstanceAiObservabilityContext)(errCtx),
});
this.instanceAiErrorReporter.report(terminalError, {
component: 'instance-ai-run',
...errCtx,
});
await this.terminalOutcome.evaluateTerminalResponse(opts.threadId, opts.runId, 'errored', {
messageGroupId,
errorMessage: userFacingErrorMessage,
errorCode: userFacingErrorCode,
});
await this.tracing.finalizeRunTracing(opts.runId, opts.tracing, {
status: 'error',
reason: errorMessage,
});
messageTraceFinalization = {
status: 'error',
reason: errorMessage,
metadata: this.tracing.buildMessageTraceMetadata(opts.threadId, opts.runId, {
status: 'error',
}),
};
const archivedWorkflowIds = await this.temporaryWorkflowService.reapForRun(opts.threadId, opts.user, undefined, this.backgroundTasks.getRunningTasks(opts.threadId).length);
this.publishRunFinish(opts.threadId, opts.runId, 'errored', userFacingErrorMessage, archivedWorkflowIds, opts.user.id, { errorMessage, errorSource: 'exception' });
await this.saveAgentTreeSnapshot(opts.threadId, opts.runId, opts.snapshotStorage);
}
finally {
this.runState.clearActiveRun(opts.threadId);
if (messageTraceFinalization) {
await this.tracing.maybeFinalizeRunTraceRoot(opts.runId, messageTraceFinalization);
if (messageTraceFinalization.status !== 'cancelled') {
this.liveness.consumeRunTimeout(opts.runId);
}
}
this.updateInternalFollowUpFailureStreak(opts.threadId, messageTraceFinalization?.status, false);
if (!this.runState.hasSuspendedRun(opts.threadId)) {
if (opts.checkpoint?.isCheckpointFollowUp) {
await this.finalizeCheckpointFollowUp(opts.user, opts.threadId, opts.checkpoint.checkpointTaskId);
}
else if (opts.plannedBuild?.isPlannedBuildFollowUp) {
await this.finalizePlannedBuildFollowUp(opts.user, opts.threadId, opts.plannedBuild);
}
else {
await this.schedulePlannedTasks(opts.user, opts.threadId);
}
await this.drainPendingCheckpointReentries(opts.user, opts.threadId);
if (completedSetupWorkflowId) {
await this.markWorkflowSetupHandled(opts.threadId, completedSetupWorkflowId, opts.runId);
}
await this.taskProjector.syncFromWorkflowLoop(opts.threadId, opts.runId);
await this.maybeStartWorkflowSetupFollowUp(opts.user, opts.threadId);
}
this.instanceAiErrorReporter.endRun(opts.runId);
}
}
spawnBackgroundTask(runId, opts, snapshotStorage, messageGroupIdOverride) {
const outcome = this.backgroundTasks.spawn({
taskId: opts.taskId,
threadId: opts.threadId,
runId,
role: opts.role,
agentId: opts.agentId,
messageGroupId: messageGroupIdOverride ?? this.runState.getMessageGroupId(opts.threadId),
plannedTaskId: opts.plannedTaskId,
workItemId: opts.workItemId,
traceContext: opts.traceContext,
createTraceContext: opts.createTraceContext,
dedupeKey: opts.dedupeKey,
parentCheckpointId: opts.parentCheckpointId,
run: opts.run,
onLimitReached: async (errorMessage) => {
await this.tracing.finalizeDetachedTraceRun(opts.taskId, opts.traceContext, {
status: 'failed',
outputs: {
taskId: opts.taskId,
agentId: opts.agentId,
role: opts.role,
},
error: errorMessage,
metadata: {
...(opts.plannedTaskId ? { planned_task_id: opts.plannedTaskId } : {}),
...(opts.workItemId ? { work_item_id: opts.workItemId } : {}),
},
});
this.eventBus.publish(opts.threadId, {
type: 'agent-completed',
runId,
agentId: opts.agentId,
payload: {
role: opts.role,
result: '',
error: errorMessage,
},
});
},
onCompleted: async (task) => {
await this.tracing.finalizeBackgroundTaskTracing(task, 'completed');
this.eventBus.publish(opts.threadId, {
type: 'agent-completed',
runId,
agentId: opts.agentId,
payload: { role: opts.role, result: task.result ?? '' },
});
const user = this.runState.getThreadUser(opts.threadId);
if (user) {
await this.handlePlannedTaskSettlement(user, task, 'succeeded');
}
},
onFailed: async (task) => {
await this.tracing.finalizeBackgroundTaskTracing(task, 'failed');
this.instanceAiErrorReporter.report(new Error(task.error ?? 'Instance AI background task failed'), {
component: 'instance-ai-background-task',
threadId: opts.threadId,
runId,
tracing: task.traceContext,
agentId: opts.agentId,
messageGroupId: task.messageGroupId,
taskId: task.taskId,
role: task.role,
});
this.eventBus.publish(opts.threadId, {
type: 'agent-completed',
runId,
agentId: opts.agentId,
payload: { role: opts.role, result: '', error: task.error ?? 'Unknown error' },
});
const user = this.runState.getThreadUser(opts.threadId);
if (user) {
await this.handlePlannedTaskSettlement(user, task, 'failed');
}
},
onSettled: async (task) => {
await this.terminalOutcome.recordBackgroundTerminalOutcome(task);
await this.saveAgentTreeSnapshot(opts.threadId, runId, snapshotStorage, true, task.messageGroupId);
if (task.plannedTaskId)
return;
await this.taskProjector.syncFromBackgroundTask(task);
const parentCheckpointId = task.parentCheckpointId;
if (parentCheckpointId) {
const user = this.runState.getThreadUser(opts.threadId);
if (!user) {
this.queuePendingCheckpointReentry(opts.threadId, parentCheckpointId);
return;
}
const reentered = await this.maybeReenterParentCheckpoint(user, opts.threadId, task);
if (!reentered) {
this.queuePendingCheckpointReentry(opts.threadId, parentCheckpointId);
}
return;
}
const remaining = this.backgroundTasks.getRunningTasks(opts.threadId);
const hasActiveRun = !!this.runState.getActiveRunId(opts.threadId);
const hasSuspendedRun = this.runState.hasSuspendedRun(opts.threadId);
if (remaining.length === 0 && !hasActiveRun && !hasSuspendedRun) {
if (this.liveness.hasTimedOutActiveRunThread(opts.threadId)) {
this.logger.debug('Skipping background auto-follow-up after active run timeout', {
threadId: opts.threadId,
taskId: task.taskId,
});
return;
}
if (task.timeoutReason) {
this.logger.debug('Skipping background auto-follow-up after task timeout', {
threadId: opts.threadId,
taskId: task.taskId,
timeoutReason: task.timeoutReason,
});
return;
}
const user = this.runState.getThreadUser(opts.threadId);
if (user) {
const verificationFollowUpStarted = await this.maybeStartWorkflowVerificationFollowUp(user, task);
if (verificationFollowUpStarted)
return;
const setupFollowUpStarted = await this.maybeStartWorkflowSetupFollowUp(user, task.threadId);
if (setupFollowUpStarted)
return;
const payload = JSON.stringify({
role: opts.role,
status: task.result ? 'completed' : task.error ? 'failed' : 'finished',
result: task.result ?? undefined,
outcome: task.outcome ?? undefined,
error: task.error ?? undefined,
}, null, 2);
await this.startInternalFollowUpRun(user, opts.threadId, `<background-task-completed>\n${payload}\n</background-task-completed>\n\n${internal_messages_1.AUTO_FOLLOW_UP_MESSAGE}`, task.messageGroupId);
}
}
},
});
if (outcome.status === 'started') {
void this.taskProjector.syncFromBackgroundTask(outcome.task);
return { status: 'started', taskId: outcome.task.taskId, agentId: outcome.task.agentId };
}
if (outcome.status === 'duplicate') {
this.logger.warn('Background task dispatch deduped — task already in flight', {
threadId: opts.threadId,
requestedTaskId: opts.taskId,
existingTaskId: outcome.existing.taskId,
plannedTaskId: opts.dedupeKey?.plannedTaskId,
workflowId: opts.dedupeKey?.workflowId,
role: opts.role,
});
void this.tracing.finalizeDetachedTraceRun(opts.taskId, opts.traceContext, {
status: 'cancelled',
outputs: {
taskId: opts.taskId,
agentId: opts.agentId,
role: opts.role,
deduped_to: outcome.existing.taskId,
},
metadata: {
deduped: true,
existing_task_id: outcome.existing.taskId,
...(opts.plannedTaskId ? { planned_task_id: opts.plannedTaskId } : {}),
...(opts.workItemId ? { work_item_id: opts.workItemId } : {}),
},
});
this.eventBus.publish(opts.threadId, {
type: 'agent-completed',
runId,
agentId: opts.agentId,
payload: {
role: opts.role,
result: '',
error: `Deduped: task already in flight as ${outcome.existing.taskId}`,
},
});
return {
status: 'duplicate',
existing: {
taskId: outcome.existing.taskId,
agentId: outcome.existing.agentId,
role: outcome.existing.role,
plannedTaskId: outcome.existing.plannedTaskId,
workItemId: outcome.existing.workItemId,
},
};
}
return { status: 'limit-reached' };
}
async buildMessageWithRunningTasks(threadId, message) {
return await (0, instance_ai_1.enrichMessageWithBackgroundTasks)(message, this.backgroundTasks.getRunningTasks(threadId), {
formatTask: async (task) => `[Running task — ${task.role}]: taskId=${task.taskId}`,
});
}
trackConfirmationRequest(threadId, confirmationEvent) {
const payload = confirmationEvent.payload;
const inputThreadId = (0, nanoid_1.nanoid)();
payload.inputThreadId = inputThreadId;
const inputType = payload.inputType;
let type;
if (inputType) {
type = inputType;
}
else if (Array.isArray(payload.setupRequests) && payload.setupRequests.length > 0) {
type = 'setup';
}
else if (Array.isArray(payload.credentialRequests) && payload.credentialRequests.length > 0) {
type = 'credential-setup';
}
else {
type = 'approval';
}
let numSteps = 1;
if (Array.isArray(payload.questions)) {
numSteps = payload.questions.length;
}
else if (Array.isArray(payload.setupRequests)) {
numSteps = payload.setupRequests.length;
}
else if (Array.isArray(payload.credentialRequests)) {
numSteps = payload.credentialRequests.length;
}
if (inputType === 'plan-review') {
const planCount = (this.planRequestsByThread.get(threadId) ?? 0) + 1;
this.planRequestsByThread.set(threadId, planCount);
type = planCount === 1 ? 'first_plan' : 'revised_plan';
if (Array.isArray(payload.planItems)) {
numSteps = payload.planItems.length;
}
}
this.telemetry.track('Builder asked for input', {
thread_id: threadId,
input_thread_id: inputThreadId,
type,
num_steps: numSteps,
});
}
async finalizeCancelledSuspendedRun(suspended, reason = 'user_cancelled') {
const runTimeout = reason === liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON
? this.liveness.consumeRunTimeout(suspended.runId)
: undefined;
if (reason === liveness_1.INSTANCE_AI_RUN_TIMEOUT_REASON) {
this.liveness.publishRunTimeoutNotice(suspended.threadId, suspended.runId);
}
await this.tracing.finalizeRunTracing(suspended.runId, suspended.tracing, {
status: 'cancelled',
reason,
});
const archivedWorkflowIds = await this.temporaryWorkflowService.reapForRun(suspended.threadId, suspended.user, undefined, this.backgroundTasks.getRunningTasks(suspended.threadId).length);
this.publishRunFinish(suspended.threadId, suspended.runId, 'cancelled', reason, archivedWorkflowIds, suspended.user.id);
await this.saveAgentTreeSnapshot(suspended.threadId, suspended.runId, this.dbSnapshotStorage, true);
await this.tracing.maybeFinalizeRunTraceRoot(suspended.runId, {
status: 'cancelled',
reason,
metadata: this.tracing.buildMessageTraceMetadata(suspended.threadId, suspended.runId, {
status: 'cancelled',
cancellationReason: reason,
...(runTimeout ? { runTimeout } : {}),
}),
});
void this.suspendedThreads.dropPendingConfirmation(suspended.requestId);
}
publishRunFinish(threadId, runId, status, reason, archivedWorkflowIds, userId, errorInfo) {
const effectiveStatus = status === 'errored' ? 'error' : status;
const hasArchived = archivedWorkflowIds && archivedWorkflowIds.length > 0;
this.eventBus.publish(threadId, {
type: 'run-finish',
runId,
agentId: (0, instance_ai_1.orchestratorAgentId)(runId),
payload: {
status: effectiveStatus,
...(status === 'cancelled'
? { reason: reason ?? 'user_cancelled' }
: status === 'errored' && reason
? { reason }
: {}),
...(hasArchived ? { archivedWorkflowIds } : {}),
},
});
this.telemetry.track('instance_ai_run_finished', {
thread_id: threadId,
run_id: runId,
status: effectiveStatus,
...(userId ? { user_id: userId } : {}),
});
if (status === 'errored') {
this.telemetry.track('Builder generation errored', {
thread_id: threadId,
run_id: runId,
error_message: errorInfo?.errorMessage ?? reason ?? 'unknown',
...(errorInfo?.errorSource ? { error_source: errorInfo.errorSource } : {}),
...(userId ? { user_id: userId } : {}),
});
}
}
async finalizeRun(threadId, runId, status, snapshotStorage, options) {
this.publishRunFinish(threadId, runId, status, options?.errorReason, options?.archivedWorkflowIds, options?.userId, options?.errorInfo);
this.emitRunMetrics(threadId, status, options);
await this.saveAgentTreeSnapshot(threadId, runId, snapshotStorage);
if (status === 'completed' && options?.userId && options?.modelId) {
void this.refineTitleIfNeeded(threadId, options.userId, options.modelId);
}
}
emitRunMetrics(threadId, status, options) {
const startedAt = this.runState.getActiveRun(threadId)?.startedAt;
this.eventService.emit('instance-ai-run-finished', {
status: status === 'errored' ? 'error' : status,
durationMs: status !== 'suspended' && startedAt !== undefined ? Date.now() - startedAt : undefined,
model: typeof options?.modelId === 'string' ? options.modelId : 'custom',
toolCalls: options?.workSummary?.totalToolCalls ?? 0,
toolErrors: options?.workSummary?.totalToolErrors ?? 0,
...(options?.usage ? { usage: options.usage } : {}),
});
}
async refineTitleIfNeeded(threadId, userId, modelId) {
try {
const memory = this.agentMemory;
const thread = await memory.getThread(threadId);
if (!thread?.title)
return;
if (thread.metadata?.titleRefined)
return;
const history = await memory.getMessages(threadId, { limit: TITLE_REFINE_HISTORY_LIMIT });
const userTexts = [];
for (const m of history) {
if (!('role' in m) || m.role !== 'user')
continue;
const text = (0, internal_messages_1.cleanStoredUserMessage)(this.extractStoredMessageText(m.content));
if (text && text.length > 0)
userTexts.push(text);
if (userTexts.length >= 5)
break;
}
if (userTexts.length === 0)
return;
const userText = userTexts.join('\n');
const baseTracing = this.tracing.getTraceContextForContinuation(threadId);
const titleTracing = await (0, instance_ai_1.createInternalOperationTraceContext)({
threadId,
conversationId: threadId,
messageId: `internal:title:${threadId}`,
runId: `title-${(0, nanoid_1.nanoid)()}`,
userId,
modelId,
operationName: 'thread_title',
input: {
message_count: userTexts.length,
source: 'thread_title_refinement',
},
proxyConfig: baseTracing?.proxyConfig,
metadata: {
n8n_version: constants_1.N8N_VERSION || undefined,
operation_name: 'thread_title',
trigger: 'run_completed',
},
});
const titleTelemetry = titleTracing?.getTelemetry?.({
agentRole: 'thread_title',
functionId: 'instance-ai.thread_title',
executionMode: 'internal',
metadata: {
operation_name: 'thread_title',
},
});
let llmTitle;
if (titleTracing) {
try {
llmTitle = await titleTracing.withActiveSpan(titleTracing.rootRun, async () => {
const title = await (0, instance_ai_1.generateTitleForRun)(modelId, userText, {
...(titleTelemetry ? { telemetry: titleTelemetry } : {}),
});
if (title) {
await titleTracing.finishRun(titleTracing.rootRun, {
outputs: { title },
metadata: { final_status: 'completed' },
});
}
else {
await titleTracing.finishRun(titleTracing.rootRun, {
outputs: { title: null },
metadata: { final_status: 'skipped' },
});
}
return title;
});
}
finally {
(0, instance_ai_1.releaseTraceClient)(titleTracing.rootRun.traceId);
}
}
else {
llmTitle = await (0, instance_ai_1.generateTitleForRun)(modelId, userText);
}
if (!llmTitle)
return;
await (0, instance_ai_1.patchThread)(memory, {
threadId,
update: ({ metadata }) => ({
title: llmTitle,
metadata: { ...metadata, titleRefined: true },
}),
});
this.eventBus.publish(threadId, {
type: 'thread-title-updated',
runId: '',
agentId: 'orchestrator',
payload: { title: llmTitle },
});
}
catch (error) {
this.logger.warn('Failed to refine thread title', {
threadId,
error: getErrorMessage(error),
});
}
}
extractStoredMessageText(content) {
if (typeof content === 'string')
return content;
if (Array.isArray(content)) {
return content.flatMap((part) => (isTextMessagePart(part) ? [part.text] : [])).join('\n');
}
return '';
}
async readDurableEventsForRuns(threadId, runIds) {
await this.eventLog.flush(threadId);
return await this.eventLog.getEventsForRuns(threadId, runIds);
}
async saveAgentTreeSnapshot(threadId, runId, snapshotStorage, isUpdate = false, overrideMessageGroupId) {
try {
const messageGroupId = overrideMessageGroupId ?? this.runState.getMessageGroupId(threadId);
let events;
let groupRunIds;
if (messageGroupId) {
groupRunIds = this.getRunIdsForMessageGroup(messageGroupId);
if (groupRunIds.length === 0) {
const snapshot = await snapshotStorage.getLatest(threadId, { messageGroupId, runId });
groupRunIds = snapshot?.runIds?.length ? snapshot.runIds : [runId];
}
events = this.instanceAiConfig.durableLog
? await this.readDurableEventsForRuns(threadId, groupRunIds)
: this.eventBus.getEventsForRuns(threadId, groupRunIds);
}
else {
events = this.instanceAiConfig.durableLog
? await this.readDurableEventsForRuns(threadId, [runId])
: this.eventBus.getEventsForRun(threadId, runId);
}
if (isUpdate && events.length === 0) {
this.logger.warn('Skipped updating empty Instance AI agent tree snapshot', {
threadId,
runId,
messageGroupId,
});
return;
}
const agentTree = (0, instance_ai_1.buildAgentTreeFromEvents)(events);
const tracing = this.tracing.getTraceContext(runId);
const saveOptions = {
messageGroupId,
runIds: groupRunIds,
traceId: tracing?.rootRun.otelTraceId,
spanId: tracing?.rootRun.otelSpanId,
langsmithRunId: tracing?.rootRun.id,
langsmithTraceId: tracing?.rootRun.traceId,
};
if (isUpdate) {
await snapshotStorage.updateLast(threadId, agentTree, runId, saveOptions);
}
else {
await snapshotStorage.save(threadId, agentTree, runId, saveOptions);
}
}
catch (error) {
this.logger.warn('Failed to save agent tree snapshot', {
threadId,
runId,
error: error instanceof Error ? error.message : String(error),
});
}
}
parseMcpServers(raw) {
if (!raw.trim())
return [];
return raw.split(',').map((entry) => {
const [name, url] = entry.trim().split('=');
return { name: name.trim(), url: url?.trim() };
});
}
trackMcpToolCall({ server, toolName, success, }) {
const serverSlug = server.metadata?.serverSlug;
const userId = server.metadata?.userId;
if (serverSlug && userId) {
this.telemetry.track('Instance AI mcp tool called', {
user_id: userId,
server_slug: serverSlug,
tool_name: toolName,
success,
});
}
}
};
exports.InstanceAiService = InstanceAiService;
__decorate([
(0, decorators_1.OnPubSubEvent)('relay-instance-ai-task-control', { instanceType: 'main' }),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Object]),
__metadata("design:returntype", Promise)
], InstanceAiService.prototype, "handleRelayTaskControl", null);
__decorate([
(0, decorators_1.OnLeaderTakeover)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", void 0)
], InstanceAiService.prototype, "startCheckpointPruning", null);
__decorate([
(0, decorators_1.OnLeaderStepdown)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", void 0)
], InstanceAiService.prototype, "stopCheckpointPruning", null);
exports.InstanceAiService = InstanceAiService = __decorate([
(0, di_1.Service)(),
__metadata("design:paramtypes", [backend_common_1.Logger, config_1.GlobalConfig, n8n_core_1.InstanceSettings, instance_ai_adapter_service_1.InstanceAiAdapterService, in_process_event_bus_1.InProcessEventBus, durable_event_log_1.DurableEventLog, interrupted_run_sweeper_1.InterruptedRunSweeper, instance_ai_settings_service_1.InstanceAiSettingsService, instance_ai_gateway_service_1.InstanceAiGatewayService, instance_ai_browser_session_service_1.InstanceAiBrowserSessionService, instance_ai_memory_service_1.InstanceAiMemoryService, typeorm_agent_memory_1.TypeORMAgentMemory, typeorm_agent_checkpoint_store_1.TypeORMAgentCheckpointStore, ai_service_1.AiService, instance_ai_thread_grant_repository_1.InstanceAiThreadGrantRepository, instance_ai_pending_confirmation_repository_1.InstanceAiPendingConfirmationRepository, url_service_1.UrlService, db_snapshot_storage_1.DbSnapshotStorage, db_iteration_log_storage_1.DbIterationLogStorage, source_control_preferences_service_ee_1.SourceControlPreferencesService, telemetry_1.Telemetry, mcp_1.InstanceAiMcpRegistryService, db_1.UserRepository, instance_ai_temporary_workflow_service_1.InstanceAiTemporaryWorkflowService, n8n_core_1.ErrorReporter, config_1.SsrfProtectionConfig, backend_network_1.SsrfProtectionService, event_service_1.EventService, thread_credential_allowlist_service_1.EvalThreadCredentialAllowlistService, instance_ai_run_probe_1.InstanceAiRunProbe, instance_ai_model_service_1.InstanceAiModelService, instance_ai_credit_service_1.InstanceAiCreditService, publisher_service_1.Publisher, instance_ai_error_reporter_service_1.InstanceAiErrorReporterService])
], InstanceAiService);
//# sourceMappingURL=instance-ai.service.js.map