@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
1,937 lines • 91.9 kB
JavaScript
/**
* External MCP Server Manager
* Handles lifecycle management of external MCP servers including:
* - Process spawning and management
* - Health monitoring and automatic restart
* - Connection management and cleanup
* - Tool discovery and registration
*/
import { EventEmitter } from "events";
import { mcpLogger } from "../utils/logger.js";
import { MCPClientFactory } from "./mcpClientFactory.js";
import { ToolDiscoveryService } from "./toolDiscoveryService.js";
import { toolRegistry } from "./toolRegistry.js";
import { HITLUserRejectedError, HITLTimeoutError } from "../hitl/hitlErrors.js";
import { detectCategory } from "../utils/mcpDefaults.js";
import { isObject, isNonNullObject } from "../utils/typeUtils.js";
import { TelemetryService } from "../telemetry/telemetryService.js";
import { tracers } from "../telemetry/tracers.js";
import { SpanStatusCode } from "@opentelemetry/api";
// `@modelcontextprotocol/sdk`'s stdio transport (imported above) does
// `import process from "node:process"` at module scope. Node's ESM/CJS
// interop builds a synthetic module facade for that import by walking every
// property getter on the builtin `process` singleton — including the
// lazily-initialized `stdin` getter — so merely importing this module
// creates and permanently refs a stdin handle, whether or not an MCP stdio
// client is ever constructed. A single ref'd handle is enough to keep the
// event loop, and the whole process, alive forever. unref() does not close
// stdin or stop it from being read (a consumer that later actually reads
// from it — e.g. the CLI's piped-input path — calls `.ref()` again right
// before it does), it only stops an otherwise-idle stdin from blocking exit.
//
// This unref runs at import time, before any consumer of THIS package —
// this package's own CLI paths and an external SDK caller alike — has had a
// chance to attach its own stdin listeners. A CLI code path knows to call
// `ensureStdinRef()` first (see src/cli/utils/stdinRef.ts), but an external
// script that just does `import { NeuroLink } from "@juspay/neurolink"` and
// then reads `process.stdin` itself — an ordinary, previously-working
// pattern — has no reason to know this package touched stdin at all, and
// without a safety net the process can exit before the piped data ever
// arrives: silently, with no error, because an unref'd handle simply stops
// keeping the event loop alive on its own.
//
// So: re-ref stdin the first time anything registers a listener that
// actually consumes it. "data" covers `.on("data", ...)` directly and
// `.pipe()` (which attaches its own "data" listener internally); "readable"
// covers paused-mode reads and the async iterator
// (`for await (const chunk of process.stdin)`, which attaches a "readable"
// listener under the hood). This restores stdin's normal, non-unref'd
// behavior for any real consumer — internal or external — while a process
// that never touches stdin still exits on its own.
if (process.stdin && typeof process.stdin.unref === "function") {
process.stdin.unref();
process.stdin.prependListener("newListener", (event) => {
if (event === "data" || event === "readable") {
process.stdin.ref();
}
});
}
/**
* Recursively substitute environment variables in strings
* Replaces ${VAR_NAME} with the value from process.env.VAR_NAME
* @param value - Value to process (string, object, array, or primitive)
* @returns Processed value with environment variables substituted
*/
function substituteEnvVariables(value) {
if (typeof value === "string") {
// Replace ${VAR_NAME} with process.env.VAR_NAME
return value.replace(/\$\{([^}]+)\}/g, (match, varName) => {
const envValue = process.env[varName.trim()];
if (envValue === undefined) {
mcpLogger.warn(`[ExternalServerManager] Environment variable ${varName} is not defined, using empty string`);
return "";
}
return envValue;
});
}
if (Array.isArray(value)) {
return value.map((item) => substituteEnvVariables(item));
}
if (isNonNullObject(value)) {
const result = {};
for (const [key, val] of Object.entries(value)) {
result[key] = substituteEnvVariables(val);
}
return result;
}
return value;
}
/**
* The two messages the MCP SDK produces when the transport underneath a
* request is gone: `Protocol._onclose` rejects every pending request with
* `MCP error -32000: Connection closed`, and `StdioClientTransport.send`
* throws a bare `Not connected` once the child has closed. Both are matched
* on the SDK's exact text so a tool's own error output that happens to
* mention a closed connection cannot be mistaken for the server dying.
*/
const CONNECTION_LOST_PATTERN = /^Not connected$|MCP error -32000: Connection closed/;
function isConnectionLostError(error) {
const message = error instanceof Error ? error.message : String(error);
return CONNECTION_LOST_PATTERN.test(message);
}
/**
* Thrown by `startServer` when a server's discovered tool count falls below
* its configured `minTools` floor. Deliberately thrown *inside* the same
* try/catch that already handles every other start failure, so the existing
* machinery (status -> "failed", `lastError`, span exception recording,
* rethrow to `addServer`'s own catch) applies unchanged; `addServer`
* recognizes this specific error via `instanceof` to attach the
* `readiness: "insufficient_tools"` + `toolsDiscovered` metadata that a
* generic start failure does not carry.
*/
class InsufficientToolsError extends Error {
serverId;
toolsDiscovered;
minTools;
readiness = "insufficient_tools";
constructor(serverId, toolsDiscovered, minTools) {
super(`Server '${serverId}' discovered ${toolsDiscovered} tool(s), below the configured minTools=${minTools}`);
this.serverId = serverId;
this.toolsDiscovered = toolsDiscovered;
this.minTools = minTools;
this.name = "InsufficientToolsError";
}
}
/**
* Consecutive idle health checks a server may fail to answer before it is
* treated as hung and restarted. Two, not one: a single missed ping right
* after a long tool call is the server draining its queue, not a wedge.
*/
const UNRESPONSIVE_CHECKS_BEFORE_RESTART = 2;
/**
* Sensitive CLI flag patterns whose following value should be masked in logs.
*/
const SENSITIVE_ARG_PATTERNS = /^--(api-key|token|secret|password|key|figma-api-key|access-token|auth|credential)$/i;
/**
* Redact values that follow sensitive flags in a CLI args array.
* Handles: "--api-key sk-abc", "--api-key=sk-abc", consecutive flags.
* E.g. ["--api-key", "sk-abc123"] → ["--api-key", "[REDACTED]"]
* ["--api-key=sk-abc123"] → ["--api-key=[REDACTED]"]
*/
function redactSensitiveArgs(args) {
if (!args || args.length === 0) {
return args;
}
const redacted = [...args];
for (let i = 0; i < redacted.length; i++) {
// Handle --flag=value inline form
const eqIdx = redacted[i].indexOf("=");
if (eqIdx !== -1) {
const flag = redacted[i].substring(0, eqIdx);
if (SENSITIVE_ARG_PATTERNS.test(flag)) {
redacted[i] = `${flag}=[REDACTED]`;
}
continue;
}
// Handle --flag value (two-part form)
if (SENSITIVE_ARG_PATTERNS.test(redacted[i]) &&
i + 1 < redacted.length &&
!redacted[i + 1].startsWith("--")) {
redacted[i + 1] = "[REDACTED]";
i++; // skip the redacted value
}
}
return redacted;
}
/**
* Type guard to validate if an object can be safely used as Record<string, JsonValue>
*/
function isValidJsonRecord(value) {
if (!isObject(value)) {
return false;
}
const record = value;
return Object.values(record).every((val) => {
// JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }
if (val === null ||
typeof val === "string" ||
typeof val === "number" ||
typeof val === "boolean") {
return true;
}
if (Array.isArray(val)) {
return val.every((item) => isValidJsonRecord(item) ||
typeof item === "string" ||
typeof item === "number" ||
typeof item === "boolean" ||
item === null);
}
if (isNonNullObject(val)) {
return isValidJsonRecord(val);
}
return false;
});
}
/**
* Safely converts unknown metadata to Record<string, JsonValue> or returns undefined
*/
function safeMetadataConversion(metadata) {
return isValidJsonRecord(metadata) ? metadata : undefined;
}
/**
* Type guard to validate external MCP server configuration
* Supports both stdio transport (requires command) and HTTP transport (requires url)
*/
function isValidExternalMCPServerConfig(config) {
if (!isNonNullObject(config)) {
return false;
}
const record = config;
// Validate blockedTools array contains only strings
if (record.blockedTools !== undefined) {
if (!Array.isArray(record.blockedTools)) {
return false;
}
if (!record.blockedTools.every((item) => typeof item === "string")) {
return false;
}
}
// Must have either command (for stdio) or url (for HTTP/SSE transport)
const hasCommand = typeof record.command === "string";
const hasUrl = typeof record.url === "string";
if (!hasCommand && !hasUrl) {
return false;
}
return ((record.command === undefined || typeof record.command === "string") &&
(record.args === undefined || Array.isArray(record.args)) &&
(record.env === undefined || isNonNullObject(record.env)) &&
(record.transport === undefined || typeof record.transport === "string") &&
(record.timeout === undefined || typeof record.timeout === "number") &&
(record.retries === undefined || typeof record.retries === "number") &&
(record.healthCheckInterval === undefined ||
typeof record.healthCheckInterval === "number") &&
(record.autoRestart === undefined ||
typeof record.autoRestart === "boolean") &&
(record.cwd === undefined || typeof record.cwd === "string") &&
(record.url === undefined || typeof record.url === "string") &&
(record.headers === undefined || isNonNullObject(record.headers)) &&
(record.httpOptions === undefined || isNonNullObject(record.httpOptions)) &&
(record.retryConfig === undefined || isNonNullObject(record.retryConfig)) &&
(record.rateLimiting === undefined ||
isNonNullObject(record.rateLimiting)) &&
(record.metadata === undefined || isNonNullObject(record.metadata)));
}
/**
* ExternalServerManager
* Core class for managing external MCP servers
*/
/**
* Process-signal cleanup, shared across every manager instance.
*
* Each delegated worker constructs its own NeuroLink and therefore its own
* manager; registering SIGINT/SIGTERM/beforeExit per instance leaked listeners
* (never removed) and crossed Node's 10-listener default on multi-worker runs.
* One process-level trio walks the live set instead; `shutdown()` removes the
* instance from the set, so a disposed manager costs nothing.
*/
const liveManagers = new Set();
let processCleanupInstalled = false;
const cleanupLiveManagers = async () => {
// Concurrent across managers, matching the pattern one level down: a
// single manager's own shutdown() already fans its servers out via
// Promise.all rather than awaiting them one at a time. A sequential
// for-await here would make total SIGTERM/beforeExit cleanup time scale
// with the number of concurrently-alive managers instead of being bounded
// by the slowest one — real for this codebase, since the shared handler
// set above exists specifically because multiple managers are commonly
// alive at once (one per worker an agent delegates to).
await Promise.all(Array.from(liveManagers).map((manager) => manager.shutdown().catch((error) => {
mcpLogger.error("[ExternalServerManager] Error during process-exit cleanup:", error);
})));
};
/**
* beforeExit fires when the event loop is about to go idle. The cleanup it
* kicks off does its own async I/O, so this cycle will not drain — Node
* fires beforeExit again once that work resolves and finds nothing left,
* which is how the process is meant to exit on its own here, without an
* explicit process.exit() call.
*/
const shutdownOnBeforeExit = () => {
void cleanupLiveManagers();
};
/**
* signal-exit@4 (reached at runtime through `ora` -> `cli-cursor` ->
* `restore-cursor`, a direct dependency this package's CLI spinners use)
* makes the mirror-image decision to the gate below: it skips its own
* post-cleanup re-raise whenever it sees another listener for the signal
* still registered. Counting signal-exit's listener as host-owned here means
* both sides defer to the other and neither ever re-raises — a live
* spinner's cursor-restore listener would silently reintroduce the exact
* defect this file exists to fix.
*
* There's no reference to signal-exit's own listener function to compare
* against (it's private state inside that package), so it's detected the
* same way signal-exit's v3/v4 majors detect each other: a shared marker
* signal-exit itself defines. Every signal-exit v4 instance is a
* `globalThis`-wide singleton keyed by `Symbol.for("signal-exit emitter")`
* — but that key is shared across ANY number of distinct resolved copies of
* the package, not just identical ones. If two transitive dependencies pin
* non-overlapping signal-exit@4 ranges, npm/pnpm will not dedupe them, and
* each loaded copy independently increments the shared emitter's `count` by
* one AND registers its own real, distinct listener per signal it covers
* (SIGINT and SIGTERM both are, in every signal-exit v4 release). So `count`
* is a faithful running total of real signal-exit-owned listeners on
* `process` — one per loaded copy — and has to be discounted in full,
* never clamped to at most one, or a second loaded copy's very real
* listener gets miscounted as a host's own and silently defeats the
* re-raise this file exists to perform.
*/
const SIGNAL_EXIT_EMITTER = Symbol.for("signal-exit emitter");
const getSignalExitOwnedListenerCount = () => {
const globalWithSignalExit = globalThis;
const count = globalWithSignalExit[SIGNAL_EXIT_EMITTER]?.count;
return typeof count === "number" && count > 0 ? count : 0;
};
/**
* SIGINT/SIGTERM: registering a listener for either suppresses Node's
* default immediate-termination behavior for that signal. The previous
* handler ran cleanup fire-and-forget and never terminated the process
* afterward, so the process survived a signal it claimed to handle and
* needed SIGKILL to actually stop.
*
* This must NOT call process.exit() itself — this is library code that can
* run inside a host application, and unilaterally choosing the process's
* exit code/timing pre-empts whatever shutdown behavior the host installed
* for this same signal. Instead, once cleanup settles, remove our own
* listener and re-send the signal: if the host registered its own handler
* it now runs (we are no longer in front of it), and if nothing else is
* listening, Node's default terminate-on-signal behavior applies and the
* exit code is the correct signal-based one without us deciding it. Either
* way the process actually stops in response to the signal, which is the
* behavior this fixes — we just stop being the one that forces it down.
*/
const shutdownOnSignal = (signal) => {
// Whether anyone ELSE is listening has to be decided HERE, synchronously, at
// the top of the dispatch — not after cleanup resolves. Node removes a
// `process.once` listener as it dispatches to it, so a host using the very
// common one-shot shutdown shape leaves a listener count of zero behind
// while its own async drain is still running. Reading the count later would
// see that zero, conclude nobody was listening, re-raise, and kill the host
// mid-shutdown — the exact harm this gate exists to prevent, reached from
// the other side. Pairing this snapshot with `prependListener` at
// registration is what makes it correct: we run before any other listener,
// so nothing has been consumed yet and every host listener is still counted.
//
// signal-exit's own listener (if loaded) is subtracted out first: it is a
// cursor-restore helper, not a host that owns this process's shutdown
// decision, and counting it as one would defer to it forever (see above).
const hadOtherListeners = process.listenerCount(signal) - getSignalExitOwnedListenerCount() > 1;
void cleanupLiveManagers().finally(() => {
// With a host handler present, that handler already received this delivery
// and already made its own keep-alive/exit decision; re-sending would run
// it a second time it never asked for, and for the common "first signal
// drains, second one forces" shape that tears it down early.
//
// Note what is NOT done in that case: the listener stays registered. A
// host that owns the signal may well decide to keep running, and
// `processCleanupInstalled` is a one-shot latch — so a listener removed
// here is never reinstalled, and any manager constructed afterwards would
// silently lose signal cleanup for the rest of the process's life. The
// removal exists for exactly one purpose, to let the re-raise reach Node's
// default disposition instead of coming back to us, so it belongs with the
// re-raise and nowhere else.
if (!hadOtherListeners) {
process.removeListener(signal, shutdownOnSignal);
process.kill(process.pid, signal);
}
});
};
const registerManagerForProcessCleanup = (manager) => {
liveManagers.add(manager);
if (!processCleanupInstalled) {
processCleanupInstalled = true;
// prependListener, not on: the gate above counts listeners at dispatch, and
// that count is only trustworthy if nothing has been consumed yet. Running
// first guarantees that. It also costs a host nothing — our handler only
// starts async cleanup and never blocks the listeners behind it.
process.prependListener("SIGINT", shutdownOnSignal);
process.prependListener("SIGTERM", shutdownOnSignal);
process.on("beforeExit", shutdownOnBeforeExit);
}
};
export class ExternalServerManager extends EventEmitter {
servers = new Map();
config;
isShuttingDown = false;
/** Servers whose previous ping has not returned; the interval must not stack. */
healthChecksInFlight = new Set();
/** Tool calls currently executing per server; a busy server is not a hung one. */
inFlightToolCalls = new Map();
/** Consecutive pings missed while idle, per server. */
unresponsiveChecks = new Map();
toolDiscovery;
enableMainRegistryIntegration;
hitlManager; // Optional HITL manager for safety mechanisms
constructor(config = {}, options = {}) {
super();
// Set defaults for configuration
// Default timeout increased to 60s and made configurable via MCP_CLIENT_TIMEOUT
// to accommodate MCP server startup latency (especially concurrent stdio servers)
const defaultMcpTimeout = Math.max(5000, Number(process.env.MCP_CLIENT_TIMEOUT) || 60000);
this.config = {
maxServers: config.maxServers ?? 10,
defaultTimeout: config.defaultTimeout ?? defaultMcpTimeout,
defaultHealthCheckInterval: config.defaultHealthCheckInterval ?? 30000,
enableAutoRestart: config.enableAutoRestart ?? true,
maxRestartAttempts: config.maxRestartAttempts ?? 3,
restartBackoffMultiplier: config.restartBackoffMultiplier ?? 2,
enablePerformanceMonitoring: config.enablePerformanceMonitoring ?? true,
logLevel: config.logLevel ?? "info",
};
// Disable main tool registry integration by default to prevent automatic tool execution
this.enableMainRegistryIntegration =
options.enableMainRegistryIntegration ?? false;
// Initialize tool discovery service
this.toolDiscovery = new ToolDiscoveryService();
// Forward tool discovery events
this.toolDiscovery.on("toolRegistered", (event) => {
this.emit("toolDiscovered", {
...event,
serverName: this.getServerName(event.serverId),
});
});
this.toolDiscovery.on("toolUnregistered", (event) => {
this.emit("toolRemoved", {
...event,
serverName: this.getServerName(event.serverId),
});
});
// Handle process cleanup through the SHARED handler set: one trio of process
// listeners for every manager, however many instances a run constructs.
// Per-instance `process.on(...)` here crossed Node's 10-listener default as
// soon as an agent delegated to a handful of workers (each worker builds its
// own NeuroLink, which builds its own manager) and warned MaxListenersExceeded.
registerManagerForProcessCleanup(this);
}
/**
* Attach a McpOutputNormalizer to the underlying ToolDiscoveryService.
* All tool outputs will be measured and (if oversized) replaced with compact
* surrogates before being returned to callers.
*/
setOutputNormalizer(normalizer) {
this.toolDiscovery.setOutputNormalizer(normalizer);
mcpLogger.debug("[ExternalServerManager] MCP output normalizer attached to ToolDiscoveryService");
}
/**
* Set HITL manager for human-in-the-loop safety mechanisms
* @param hitlManager - HITL manager instance (optional, can be undefined to disable)
*/
setHITLManager(hitlManager) {
this.hitlManager = hitlManager;
if (hitlManager && hitlManager.isEnabled()) {
mcpLogger.info("[ExternalServerManager] HITL safety mechanisms enabled for external tool execution");
}
else {
mcpLogger.debug("[ExternalServerManager] HITL safety mechanisms disabled or not configured");
}
}
/**
* Get current HITL manager
*/
getHITLManager() {
return this.hitlManager;
}
/**
* Resolve the human-readable server name for an event payload.
* Falls back to serverId if the instance or config.name isn't available.
*/
getServerName(serverId) {
const instance = this.servers.get(serverId);
return instance?.config?.name || serverId;
}
/**
* Load MCP server configurations from .mcp-config.json file with parallel loading support
* Automatically registers servers found in the configuration
* @param configPath Optional path to config file (defaults to .mcp-config.json in cwd)
* @param options Loading options including parallel support
* @returns Promise resolving to { serversLoaded, errors }
*/
async loadMCPConfiguration(configPath, options = {}) {
if (options.parallel) {
return this.loadMCPConfigurationParallel(configPath);
}
return this.loadMCPConfigurationSequential(configPath);
}
/**
* Load MCP servers in parallel for improved performance
* @param configPath Optional path to config file (defaults to .mcp-config.json in cwd)
* @returns Promise resolving to batch operation result
*/
async loadMCPConfigurationParallel(configPath) {
const fs = await import("fs");
const path = await import("path");
const finalConfigPath = configPath || path.join(process.cwd(), ".mcp-config.json");
if (!fs.existsSync(finalConfigPath)) {
mcpLogger.debug(`[ExternalServerManager] No MCP config found at ${finalConfigPath}`);
return { serversLoaded: 0, errors: [] };
}
mcpLogger.debug(`[ExternalServerManager] Loading MCP configuration in PARALLEL mode from ${finalConfigPath}`);
try {
const configContent = fs.readFileSync(finalConfigPath, "utf8");
const config = JSON.parse(configContent);
if (!config.mcpServers || typeof config.mcpServers !== "object") {
mcpLogger.debug("[ExternalServerManager] No mcpServers found in configuration");
return { serversLoaded: 0, errors: [] };
}
// Create promises for all servers to start them concurrently
const serverPromises = Object.entries(config.mcpServers).map(async ([serverId, serverConfig]) => {
try {
// Validate and convert config format to MCPServerInfo
if (!isValidExternalMCPServerConfig(serverConfig)) {
throw new Error(`Invalid server config for ${serverId}: missing required properties or wrong types`);
}
const externalConfig = {
id: serverId,
name: serverId,
description: typeof serverConfig.description === "string"
? serverConfig.description
: `External MCP server: ${serverId}`,
transport: typeof serverConfig.transport === "string"
? serverConfig.transport
: "stdio",
status: "initializing",
tools: [],
command: typeof serverConfig.command === "string"
? serverConfig.command
: undefined,
args: Array.isArray(serverConfig.args)
? serverConfig.args
: [],
env: isNonNullObject(serverConfig.env)
? substituteEnvVariables(serverConfig.env)
: {},
timeout: typeof serverConfig.timeout === "number"
? serverConfig.timeout
: undefined,
retries: typeof serverConfig.retries === "number"
? serverConfig.retries
: undefined,
healthCheckInterval: typeof serverConfig.healthCheckInterval === "number"
? serverConfig.healthCheckInterval
: undefined,
autoRestart: typeof serverConfig.autoRestart === "boolean"
? serverConfig.autoRestart
: undefined,
cwd: typeof serverConfig.cwd === "string"
? serverConfig.cwd
: undefined,
url: typeof serverConfig.url === "string"
? serverConfig.url
: undefined,
// HTTP transport-specific fields
headers: isNonNullObject(serverConfig.headers)
? substituteEnvVariables(serverConfig.headers)
: undefined,
httpOptions: isNonNullObject(serverConfig.httpOptions)
? serverConfig.httpOptions
: undefined,
retryConfig: isNonNullObject(serverConfig.retryConfig)
? serverConfig.retryConfig
: undefined,
rateLimiting: isNonNullObject(serverConfig.rateLimiting)
? serverConfig.rateLimiting
: undefined,
blockedTools: Array.isArray(serverConfig.blockedTools)
? serverConfig.blockedTools
: undefined,
metadata: safeMetadataConversion(serverConfig.metadata),
};
const result = await this.addServer(serverId, externalConfig);
return { serverId, result };
}
catch (error) {
const errorMsg = `Failed to load MCP server ${serverId}: ${error instanceof Error ? error.message : String(error)}`;
// No log here: the result-processing loop below owns the single
// ERROR record for this failure (logging in both places
// double-counted every parallel-load failure).
return { serverId, error: errorMsg };
}
});
// Start all servers concurrently and wait for completion
const results = await Promise.allSettled(serverPromises);
// Process results to count successes and collect errors
let serversLoaded = 0;
const errors = [];
for (const result of results) {
if (result.status === "fulfilled") {
const { serverId, result: serverResult, error } = result.value;
if (serverResult && serverResult.success) {
serversLoaded++;
mcpLogger.debug(`[ExternalServerManager] Successfully loaded MCP server in parallel: ${serverId}`);
}
else if (error) {
errors.push(error);
// Config-loaded servers never pass through
// NeuroLink.addExternalMCPServer, so this is the outermost point
// for them — it owns the single ERROR record per failed server.
mcpLogger.error(`[ExternalServerManager] Failed to load server ${serverId}: ${error}`);
}
else if (serverResult && !serverResult.success) {
const errorMsg = `Failed to load server ${serverId}: ${serverResult.error}`;
errors.push(errorMsg);
// See above: outermost point for config-loaded servers.
mcpLogger.error(`[ExternalServerManager] ${errorMsg}`);
}
}
else {
// Promise.allSettled rejected - this shouldn't happen with our error handling
const errorMsg = `Unexpected error during parallel loading: ${result.reason}`;
errors.push(errorMsg);
mcpLogger.error(`[ExternalServerManager] ${errorMsg}`);
}
}
mcpLogger.info(`[ExternalServerManager] PARALLEL MCP configuration loading complete: ${serversLoaded} servers loaded, ${errors.length} errors`);
return { serversLoaded, errors };
}
catch (error) {
const errorMsg = `Failed to load MCP configuration in parallel mode: ${error instanceof Error ? error.message : String(error)}`;
mcpLogger.error(`[ExternalServerManager] ${errorMsg}`);
return { serversLoaded: 0, errors: [errorMsg] };
}
}
/**
* Load MCP servers sequentially (original implementation for backward compatibility)
* @param configPath Optional path to config file (defaults to .mcp-config.json in cwd)
* @returns Promise resolving to batch operation result
*/
async loadMCPConfigurationSequential(configPath) {
const fs = await import("fs");
const path = await import("path");
const finalConfigPath = configPath || path.join(process.cwd(), ".mcp-config.json");
if (!fs.existsSync(finalConfigPath)) {
mcpLogger.debug(`[ExternalServerManager] No MCP config found at ${finalConfigPath}`);
return { serversLoaded: 0, errors: [] };
}
mcpLogger.debug(`[ExternalServerManager] Loading MCP configuration from ${finalConfigPath}`);
try {
const configContent = fs.readFileSync(finalConfigPath, "utf8");
const config = JSON.parse(configContent);
if (!config.mcpServers || typeof config.mcpServers !== "object") {
mcpLogger.debug("[ExternalServerManager] No mcpServers found in configuration");
return { serversLoaded: 0, errors: [] };
}
let serversLoaded = 0;
const errors = [];
for (const [serverId, serverConfig] of Object.entries(config.mcpServers)) {
try {
// Validate and convert config format to MCPServerInfo
if (!isValidExternalMCPServerConfig(serverConfig)) {
throw new Error(`Invalid server config for ${serverId}: missing required properties or wrong types`);
}
const externalConfig = {
id: serverId,
name: serverId,
description: typeof serverConfig.description === "string"
? serverConfig.description
: `External MCP server: ${serverId}`,
transport: typeof serverConfig.transport === "string"
? serverConfig.transport
: "stdio",
status: "initializing",
tools: [],
command: typeof serverConfig.command === "string"
? serverConfig.command
: undefined,
args: Array.isArray(serverConfig.args)
? serverConfig.args
: [],
env: isNonNullObject(serverConfig.env)
? substituteEnvVariables(serverConfig.env)
: {},
timeout: typeof serverConfig.timeout === "number"
? serverConfig.timeout
: undefined,
retries: typeof serverConfig.retries === "number"
? serverConfig.retries
: undefined,
healthCheckInterval: typeof serverConfig.healthCheckInterval === "number"
? serverConfig.healthCheckInterval
: undefined,
autoRestart: typeof serverConfig.autoRestart === "boolean"
? serverConfig.autoRestart
: undefined,
cwd: typeof serverConfig.cwd === "string"
? serverConfig.cwd
: undefined,
url: typeof serverConfig.url === "string"
? serverConfig.url
: undefined,
// HTTP transport-specific fields
headers: isNonNullObject(serverConfig.headers)
? substituteEnvVariables(serverConfig.headers)
: undefined,
httpOptions: isNonNullObject(serverConfig.httpOptions)
? serverConfig.httpOptions
: undefined,
retryConfig: isNonNullObject(serverConfig.retryConfig)
? serverConfig.retryConfig
: undefined,
rateLimiting: isNonNullObject(serverConfig.rateLimiting)
? serverConfig.rateLimiting
: undefined,
blockedTools: Array.isArray(serverConfig.blockedTools)
? serverConfig.blockedTools
: undefined,
metadata: safeMetadataConversion(serverConfig.metadata),
};
const result = await this.addServer(serverId, externalConfig);
if (result.success) {
serversLoaded++;
mcpLogger.debug(`[ExternalServerManager] Successfully loaded MCP server: ${serverId}`);
}
else {
const error = `Failed to load server ${serverId}: ${result.error}`;
errors.push(error);
// Config-loaded servers never pass through
// NeuroLink.addExternalMCPServer — this sequential-load branch is
// their outermost point and owns the single ERROR record.
mcpLogger.error(`[ExternalServerManager] ${error}`);
}
}
catch (error) {
const errorMsg = `Failed to load MCP server ${serverId}: ${error instanceof Error ? error.message : String(error)}`;
errors.push(errorMsg);
// See above: outermost point for config-loaded servers.
mcpLogger.error(`[ExternalServerManager] ${errorMsg}`);
// Continue with other servers - don't let one failure break everything
}
}
mcpLogger.info(`[ExternalServerManager] MCP configuration loading complete: ${serversLoaded} servers loaded, ${errors.length} errors`);
return { serversLoaded, errors };
}
catch (error) {
const errorMsg = `Failed to load MCP configuration: ${error instanceof Error ? error.message : String(error)}`;
mcpLogger.error(`[ExternalServerManager] ${errorMsg}`);
return { serversLoaded: 0, errors: [errorMsg] };
}
}
/**
* Validate external MCP server configuration
*/
validateConfig(config) {
const errors = [];
const warnings = [];
const suggestions = [];
// Required fields validation
if (!config.id || typeof config.id !== "string") {
errors.push("Server ID is required and must be a string");
}
if (!["stdio", "sse", "websocket", "http"].includes(config.transport)) {
errors.push("Transport must be one of: stdio, sse, websocket, http");
}
// Transport-specific validation
if (config.transport === "stdio") {
// stdio transport requires command
if (!config.command || typeof config.command !== "string") {
errors.push("Command is required and must be a string for stdio transport");
}
if (!Array.isArray(config.args)) {
errors.push("Args must be an array");
}
}
else if (config.transport === "sse" ||
config.transport === "websocket" ||
config.transport === "http") {
// HTTP-based transports require URL
if (!config.url || typeof config.url !== "string") {
errors.push(`URL is required for ${config.transport} transport`);
}
}
// Warnings for common issues
if (config.timeout && config.timeout < 5000) {
warnings.push("Timeout less than 5 seconds may cause connection issues");
}
if (config.retries && config.retries > 5) {
warnings.push("High retry count may slow down error recovery");
}
// Suggestions for optimization
if (!config.healthCheckInterval) {
suggestions.push("Consider setting a health check interval for better reliability");
}
if (config.autoRestart === undefined) {
suggestions.push("Consider enabling auto-restart for production use");
}
return {
isValid: errors.length === 0,
errors,
warnings,
suggestions,
};
}
/**
* Convert MCPServerInfo format (keeping for backward compatibility)
* Helper function for transitioning to zero-conversion architecture
*/
convertConfigToMCPServerInfo(serverId, config) {
return {
id: serverId,
name: String(config.metadata?.title || serverId),
description: `External MCP server (${config.transport})`,
status: "initializing",
transport: config.transport,
command: config.command,
args: config.args,
env: config.env,
tools: [], // Will be populated after server connection
blockedTools: config.blockedTools,
// Preserve top-level operational fields so startServer can read them
timeout: config.timeout,
retries: config.retries,
healthCheckInterval: config.healthCheckInterval,
autoRestart: config.autoRestart,
minTools: config.minTools,
cwd: config.cwd,
url: config.url,
metadata: {
category: "external",
...(safeMetadataConversion(config.metadata) || {}),
},
};
}
async addServer(serverId, configOrServerInfo) {
const startTime = Date.now();
try {
// Use MCPServerInfo directly (zero-conversion architecture)
const serverInfo = "transport" in configOrServerInfo &&
"command" in configOrServerInfo &&
!("tools" in configOrServerInfo)
? this.convertConfigToMCPServerInfo(serverId, configOrServerInfo)
: configOrServerInfo;
// Check server limit
if (this.servers.size >= this.config.maxServers) {
return {
success: false,
error: `Maximum number of servers (${this.config.maxServers}) reached`,
serverId,
duration: Date.now() - startTime,
};
}
// Validate configuration (for backward compatibility, create temporary config)
const tempConfig = {
id: serverId,
name: serverInfo.name,
description: serverInfo.description,
transport: serverInfo.transport,
status: serverInfo.status,
tools: serverInfo.tools,
command: serverInfo.command || "",
args: serverInfo.args || [],
env: serverInfo.env || {},
timeout: serverInfo.timeout,
retries: serverInfo.retries,
healthCheckInterval: serverInfo.healthCheckInterval,
autoRestart: serverInfo.autoRestart,
minTools: serverInfo.minTools,
cwd: serverInfo.cwd,
url: serverInfo.url,
blockedTools: serverInfo.blockedTools,
metadata: safeMetadataConversion(serverInfo.metadata),
// HTTP transport-specific fields
headers: serverInfo.headers,
httpOptions: serverInfo.httpOptions,
retryConfig: serverInfo.retryConfig,
rateLimiting: serverInfo.rateLimiting,
};
const validation = this.validateConfig(tempConfig);
if (!validation.isValid) {
return {
success: false,
error: `Configuration validation failed: ${validation.errors.join(", ")}`,
serverId,
duration: Date.now() - startTime,
};
}
// Check for duplicate server ID
if (this.servers.has(serverId)) {
return {
success: false,
error: `Server with ID '${serverId}' already exists`,
serverId,
duration: Date.now() - startTime,
};
}
mcpLogger.info(`[ExternalServerManager] Adding server: ${serverId}`, {
command: serverInfo.command,
transport: serverInfo.transport,
});
// Create server instance as RuntimeMCPServerInfo (transition to zero-conversion)
const instance = {
...serverInfo,
process: null,
client: null,
transportInstance: null,
status: "initializing",
reconnectAttempts: 0,
maxReconnectAttempts: this.config.maxRestartAttempts,
toolsMap: new Map(),
metrics: {
totalConnections: 0,
totalDisconnections: 0,
totalErrors: 0,
totalToolCalls: 0,
averageResponseTime: 0,
lastResponseTime: 0,
},
config: tempConfig,
};
// Store the instance
this.servers.set(serverId, instance);
// Start the server
await this.startServer(serverId);
const finalInstance = this.servers.get(serverId);
if (!finalInstance) {
throw new Error(`Server ${serverId} not found after registration`);
}
// Convert RuntimeMCPServerInfo to ExternalMCPServerInstance for return
const convertedInstance = {
config: finalInstance.config,
process: finalInstance.process,
pid: finalInstance.pid,
client: finalInstance.client,
transport: finalInstance.transportInstance,
status: finalInstance.status,
lastError: finalInstance.lastError,
startTime: finalInstance.startTime,
lastHealthCheck: finalInstance.lastHealthCheck,
reconnectAttempts: finalInstance.reconnectAttempts,
maxReconnectAttempts: finalInstance.maxReconnectAttempts,
tools: finalInstance.toolsMap,
toolsArray: finalInstance.toolsArray,
capabilities: finalInstance.capabilities,
healthTimer: finalInstance.healthTimer,
restartTimer: finalInstance.restartTimer,
metrics: finalInstance.metrics,
};
return {
success: true,
data: convertedInstance,
serverId,
duration: Date.now() - startTime,
metadata: {
timestamp: Date.now(),
operation: "addServer",
readiness: "ready",
toolsDiscovered: finalInstance.tools.length,
},
};
}
catch (error) {
// debug, not error: NeuroLink.addExternalMCPServer (the public entry
// point) emits the single ERROR record for a failed registration.
mcpLogger.debug(`[ExternalServerManager] Failed to add server ${serverId}:`, error);
// Clean up if instance was created
this.servers.delete(serverId);
// The minTools readiness gate carries structured metadata a generic
// start failure does not: how many tools were actually discovered, so
// a caller can distinguish "too few tools" from "could not connect".
if (error instanceof InsufficientToolsError) {
return {
success: false,
error: error.message,
serverId,
duration: Date.now() - startTime,
metadata: {
timestamp: Date.now(),
operation: "addServer",
readiness: error.readiness,
toolsDiscovered: error.toolsDiscovered,
},
};
}
return {
success: false,
error: error instanceof Error ? error.message : String(error),
serverId,
duration: Date.now() - startTime,
};
}
}
/**
* Remove an external MCP server
*/
async removeServer(serverId) {
const startTime = Date.now();
try {
const instance = this.servers.get(serverId);
if (!instance) {
return {
success: false,
error: `Server '${serverId}' not found`,
serverId,
duration: Date.now() - startTime,
};
}
mcpLogger.info(`[ExternalServerManager] Removing server: ${serverId}`);
// Capture name before deletion removes the instance
const serverName = this.getServerName(serverId);
// Stop the server
await this.stopServer(serverId);
// Remove from registry
this.servers.delete(serverId);
// Emit event
this.emit("disconnected", {
serverId,
serverName,
reason: "Manually removed",
timestamp: new Date(),
});
return {
success: true,
serverId,
duration: Date.now() - startTime,
metadata: {
timestamp: Date.now(),
operation: "removeServer",
},
};
}
catch (error) {
mcpLogger.error(`[ExternalServerManager] Failed to remove server ${serverId}:`, error);
return {
success: false,
error: error instanceof Error ? error.message : String(error),
serverId,
duration: Date.now() - startTime,
};
}
}
/**
* Start an external MCP server
*/
async startServer(serverId) {
const instance = this.servers.get(serverId);
if (!instance) {
throw new Error(`Server '${serverId}' not found`);
}
const config = instance.config;
const span = tracers.mcp.startSpan("neurolink.mcp.server.start", {
attributes: {
"mcp.server_id": serverId,
"mcp.transport": config.transport,
"mcp.command_name": config.command
? config.command.split(/[\\/]/).pop() || ""
: "",
"mcp.command_present": Boolean(config.command),
},
});
try {
this.updateServerStatus(serverId, "connecting");
mcpLogger.debug(`[ExternalServerManager] Starting server: ${serverId}`, {
command: config.command,
args: redactSensitiveArgs(config.args),
transport: config.transport,
});
// Create MCP client using the factory
const clientResult = await MCPClientFactory.createClient(config, config.timeout || this.config.defaultTimeout);
if (!clientResult.success ||
!clientResult.client ||
!clientResult.transport) {
throw new Error(`Failed to create MCP client: ${clientResult.error}`);
}
// Store client components
instance.client = clientResult.client;
instance.transportInstance = clientResult.transport;
instance.process = clientResult.process || null;
const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js");
instance.pid =
clientResult.transport instanceof StdioClientTransport
? (clientResult.transport.pid ?? undefined)
: undefined;
instance.capabilities = safeMetadataConversion(clientResult.capabilities);
instance.startTime = new Date();
instance.lastHealthCheck = new Date();
instance.metrics.totalConnections++;
this.unresponsiveChecks.delete(serverId);
this.attachClientLifecycle(serverId, clientResult.client, clientResult.transport);
this.updateServerStatus(serverId, "connected");
// Discover tools from the server
await this.discoverServerTools(serverId);
// Readiness gate: a server whose discovered tool count falls below
// its configured minTools (default 0 — no minimum, so resource/prompt
// -only servers stay valid) is torn back down instead of being marked
// connected/healthy. Thrown here — before tools are registered with
// the main registry and before health monitoring starts — so a
// rejected server is never marked ready even transiently, and the
// throw is caught by this method's own catch below, which sets status
// "failed" and rethrows to addServer's catch the same way any other
// start failure does.
const minTools = config.minTools ?? 0;
if (instance.toolsMap.size < minTools) {
const toolsDiscovered = instance.toolsMap.size;
mcpLogger.warn(`[ExternalServerManager] Server '${serverId}' discovered ${toolsDiscovered} tool(s), below the configured minTools=${minTools}; tearing the connection back down`);
await this.closeInstanceConnection(serverId, instance);
throw new InsufficientToolsError(serverId, toolsDiscovered, minTools);
}
// Register tools with main registry if integration is enabled
if (this.enableMainRegistryIntegration) {
await this.registerServerToolsWithMainRegistry(serverId);
}
// Start health monitoring
this.startHealthMonitoring(serverId);
// Emit connected event
this.emit("connected", {
serverId,
serverName: this.getServerName(serverId),
toolCount: instance.toolsMap.size,
timestamp: new Date(),
});
span.setAttribute("mcp.tool_count", instance.toolsMap.size);
span.setStatus({ code: SpanStatusCode.OK });
mcpLogger.info(`[ExternalServerManager] Server started successfully: ${serverId}`);
}
catch (error) {
// debug, not error: the failure is rethrown below and surfaces once at
// the NeuroLink.addExternalMCPServer entry point.
mcpLogger.debug(`[ExternalServerManager] Failed to start server ${serverId}:`, error);
this.updateServerStatus(serverId, "failed");
instance.lastError =
error instanceof Error ? error.message : String(error);
span.recordException(error instanceof Error ? error : new Error(String(error)));
span.setStatus({
code: SpanStatusCode.ERROR,
message: error instanceof Error ? error.message : String(error),
});
throw error;
}
finally {
span.end();
}
}
/**
* Close a live client/transport pair, clearing the lifecycle hooks first
* so the `onclose` they would otherwise trigger cannot schedule a restart
* of a connection being torn down deliberately. Shared by `stopServer` and
* the minTools readiness gate in `startServer`, which rejects a server
* before it is ever registered as connected. Leaves status untouched —
* callers set whatever status fits their case.
*/
async closeInstanceConnection(serverId, instance) {
if (!instance.client || !instance.transportInstance) {
return;
}
instance.client.onclose = undefined;
instance.client.onerror = undefined;
try {
// Forward the child handle so the factory's SIGKILL escalation can run
// when the transport's own close does not bring the process down.
// NOTE: `instance.process` is currently always null for stdio servers —
// MCPClientFactory.createStdioTransport returns only `{ transport }`, so
// `clientResult.process` is never populated and the escalation inside
// closeClient remains unreachable. Surfacing the spawned child from the
// transport is a separate change; this argument is wired so that it
// starts working the moment it is.
await MCPClientFactory.closeClient(instance.client, instance.transportInstance, instance.process ?? undefined);
}
catch (error) {
mcpLogger.debug(`[ExternalServerManager] Error closing client for ${serverId}:`, error);
}
instance.client = null;
instance.transportInstance = null;
instance.process = null;
instance.pid = undefined;
}
/**
* Stop an external MCP server
*/
async stopServer(serverId) {
const instance = this.servers.get(serverId);
if (!instance) {
return;
}
const span = tracers.mcp.startSpan("neurolink.mcp.server.stop", {
attributes: {
"mcp.server_id": serverId,
},
});
try {
this.updateServerStatus(serverId, "stopping");
// Clear timers
if (instance.healthTimer) {
clearInterval(instance.healthTimer);
instance.healthTimer = undefined;
}
if (instance.restartTimer) {
clearTimeout(instance.restartTimer);
instance.restartTimer = undefined;
}
// Unregister tools from main registry if integration is enabled
if (this.enableMainRegistryIntegration) {
this.unregisterServerToolsFromMainRegistry(serverId);
}
// Clear server tools from discovery service
this.toolDiscovery.clearServerTools(serverId);
// Close MCP client using factory cleanup. This close is ours, not a
// crash: closeInstanceConnection drops the lifecycle hooks first so
// the onclose it triggers cannot schedule a restart of a server we
// are deliberately stopping.
await this.closeInstanceConnection(serverId, instance);
this.updateServerStatus(serverId, "stopped");
span.setStatus({ code: SpanStatusCode.OK });
mcpLogger.info(`[ExternalServerManager] Server stopped: ${serverId}`);
}
catch (error) {
mcpLogger.error(`[ExternalServerManager] Error stopping server ${serverId}:`, error);
this.updateServerStatus(serverId, "failed");
span.recordException(error instanceof Error ? error : new Error(String(error)));
span.setStatus({
code: SpanStatusCode.ERROR,
message: error instanceof Error ? error.message : String(error),
});
}
finally {
span.end();
}
}
/**
* Update server status and emit events
*/
updateServerStatus(serverId, newStatus) {
const instance = this.servers.get(serverId);
if (!instance) {
return;
}
const oldStatus = instance.status;
// Map ExternalMCPServerStatus to MCPServerInfo status
const mappedStatus = newStatus === "connecting" || newStatus === "restarting"
? "initializing"
: newStatus === "stopping" || newStatus === "stopped"
? "stopping"
: newStatus === "connected"
? "connected"
: newStatus === "disconnected"
? "disconnected"
: "failed";
instance.status = mappedStatus;
// Emit status change event
this.emit("statusChanged", {
serverId,
serverName: this.getServerName(serverId),
oldStatus,
newStatus,
timestamp: new Date(),
});
mcpLogger.debug(`[ExternalServerManager] Status changed for ${serverId}: ${oldStatus} -> ${newStatus}`);
}
/**
* Hook the SDK client's lifecycle so the manager learns about the server
* dying from the object that actually owns the process.
*
* `client.onclose` fires when the stdio child closes or a network
* transport drops. It also fires during our own `stopServer()`, which is
* why every path into `handleConnectionLost` checks that the client is
* still the instance's current one and the status is still "connected":
* a close that arrives while stopping or restarting is intentional.
*/
attachClientLifecycle(serverId, client, transport) {
client.onclose = () => {
const instance = this.servers.get(serverId);
const pidNote = instance?.pid ? ` (pid ${instance.pid})` : "";
const stderrTail = MCPClientFactory.getStderrTail(transport);
const reason = stderrTail.length > 0
? `Server process closed${pidNote}; last stderr: ${stderrTail.slice(-5).join(" | ")}`
: `Server process closed${pidNote}`;
this.handleConnectionLost(serverId, client, reason, stderrTail);
};
client.onerror = (error) => {
// Transport-level noise: a stdin EPIPE after the child died, a line on
// stdout that is not JSON-RPC. Diagnostic only — a fatal error is
// followed by onclose, and that is where state changes.
mcpLogger.warn(`[ExternalServerManager] Transport error for ${serverId}: ${error.message}`);
};
}
/**
* Transition a server whose connection is gone, exactly once.
*
* Reached from three places that can all observe the same death — the
* client's onclose, a failed health-check ping, and a tool call rejected
* with the SDK's connection-closed error — so it is idempotent on the
* (client, status) pair rather than trusting any one caller to be first.
*/
handleConnectionLost(serverId, client, reason, stderrTail = []) {
const instance = this.servers.get(serverId);
if (!instance ||
instance.client !== client ||
instance.status !== "connected") {
return;
}
mcpLogger.warn(`[ExternalServerManager] Connection lost for ${serverId}: ${reason}`, stderrTail.length > 0 ? { stderrTail } : undefined);
instance.lastError = reason;
this.handleServerDisconnection(serverId, reason);
}
trackToolCall(serverId, delta) {
const next = (this.inFlightToolCalls.get(serverId) ?? 0) + delta;
if (next <= 0) {
this.inFlightToolCalls.delete(serverId);
}
else {
this.inFlightToolCalls.set(serverId, next);
}
}
/**
* Handle server errors
*/
handleServerError(serverId, error) {
const instance = this.servers.get(serverId);
if (!instance) {
return;
}
instance.lastError = error.message;
instance.metrics.totalErrors++;
mcpLogger.error(`[ExternalServerManager] Server error for ${serverId}:`, error);
// Emit failed event
this.emit("failed", {
serverId,
serverName: this.getServerName(serverId),
error: error.message,
timestamp: new Date(),
});
// Attempt restart if enabled
if (this.config.enableAutoRestart && !this.isShuttingDown) {
this.scheduleRestart(serverId);
}
else {
this.updateServerStatus(serverId, "failed");
}
}
/**
* Handle server disconnection
*/
handleServerDisconnection(serverId, reason) {
const instance = this.servers.get(serverId);
if (!instance) {
return;
}
instance.metrics.totalDisconnections++;
mcpLogger.warn(`[ExternalServerManager] Server disconnected ${serverId}: ${reason}`);
// Emit disconnected event
this.emit("disconnected", {
serverId,
serverName: this.getServerName(serverId),
reason,
timestamp: new Date(),
});
// Attempt restart if enabled — prefer server-specific setting, fall back to global
if ((instance.config.autoRestart ?? this.config.enableAutoRestart) &&
!this.isShuttingDown) {
this.scheduleRestart(serverId);
}
else {
this.updateServerStatus(serverId, "disconnected");
}
}
/**
* Schedule server restart with exponential backoff
*/
scheduleRestart(serverId) {
const instance = this.servers.get(serverId);
if (!instance) {
return;
}
if (instance.reconnectAttempts >= instance.maxReconnectAttempts) {
mcpLogger.error(`[ExternalServerManager] Max restart attempts reached for ${serverId}`);
this.updateServerStatus(serverId, "failed");
return;
}
instance.reconnectAttempts++;
this.updateServerStatus(serverId, "restarting");
const delay = Math.min(1000 *
Math.pow(this.config.restartBackoffMultiplier, instance.reconnectAttempts - 1), 30000);
mcpLogger.info(`[ExternalServerManager] Scheduling restart for ${serverId} in ${delay}ms (attempt ${instance.reconnectAttempts})`);
if (instance.restartTimer) {
return;
} // already scheduled
instance.restartTimer = setTimeout(async () => {
const restartSpan = tracers.mcp.startSpan("neurolink.mcp.server.restart", {
attributes: {
"mcp.server_id": serverId,
"mcp.restart_attempt": instance.reconnectAttempts,
"mcp.restart_delay_ms": delay,
},
});
try {
await this.stopServer(serverId);
await this.startServer(serverId);
// Reset restart attempts on successful restart
instance.reconnectAttempts = 0;
restartSpan.setStatus({ code: SpanStatusCode.OK });
}
catch (error) {
mcpLogger.error(`[ExternalServerManager] Restart failed for ${serverId}:`, error);
restartSpan.recordException(error instanceof Error ? error : new Error(String(error)));
restartSpan.setStatus({
code: SpanStatusCode.ERROR,
message: error instanceof Error ? error.message : String(error),
});
this.scheduleRestart(serverId); // Try again
}
finally {
restartSpan.end();
}
}, delay);
}
/**
* Start health monitoring for a server
*/
startHealthMonitoring(serverId) {
const instance = this.servers.get(serverId);
if (!instance || !this.config.enablePerformanceMonitoring) {
return;
}
const interval = instance.config.healthCheckInterval ??
this.config.defaultHealthCheckInterval;
instance.healthTimer = setInterval(async () => {
await this.performHealthCheck(serverId);
}, interval);
// Unref'd so a live health check on a connected server doesn't itself
// keep an otherwise-idle process alive — a caller that never calls
// shutdown()/stopServer() (e.g. a script that just uses the SDK and
// lets the process exit naturally) shouldn't be forced to wait out
// this interval. stopServer() still clearInterval()s it for explicit
// lifecycle management.
if (instance.healthTimer.unref) {
instance.healthTimer.unref();
}
}
/**
* Perform a health check by pinging the server over its live transport.
*
* The previous check read `process.killed` — a flag Node sets only when we
* call `kill()` ourselves, on a process handle that was not even the
* server's — so it could never observe a server that died on its own.
*
* A ping that comes back with the SDK's connection-closed error is a
* definitive death and is handled immediately. A ping that merely times
* out is ambiguous: single-threaded servers (a Python MCP server running a
* synchronous tool) cannot answer while a tool call is executing, and
* restarting one mid-call would destroy the work in flight. So a timeout
* counts toward a restart only while the server has no tool call in
* flight, and only after UNRESPONSIVE_CHECKS_BEFORE_RESTART consecutive
* misses.
*/
async performHealthCheck(serverId) {
const instance = this.servers.get(serverId);
if (!instance || instance.status !== "connected" || !instance.client) {
return;
}
if (this.healthChecksInFlight.has(serverId)) {
return;
}
this.healthChecksInFlight.add(serverId);
const client = instance.client;
const interval = instance.config.healthCheckInterval ??
this.config.defaultHealthCheckInterval;
const pingTimeoutMs = Math.max(1, Math.min(instance.config.timeout || this.config.defaultTimeout, interval));
const startTime = Date.now();
const issues = [];
let isHealthy = true;
let connectionLost = false;
let unresponsive = false;
try {
try {
await client.ping({ timeout: pingTimeoutMs });
this.unresponsiveChecks.delete(serverId);
}
catch (error) {
isHealthy = false;
if (isConnectionLostError(error)) {
connectionLost = true;
issues.push("connection closed");
}
else {
const inFlight = this.inFlightToolCalls.get(serverId) ?? 0;
if (inFlight > 0) {
issues.push(`ping timed out after ${pingTimeoutMs}ms while ${inFlight} tool call(s) in flight`);
}
else {
const misses = (this.unresponsiveChecks.get(serverId) ?? 0) + 1;
this.unresponsiveChecks.set(serverId, misses);
issues.push(`ping timed out after ${pingTimeoutMs}ms while idle (${misses}/${UNRESPONSIVE_CHECKS_BEFORE_RESTART})`);
unresponsive = misses >= UNRESPONSIVE_CHECKS_BEFORE_RESTART;
}
}
}
const responseTime = Date.now() - startTime;
instance.lastHealthCheck = new Date();
const health = {
serverId,
isHealthy,
status: instance.status,
checkedAt: new Date(),
responseTime,
toolCount: instance.toolsMap.size,
issues,
performance: {
uptime: instance.startTime
? Date.now() - instance.startTime.getTime()
: 0,
averageResponseTime: instance.metrics.averageResponseTime,
},
};
// Emit health check event
this.emit("healthCheck", {
serverId,
serverName: this.getServerName(serverId),
health,
timestamp: new Date(),
});
if (connectionLost || unresponsive) {
this.handleConnectionLost(serverId, client, `Health check failed: ${issues.join(", ")}`);
}
else if (!isHealthy) {
mcpLogger.warn(`[ExternalServerManager] Health check failed for ${serverId}:`, issues);
}
}
catch (error) {
mcpLogger.error(`[ExternalServerManager] Health check error for ${serverId}:`, error);
}
finally {
this.healthChecksInFlight.delete(serverId);
}
}
/**
* Get server instance - converted to ExternalMCPServerInstance for compatibility
*/
getServer(serverId) {
const runtime = this.servers.get(serverId);
if (!runtime) {
return undefined;
}
return {
config: runtime.config,
process: runtime.process,
pid: runtime.pid,
client: runtime.client,
transport: runtime.transportInstance,
status: runtime.status,
lastError: runtime.lastError,
startTime: runtime.startTime,
lastHealthCheck: runtime.lastHealthCheck,
reconnectAttempts: runtime.reconnectAttempts,
maxReconnectAttempts: runtime.maxReconnectAttempts,
tools: runtime.toolsMap,
toolsArray: runtime.toolsArray,
capabilities: runtime.capabilities,
healthTimer: runtime.healthTimer,
restartTimer: runtime.restartTimer,
metrics: runtime.metrics,
};
}
/**
* Get all servers - converted to ExternalMCPServerInstance for compatibility
*/
getAllServers() {
const converted = new Map();
for (const [serverId, runtime] of this.servers.entries()) {
converted.set(serverId, {
config: runtime.config,
process: runtime.process,
pid: runtime.pid,
client: runtime.client,
transport: runtime.transportInstance,
status: runtime.status,
lastError: runtime.lastError,
startTime: runtime.startTime,
lastHealthCheck: runtime.lastHealthCheck,
reconnectAttempts: runtime.reconnectAttempts,
maxReconnectAttempts: runtime.maxReconnectAttempts,
tools: runtime.toolsMap,
toolsArray: runtime.toolsArray,
capabilities: runtime.capabilities,
healthTimer: runtime.healthTimer,
restartTimer: runtime.restartTimer,
metrics: runtime.metrics,
});
}
return converted;
}
/**
* List servers as MCPServerInfo - ZERO conversion needed
*/
listServers() {
return Array.from(this.servers.values());
}
/**
* Get server statuses
*/
getServerStatuses() {
const statuses = [];
for (const [serverId, instance] of Array.from(this.servers.entries())) {
const uptime = instance.startTime
? Date.now() - instance.startTime.getTime()
: 0;
statuses.push({
serverId,
isHealthy: instance.status === "connected",
status: instance.status,
checkedAt: instance.lastHealthCheck || new Date(),
toolCount: instance.toolsMap.size,
issues: instance.lastError ? [instance.lastError] : [],
performance: {
uptime,
averageResponseTime: instance.metrics.averageResponseTime,
},
});
}
return statuses;
}
/**
* Shutdown all servers and clean up resources
* This method should be called during application shutdown to prevent memory leaks
*/
async shutdown() {
if (this.isShuttingDown) {
return;
}
this.isShuttingDown = true;
liveManagers.delete(this);
mcpLogger.info("[ExternalServerManager] Shutting down all servers...");
const shutdownPromises = Array.from(this.servers.keys()).map((serverId) => this.stopServer(serverId).catch((error) => {
mcpLogger.error(`[ExternalServerManager] Error shutting down ${serverId}:`, error);
}));
await Promise.all(shutdownPromises);
this.servers.clear();
// Clean up the tool discovery service to prevent memory leaks
// from accumulated event listeners
this.toolDiscovery.destroy();
// Remove all event listeners from this manager
this.removeAllListeners();
mcpLogger.info("[ExternalServerManager] All servers shut down and resources cleaned up");
}
/**
* Destroy the manager and all associated resources
* Alias for shutdown() to match the pattern used by other components
*/
async destroy() {
return this.shutdown();
}
/**
* Get manager statistics
*/
getStatistics() {
let connectedServers = 0;
let failedServers = 0;
let totalTools = 0;
let totalConnections = 0;
let totalErrors = 0;
for (const instance of Array.from(this.servers.values())) {
if (instance.status === "connected") {
connectedServers++;
}
else if (instance.status === "failed") {
failedServers++;
}
totalTools += instance.toolsMap.size;
totalConnections += instance.metrics.totalConnections;
totalErrors += instance.metrics.totalErrors;
}
return {
totalServers: this.servers.size,
connectedServers,
failedServers,
totalTools,
totalConnections,
totalErrors,
};
}
/**
* Discover tools from a server
*/
async discoverServerTools(serverId) {
const instance = this.servers.get(serverId);
if (!instance || !instance.client) {
throw new Error(`Server '${serverId}' not found or not connected`);
}
try {
mcpLogger.debug(`[ExternalServerManager] Discovering tools for server: ${serverId}`);
const discoveryResult = await this.toolDiscovery.discoverTools(serverId, instance.client, instance.config.timeout || this.config.defaultTimeout);
if (discoveryResult.success) {
instance.toolsMap.clear();
instance.toolsArray = undefined;
instance.tools = [];
const blockedTools = instance.blockedTools || [];
let blockedCount = 0;
for (const tool of discoveryResult.tools) {
// Check if tool is blocked
if (blockedTools.includes(tool.name)) {
mcpLogger.info(`[ExternalServerManager] Blocking tool '${tool.name}' from server '${serverId}' (configured in blockedTools)`);
blockedCount++;
continue; // Skip blocked tools
}
instance.toolsMap.set(tool.name, tool);
instance.tools.push({
name: tool.name,
description: tool.description,
inputSchema: tool.inputSchema,
});
}
mcpLogger.info(`[ExternalServerManager] Discovered ${discoveryResult.toolCount} tools for ${serverId} (${blockedCount} blocked, ${instance.toolsMap.size} available)`);
}
else {
mcpLogger.warn(`[ExternalServerManager] Tool discovery failed for ${serverId}: ${discoveryResult.error}`);
}
}
catch (error) {
mcpLogger.error(`[ExternalServerManager] Tool discovery error for ${serverId}:`, error);
}
}
/**
* Register server tools with main tool registry for unified access
* This enables external MCP tools to be accessed via the main toolRegistry.executeTool()
*/
async registerServerToolsWithMainRegistry(serverId) {
const instance = this.servers.get(serverId);
if (!instance) {
throw new Error(`Server '${serverId}' not found`);
}
try {
mcpLogger.debug(`[ExternalServerManager] Registering ${instance.toolsMap.size} tools with main registry for server: ${serverId}`);
const registrations = [];
for (const [toolName, tool] of instance.toolsMap.entries()) {
const toolId = `${serverId}.${toolName}`;
const toolInfo = {
name: toolName,
description: tool.description || toolName,
inputSchema: tool.inputSchema || {},
serverId: serverId,
category: detectCategory({ isExternal: true, serverId }),
};
try {
registrations.push(toolRegistry.registerTool(toolId, toolInfo, {
execute: async (params, _context) => {
// Execute tool via ExternalServerManager for proper lifecycle management
return await this.executeTool(serverId, toolName, params, {
timeout: instance.config.timeout || this.config.defaultTimeout,
});
},
}));
mcpLogger.debug(`[ExternalServerManager] Registered tool with main registry: ${toolId}`);
}
catch (registrationError) {
mcpLogger.warn(`[ExternalServerManager] Failed to register tool ${toolId} with main registry:`, registrationError);
}
}
const results = await Promise.allSettled(registrations);
const ok = results.filter((r) => r.status === "fulfilled").length;
const failed = results.length - ok;
mcpLogger.info(`[ExternalServerManager] Registered ${ok}/${results.length} tools with main registry for ${serverId}${failed ? ` (${failed} failed)` : ""}`);
}
catch (error) {
mcpLogger.error(`[ExternalServerManager] Failed to register tools with main registry for ${serverId}:`, error);
}
}
/**
* Unregister server tools from main tool registry
*/
unregisterServerToolsFromMainRegistry(serverId) {
const instance = this.servers.get(serverId);
if (!instance || !this.enableMainRegistryIntegration) {
return;
}
try {
mcpLogger.debug(`[ExternalServerManager] Unregistering tools from main registry for server: ${serverId}`);
for (const [toolName] of instance.toolsMap.entries()) {
const toolId = `${serverId}.${toolName}`;
try {
toolRegistry.removeTool(toolId);
mcpLogger.debug(`[ExternalServerManager] Unregistered tool from main registry: ${toolId}`);
}
catch (error) {
mcpLogger.debug(`[ExternalServerManager] Failed to unregister tool ${toolId}:`, error);
}
}
mcpLogger.debug(`[ExternalServerManager] Completed unregistering tools from main registry for ${serverId}`);
}
catch (error) {
mcpLogger.error(`[ExternalServerManager] Error unregistering tools from main registry for ${serverId}:`, error);
}
}
/**
* Execute a tool on a specific server
*/
async executeTool(serverId, toolName, parameters, options) {
const instance = this.servers.get(serverId);
if (!instance) {
throw new Error(`Server '${serverId}' not found`);
}
const client = instance.client;
if (!client) {
throw new Error(`Server '${serverId}' is not connected`);
}
if (instance.status !== "connected") {
throw new Error(`Server '${serverId}' is not in connected state: ${instance.status}`);
}
// Check if tool is blocked
const blockedTools = instance.blockedTools || [];
if (blockedTools.includes(toolName)) {
throw new Error(`Tool '${toolName}' is blocked on server '${serverId}' by configuration`);
}
const startTime = Date.now();
try {
// HITL Safety Check: Request confirmation if required
let finalParameters = parameters;
if (this.hitlManager && this.hitlManager.isEnabled()) {
const requiresConfirmation = this.hitlManager.requiresConfirmation(toolName, parameters);
if (requiresConfirmation) {
mcpLogger.info(`[ExternalServerManager] External tool '${toolName}' on server '${serverId}' requires HITL confirmation`);
try {
const confirmationResult = await this.hitlManager.requestConfirmation(toolName, parameters, {
serverId: serverId,
sessionId: `external-${serverId}-${Date.now()}`,
userId: undefined, // External tools don't have user context by default
});
if (!confirmationResult.approved) {
// User rejected the tool execution
throw new HITLUserRejectedError(`External tool execution rejected by user: ${confirmationResult.reason || "No reason provided"}`, toolName, confirmationResult.reason);
}
// User approved - use modified arguments if provided
if (confirmationResult.modifiedArguments !== undefined) {
finalParameters =
confirmationResult.modifiedArguments;
mcpLogger.info(`[ExternalServerManager] External tool '${toolName}' arguments modified by user`);
}
mcpLogger.info(`[ExternalServerManager] External tool '${toolName}' approved for execution (response time: ${confirmationResult.responseTime}ms)`);
}
catch (error) {
if (error instanceof HITLTimeoutError) {
// Timeout occurred - user didn't respond in time
mcpLogger.warn(`[ExternalServerManager] External tool '${toolName}' execution timed out waiting for user confirmation`);
throw error;
}
else if (error instanceof HITLUserRejectedError) {
// User explicitly rejected
mcpLogger.info(`[ExternalServerManager] External tool '${toolName}' execution rejected by user`);
throw error;
}
else {
// Other HITL error (configuration, system error, etc.)
mcpLogger.error(`[ExternalServerManager] HITL confirmation failed for external tool '${toolName}':`, error);
throw new Error(`HITL confirmation failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
}
}
}
else {
mcpLogger.debug(`[ExternalServerManager] External tool '${toolName}' does not require HITL confirmation`);
}
}
// Execute tool through discovery service (with potentially modified parameters)
this.trackToolCall(serverId, 1);
let result;
try {
result = await this.toolDiscovery.executeTool(toolName, serverId, client, finalParameters, {
timeout: options?.timeout ||
instance.config.timeout ||
this.config.defaultTimeout,
});
}
finally {
this.trackToolCall(serverId, -1);
}
const duration = Date.now() - startTime;
// Update metrics
instance.metrics.totalToolCalls++;
instance.metrics.lastResponseTime = duration;
// Update average response time
const totalTime = instance.metrics.averageResponseTime *
(instance.metrics.totalToolCalls - 1) +
duration;
instance.metrics.averageResponseTime =
totalTime / instance.metrics.totalToolCalls;
if (result.success) {
// A resolved `{ isError: true }` still comes back with success:true so
// the caller receives the MCP error payload as-is, but it is a failed
// call for telemetry: label mcp_tool_calls_total accordingly instead
// of counting every resolved error as a success.
const resolvedError = result.isErrorResult === true;
if (resolvedError) {
// Keep this instance's totalErrors (and the getStatistics() /
// getExternalMCPStatistics() it feeds) consistent with the breaker,
// TelemetryService and ToolDiscoveryService's failedCalls, all of
// which already count a resolved isError result as a failure.
instance.metrics.totalErrors++;
}
mcpLogger.debug(resolvedError
? `[ExternalServerManager] Tool returned a resolved MCP error: ${toolName} on ${serverId}`
: `[ExternalServerManager] Tool executed successfully: ${toolName} on ${serverId}`, {
duration,
});
try {
TelemetryService.getInstance()?.recordMCPToolCall(toolName, duration, !resolvedError);
}
catch {
/* telemetry should not break execution */
}
return result.data;
}
else {
throw new Error(result.error || "Tool execution failed");
}
}
catch (error) {
instance.metrics.totalErrors++;
try {
const errorDuration = Date.now() - startTime;
TelemetryService.getInstance()?.recordMCPToolCall(toolName, errorDuration, false);
}
catch {
/* telemetry should not break execution */
}
mcpLogger.error(`[ExternalServerManager] Tool execution failed: ${toolName} on ${serverId}`, error);
// The SDK rejects with its connection-closed error when the server is
// gone. Without this transition the instance stayed "connected" and
// every later call failed the same way with no restart ever scheduled.
if (isConnectionLostError(error)) {
this.handleConnectionLost(serverId, client, `Connection lost during tool call '${toolName}': ${error instanceof Error ? error.message : String(error)}`);
}
throw error;
}
}
/**
* Get all tools from all servers
*/
getAllTools() {
return this.toolDiscovery.getAllTools();
}
/**
* Get tools for a specific server
*/
getServerTools(serverId) {
return this.toolDiscovery.getServerTools(serverId);
}
/**
* Get tool discovery service
*/
getToolDiscovery() {
return this.toolDiscovery;
}
}