@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
483 lines (482 loc) • 19 kB
JavaScript
/**
* MCP Circuit Breaker
* Implements circuit breaker pattern for external MCP operations
* Provides fault tolerance and prevents cascading failures
*/
import { EventEmitter } from "events";
import { trace } from "@opentelemetry/api";
import { mcpLogger } from "../utils/logger.js";
import { CircuitBreakerOpenError } from "../types/index.js";
// Re-export CircuitBreakerOpenError from shared types to preserve public API
export { CircuitBreakerOpenError } from "../types/index.js";
/**
* Default operation timeout for circuit breaker protected operations.
* Configurable via MCP_OPERATION_TIMEOUT env var (in milliseconds).
* Increased from 30s to 60s to accommodate MCP server startup latency,
* especially when multiple servers start concurrently.
*/
const DEFAULT_OPERATION_TIMEOUT = Math.max(10000, Number(process.env.MCP_OPERATION_TIMEOUT) || 60000);
/**
* MCPCircuitBreaker
* Implements circuit breaker pattern for fault tolerance
*/
export class MCPCircuitBreaker extends EventEmitter {
name;
state = "closed";
config;
callHistory = [];
lastFailureTime = 0;
halfOpenCalls = 0;
lastStateChange = new Date();
// Store the cleanup timer reference for proper cleanup
cleanupTimer;
constructor(name, config = {}) {
super();
this.name = name;
// Set default configuration
this.config = {
failureThreshold: config.failureThreshold ?? 5,
resetTimeout: config.resetTimeout ?? 60000,
halfOpenMaxCalls: config.halfOpenMaxCalls ?? 3,
operationTimeout: config.operationTimeout ?? DEFAULT_OPERATION_TIMEOUT,
minimumCallsBeforeCalculation: config.minimumCallsBeforeCalculation ?? 10,
statisticsWindowSize: config.statisticsWindowSize ?? 300000, // 5 minutes
};
// Clean up old call records periodically - now storing the timer reference.
// Unref'd so a live breaker (the manager's Map is a module-level
// singleton, never torn down by a script that merely imports the SDK)
// doesn't block process exit on its own — destroy()/destroyAll() still
// clearInterval() it for callers that do explicit lifecycle management.
this.cleanupTimer = setInterval(() => this.cleanupCallHistory(), 60000);
if (this.cleanupTimer.unref) {
this.cleanupTimer.unref();
}
}
/**
* Execute an operation with circuit breaker protection.
*
* `operation` is handed a `recordResolvedFailure` callback so it can flag a
* *resolved* result as a logical failure without throwing. This matters for
* MCP tool calls: the client does not throw on a protocol error, it
* resolves with `{ isError: true, ... }`. Before this callback existed,
* such a call always fell through to the success path below — the breaker
* could never open for a tool that only ever "fails" by resolving an error
* result, and callers still need that resolved value returned as-is (not
* replaced by a thrown transport error). Call `recordResolvedFailure()`
* from inside `operation` when that resolved value represents a failure;
* the call is still counted as a normal, non-throwing success from the
* `Promise.race` below, so this callback is what tells `execute()`
* otherwise. Callers that never call it see no behavior change.
*/
async execute(operation) {
const startTime = Date.now();
try {
// Check if circuit is open
if (this.state === "open") {
const retryAfterMs = this.config.resetTimeout - (Date.now() - this.lastFailureTime);
if (retryAfterMs > 0) {
throw new CircuitBreakerOpenError({
breakerName: this.name,
retryAfter: new Date(this.lastFailureTime + this.config.resetTimeout),
retryAfterMs,
breakerState: "open",
failureCount: this.getStats().failedCalls,
});
}
// Transition to half-open
this.changeState("half-open", "Reset timeout reached");
}
// Check half-open call limit
if (this.state === "half-open" &&
this.halfOpenCalls >= this.config.halfOpenMaxCalls) {
// Half-open call limit exceeded — revert to open with fresh cooldown
this.lastFailureTime = Date.now();
this.changeState("open", "Half-open call limit reached, reverting to open");
throw new CircuitBreakerOpenError({
breakerName: this.name,
retryAfter: new Date(this.lastFailureTime + this.config.resetTimeout),
retryAfterMs: this.config.resetTimeout,
breakerState: "open",
failureCount: this.getStats().failedCalls,
});
}
// NLK-GAP-009: Record half-open test event when executing in half-open state
if (this.state === "half-open") {
const activeSpan = trace.getActiveSpan();
if (activeSpan) {
activeSpan.addEvent("circuit.half_open_test", {
"circuit.name": this.name,
"circuit.half_open_call": this.halfOpenCalls + 1,
"circuit.half_open_max_calls": this.config.halfOpenMaxCalls,
});
}
}
// resolvedFailureReason is set (via recordResolvedFailure, passed into
// operation() below) when operation() resolves normally but represents
// a logical failure — e.g. an MCP tool call that resolved
// { isError: true } instead of throwing. It is call-scoped (a local,
// not instance state), so concurrent execute() calls on the same
// breaker (the same tool invoked in parallel) can't cross-contaminate
// each other's outcome.
let resolvedFailureReason;
const recordResolvedFailure = (reason) => {
resolvedFailureReason = reason || "Operation resolved a failure result";
};
// Execute operation with timeout
const result = await this.raceWithTimeout(operation(recordResolvedFailure), this.config.operationTimeout);
if (resolvedFailureReason !== undefined) {
// Same bookkeeping as a thrown error below, but the resolved value
// is still returned to the caller unchanged — no transport error is
// synthesized for a result the protocol itself resolved successfully.
this.recordFailureOutcome(resolvedFailureReason, Date.now() - startTime);
return result;
}
// Record successful call
this.recordCall(true, Date.now() - startTime);
// Handle half-open success
if (this.state === "half-open") {
this.halfOpenCalls++;
// If enough successful calls in half-open, close the circuit
if (this.halfOpenCalls >= this.config.halfOpenMaxCalls) {
this.changeState("closed", "Half-open test successful");
}
}
return result;
}
catch (error) {
// Record failed call
const duration = Date.now() - startTime;
const errorMessage = error instanceof Error ? error.message : String(error);
this.recordFailureOutcome(errorMessage, duration);
throw error;
}
}
/**
* Shared bookkeeping for a failed outcome — a thrown error, or a resolved
* result the caller flagged via `recordResolvedFailure`. Records the call
* as a failure, emits `callFailure`, and runs the state transitions that
* used to live inline in the `catch` block: half-open failure immediately
* re-opens the circuit; closed checks the failure threshold.
*/
recordFailureOutcome(reason, duration) {
this.recordCall(false, duration);
this.emit("callFailure", {
error: reason,
duration,
timestamp: new Date(),
});
if (this.state === "half-open") {
// Failure in half-open immediately opens circuit
this.changeState("open", `Half-open test failed: ${reason}`);
}
else if (this.state === "closed") {
// Check if we should open the circuit
this.checkFailureThreshold();
}
}
/**
* Record a call in the history
*/
recordCall(success, duration) {
const now = Date.now();
this.callHistory.push({
timestamp: now,
success,
duration,
});
// Emit success event
if (success) {
this.emit("callSuccess", {
duration,
timestamp: new Date(),
});
}
// Update failure time
if (!success) {
this.lastFailureTime = now;
}
}
/**
* Check if failure threshold is exceeded
*/
checkFailureThreshold() {
const windowStart = Date.now() - this.config.statisticsWindowSize;
const windowCalls = this.callHistory.filter((call) => call.timestamp >= windowStart);
// Need minimum calls before calculating failure rate
if (windowCalls.length < this.config.minimumCallsBeforeCalculation) {
return;
}
const failedCalls = windowCalls.filter((call) => !call.success).length;
const failureRate = failedCalls / windowCalls.length;
mcpLogger.debug(`[CircuitBreaker:${this.name}] Failure rate: ${(failureRate * 100).toFixed(1)}% (${failedCalls}/${windowCalls.length})`);
// Open circuit if failure rate exceeds threshold
if (failedCalls >= this.config.failureThreshold) {
this.changeState("open", `Failure threshold exceeded: ${failedCalls} failures`);
this.emit("circuitOpen", {
failureRate,
totalCalls: windowCalls.length,
timestamp: new Date(),
});
}
}
/**
* Change circuit breaker state
*/
changeState(newState, reason) {
const oldState = this.state;
this.state = newState;
this.lastStateChange = new Date();
// NLK-GAP-009: Record state transition on active OTel span
const activeSpan = trace.getActiveSpan();
if (activeSpan) {
activeSpan.addEvent("circuit.state_change", {
"circuit.name": this.name,
"circuit.from_state": oldState,
"circuit.to_state": newState,
"circuit.reason": reason.slice(0, 128),
"circuit.failure_count": this.callHistory.filter((c) => !c.success)
.length,
});
}
// Reset counters based on state
if (newState === "half-open") {
this.halfOpenCalls = 0;
this.emit("circuitHalfOpen", {
timestamp: new Date(),
});
}
else if (newState === "closed") {
this.halfOpenCalls = 0;
this.emit("circuitClosed", {
timestamp: new Date(),
});
}
mcpLogger.info(`[CircuitBreaker:${this.name}] State changed: ${oldState} -> ${newState} (${reason})`);
// Emit state change event
this.emit("stateChange", {
oldState,
newState,
reason,
timestamp: new Date(),
});
}
/**
* Race an operation against a timeout, clearing the timer on either
* outcome. A bare `Promise.race([operation, timeoutPromise])` leaves the
* timeout's `setTimeout` handle ref'd and pending until it fires even
* after `operation` already won the race — on the (common) success path
* that stray timer holds the event loop open for the full timeout
* duration for no reason.
*/
async raceWithTimeout(operation, timeout) {
let timeoutId;
const timeoutPromise = new Promise((_, reject) => {
timeoutId = setTimeout(() => {
reject(new Error(`Operation timed out after ${timeout}ms`));
}, timeout);
});
try {
return await Promise.race([operation, timeoutPromise]);
}
finally {
clearTimeout(timeoutId);
}
}
/**
* Clean up old call records
*/
cleanupCallHistory() {
const cutoffTime = Date.now() - this.config.statisticsWindowSize;
const originalLength = this.callHistory.length;
this.callHistory = this.callHistory.filter((call) => call.timestamp >= cutoffTime);
const removed = originalLength - this.callHistory.length;
if (removed > 0) {
mcpLogger.debug(`[CircuitBreaker:${this.name}] Cleaned up ${removed} old call records`);
}
}
/**
* Get current statistics
*/
getStats() {
const windowStart = Date.now() - this.config.statisticsWindowSize;
const windowCalls = this.callHistory.filter((call) => call.timestamp >= windowStart);
const successfulCalls = windowCalls.filter((call) => call.success).length;
const failedCalls = windowCalls.length - successfulCalls;
const failureRate = windowCalls.length > 0 ? failedCalls / windowCalls.length : 0;
return {
state: this.state,
totalCalls: this.callHistory.length,
successfulCalls: this.callHistory.filter((call) => call.success).length,
failedCalls: this.callHistory.filter((call) => !call.success).length,
failureRate,
windowCalls: windowCalls.length,
lastStateChange: this.lastStateChange,
nextRetryTime: this.state === "open"
? new Date(this.lastFailureTime + this.config.resetTimeout)
: undefined,
halfOpenCalls: this.halfOpenCalls,
};
}
/**
* Manually reset the circuit breaker
*/
reset() {
this.changeState("closed", "Manual reset");
this.callHistory = [];
this.lastFailureTime = 0;
this.halfOpenCalls = 0;
}
/**
* Force open the circuit breaker
*/
forceOpen(reason = "Manual force open") {
this.changeState("open", reason);
this.lastFailureTime = Date.now();
}
/**
* Get circuit breaker name
*/
getName() {
return this.name;
}
/**
* Check if circuit is open
*/
isOpen() {
return this.state === "open";
}
/**
* Check if circuit is closed
*/
isClosed() {
return this.state === "closed";
}
/**
* Check if circuit is half-open
*/
isHalfOpen() {
return this.state === "half-open";
}
/**
* Destroy the circuit breaker and clean up resources
* This method should be called when the circuit breaker is no longer needed
* to prevent memory leaks from the cleanup timer
*/
destroy() {
// Clear the interval timer to prevent memory leaks
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
this.cleanupTimer = undefined;
mcpLogger.debug(`[CircuitBreaker:${this.name}] Cleanup timer cleared`);
}
// Clear any remaining event listeners
this.removeAllListeners();
// Clear call history to free memory
this.callHistory = [];
mcpLogger.debug(`[CircuitBreaker:${this.name}] Destroyed and cleaned up`);
}
}
/**
* Circuit breaker manager for multiple circuit breakers
*/
export class CircuitBreakerManager {
breakers = new Map();
/**
* Get or create a circuit breaker
*/
getBreaker(name, config) {
if (!this.breakers.has(name)) {
const breaker = new MCPCircuitBreaker(name, config);
this.breakers.set(name, breaker);
mcpLogger.debug(`[CircuitBreakerManager] Created circuit breaker: ${name}`);
}
const breaker = this.breakers.get(name);
if (!breaker) {
throw new Error(`Circuit breaker ${name} not found after creation`);
}
return breaker;
}
/**
* Remove a circuit breaker and clean up its resources
*/
removeBreaker(name) {
const breaker = this.breakers.get(name);
if (breaker) {
// Destroy the breaker to clean up its timer and resources
breaker.destroy();
this.breakers.delete(name);
mcpLogger.debug(`[CircuitBreakerManager] Removed and cleaned up circuit breaker: ${name}`);
return true;
}
return false;
}
/**
* Get all circuit breaker names
*/
getBreakerNames() {
return Array.from(this.breakers.keys());
}
/**
* Get statistics for all circuit breakers
*/
getAllStats() {
const stats = {};
for (const [name, breaker] of this.breakers) {
stats[name] = breaker.getStats();
}
return stats;
}
/**
* Reset all circuit breakers
*/
resetAll() {
for (const breaker of this.breakers.values()) {
breaker.reset();
}
mcpLogger.info("[CircuitBreakerManager] Reset all circuit breakers");
}
/**
* Get health summary
*/
getHealthSummary() {
let closedBreakers = 0;
let openBreakers = 0;
let halfOpenBreakers = 0;
const unhealthyBreakers = [];
for (const [name, breaker] of this.breakers) {
const stats = breaker.getStats();
switch (stats.state) {
case "closed":
closedBreakers++;
break;
case "open":
openBreakers++;
unhealthyBreakers.push(name);
break;
case "half-open":
halfOpenBreakers++;
break;
}
}
return {
totalBreakers: this.breakers.size,
closedBreakers,
openBreakers,
halfOpenBreakers,
unhealthyBreakers,
};
}
/**
* Destroy all circuit breakers and clean up their resources
* This should be called during application shutdown to prevent memory leaks
*/
destroyAll() {
for (const breaker of this.breakers.values()) {
breaker.destroy();
}
this.breakers.clear();
mcpLogger.info("[CircuitBreakerManager] Destroyed all circuit breakers");
}
}
// Global circuit breaker manager instance
export const globalCircuitBreakerManager = new CircuitBreakerManager();