okta-mcp-server
Version:
Model Context Protocol (MCP) server for Okta API operations with support for bulk operations and caching
606 lines • 24.3 kB
JavaScript
/**
* Circuit Breaker Monitoring and Metrics System
*
* Provides comprehensive monitoring, metrics collection, and alerting
* for circuit breaker instances with Prometheus export support.
*/
import { EventEmitter } from 'events';
import { CircuitState, } from './types.js';
import { MetricType, } from './monitoring-types.js';
// Re-export for convenience
export { MetricType } from './monitoring-types.js';
/**
* Circuit Breaker Monitor implementation
*/
export class CircuitBreakerMonitor extends EventEmitter {
circuits = new Map();
metricsRetention;
metricsInterval;
cleanupTimer;
// Alert management
alertConditions = new Map();
activeAlerts = new Map();
alertCooldowns = new Map();
// Cache metrics
cacheMetrics = new Map();
constructor(options = {}) {
super();
this.metricsRetention = options.metricsRetention || 3600000; // 1 hour default
this.metricsInterval = options.metricsInterval || 1000; // 1 second default
// Start cleanup timer
this.startCleanupTimer();
}
/**
* Register a circuit breaker for monitoring
*/
register(name, circuitBreaker) {
if (this.circuits.has(name)) {
throw new Error(`Circuit breaker '${name}' is already registered`);
}
const metrics = this.createInitialMetrics(name);
const eventListeners = new Map();
// Subscribe to circuit breaker events
const emitter = circuitBreaker.getEventEmitter();
// State changes
const stateChangeHandler = (from, to, stats) => {
this.recordStateChange(name, from, to, stats);
};
emitter.on('state-change', stateChangeHandler);
eventListeners.set('state-change', stateChangeHandler);
// Success events
const successHandler = (result, latency) => {
this.recordSuccess(name, latency);
};
emitter.on('success', successHandler);
eventListeners.set('success', successHandler);
// Failure events
const failureHandler = (error, latency) => {
this.recordFailure(name, error, latency);
};
emitter.on('failure', failureHandler);
eventListeners.set('failure', failureHandler);
// Timeout events
const timeoutHandler = (latency) => {
this.recordTimeout(name, latency);
};
emitter.on('timeout', timeoutHandler);
eventListeners.set('timeout', timeoutHandler);
// Rejection events
const rejectHandler = (reason) => {
this.recordRejection(name, reason);
};
emitter.on('reject', rejectHandler);
eventListeners.set('reject', rejectHandler);
// Fallback events (cache hit)
const fallbackHandler = () => {
this.recordCacheHit(name);
};
emitter.on('fallback', fallbackHandler);
eventListeners.set('fallback', fallbackHandler);
this.circuits.set(name, {
breaker: circuitBreaker,
metrics,
eventListeners,
});
// Initialize cache metrics
this.cacheMetrics.set(name, {
hits: 0,
misses: 0,
lastReset: Date.now(),
});
}
/**
* Unregister a circuit breaker from monitoring
*/
unregister(name) {
const circuit = this.circuits.get(name);
if (!circuit)
return;
// Remove event listeners
const emitter = circuit.breaker.getEventEmitter();
circuit.eventListeners.forEach((listener, event) => {
emitter.off(event, listener);
});
this.circuits.delete(name);
this.cacheMetrics.delete(name);
this.alertConditions.delete(name);
this.activeAlerts.delete(name);
}
/**
* Add alert condition for a circuit
*/
addAlertCondition(circuitName, condition) {
const conditions = this.alertConditions.get(circuitName) || [];
conditions.push(condition);
this.alertConditions.set(circuitName, conditions);
}
/**
* Get metrics for a specific circuit
*/
getMetrics(circuitName) {
return this.circuits.get(circuitName)?.metrics;
}
/**
* Get all metrics
*/
getAllMetrics() {
const allMetrics = new Map();
this.circuits.forEach((circuit, name) => {
allMetrics.set(name, circuit.metrics);
});
return allMetrics;
}
/**
* Export metrics in Prometheus format
*/
exportPrometheus() {
const lines = [];
this.circuits.forEach((circuit, name) => {
const metrics = circuit.metrics;
const labels = `circuit="${name}"`;
// State metric (0=closed, 1=half-open, 2=open)
const stateValue = metrics.state === CircuitState.CLOSED
? 0
: metrics.state === CircuitState.HALF_OPEN
? 1
: 2;
lines.push(`# HELP circuit_breaker_state Current state of the circuit breaker`);
lines.push(`# TYPE circuit_breaker_state gauge`);
lines.push(`circuit_breaker_state{${labels}} ${stateValue}`);
// Request metrics
const latestRequests = this.getLatestValue(metrics.requests.total);
lines.push(`# HELP circuit_breaker_requests_total Total number of requests`);
lines.push(`# TYPE circuit_breaker_requests_total counter`);
lines.push(`circuit_breaker_requests_total{${labels}} ${latestRequests}`);
// Success/failure metrics
const latestSuccess = this.getLatestValue(metrics.requests.success);
const latestFailure = this.getLatestValue(metrics.requests.failure);
lines.push(`# HELP circuit_breaker_requests_success_total Total successful requests`);
lines.push(`# TYPE circuit_breaker_requests_success_total counter`);
lines.push(`circuit_breaker_requests_success_total{${labels}} ${latestSuccess}`);
lines.push(`# HELP circuit_breaker_requests_failure_total Total failed requests`);
lines.push(`# TYPE circuit_breaker_requests_failure_total counter`);
lines.push(`circuit_breaker_requests_failure_total{${labels}} ${latestFailure}`);
// Latency metrics
lines.push(`# HELP circuit_breaker_request_duration_seconds Request latency in seconds`);
lines.push(`# TYPE circuit_breaker_request_duration_seconds summary`);
lines.push(`circuit_breaker_request_duration_seconds{${labels},quantile="0.5"} ${metrics.latency.percentiles.p50 / 1000}`);
lines.push(`circuit_breaker_request_duration_seconds{${labels},quantile="0.9"} ${metrics.latency.percentiles.p90 / 1000}`);
lines.push(`circuit_breaker_request_duration_seconds{${labels},quantile="0.95"} ${metrics.latency.percentiles.p95 / 1000}`);
lines.push(`circuit_breaker_request_duration_seconds{${labels},quantile="0.99"} ${metrics.latency.percentiles.p99 / 1000}`);
// Cache metrics
const cacheStats = this.cacheMetrics.get(name);
if (cacheStats) {
lines.push(`# HELP circuit_breaker_cache_hits_total Total cache hits`);
lines.push(`# TYPE circuit_breaker_cache_hits_total counter`);
lines.push(`circuit_breaker_cache_hits_total{${labels}} ${cacheStats.hits}`);
lines.push(`# HELP circuit_breaker_cache_misses_total Total cache misses`);
lines.push(`# TYPE circuit_breaker_cache_misses_total counter`);
lines.push(`circuit_breaker_cache_misses_total{${labels}} ${cacheStats.misses}`);
const hitRate = cacheStats.hits + cacheStats.misses > 0
? cacheStats.hits / (cacheStats.hits + cacheStats.misses)
: 0;
lines.push(`# HELP circuit_breaker_cache_hit_rate Cache hit rate`);
lines.push(`# TYPE circuit_breaker_cache_hit_rate gauge`);
lines.push(`circuit_breaker_cache_hit_rate{${labels}} ${hitRate}`);
}
// Rate limit metrics
const latestRateLimits = this.getLatestValue(metrics.rateLimits.occurrences);
lines.push(`# HELP circuit_breaker_rate_limit_occurrences_total Rate limit occurrences`);
lines.push(`# TYPE circuit_breaker_rate_limit_occurrences_total counter`);
lines.push(`circuit_breaker_rate_limit_occurrences_total{${labels}} ${latestRateLimits}`);
});
return lines.join('\n');
}
/**
* Get dashboard data for real-time monitoring
*/
getDashboard() {
const circuits = new Map();
let totalRequests = 0;
let totalSuccess = 0;
let activeAlerts = 0;
this.circuits.forEach((circuit, name) => {
const stats = circuit.breaker.getStats();
const alerts = this.activeAlerts.get(name) || [];
activeAlerts += alerts.length;
totalRequests += stats.totalRequests;
totalSuccess += stats.successes;
const health = this.calculateHealth(stats, alerts);
circuits.set(name, {
current: stats,
metrics: circuit.metrics,
alerts,
health,
});
});
return {
circuits,
systemHealth: {
totalRequests,
avgSuccessRate: totalRequests > 0 ? (totalSuccess / totalRequests) * 100 : 100,
activeAlerts,
lastUpdated: Date.now(),
},
};
}
/**
* Get real-time stream of metrics
*/
async *streamMetrics(interval = 1000) {
while (true) {
yield this.getDashboard();
await new Promise((resolve) => setTimeout(resolve, interval));
}
}
/**
* Record cache hit
*/
recordCacheHit(circuitName) {
const cache = this.cacheMetrics.get(circuitName);
if (cache) {
cache.hits++;
}
const circuit = this.circuits.get(circuitName);
if (circuit) {
this.addMetricPoint(circuit.metrics.requests.cacheHit, 1);
}
}
/**
* Record cache miss
*/
recordCacheMiss(circuitName) {
const cache = this.cacheMetrics.get(circuitName);
if (cache) {
cache.misses++;
}
const circuit = this.circuits.get(circuitName);
if (circuit) {
this.addMetricPoint(circuit.metrics.requests.cacheMiss, 1);
}
}
/**
* Destroy the monitor and clean up resources
*/
destroy() {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
}
// Unregister all circuits
Array.from(this.circuits.keys()).forEach((name) => {
this.unregister(name);
});
this.removeAllListeners();
}
// Private methods
createInitialMetrics(name) {
return {
circuitName: name,
state: CircuitState.CLOSED,
stateChanges: this.createTimeSeries('state_changes', MetricType.GAUGE, 'Circuit state changes'),
requests: {
total: this.createTimeSeries('requests_total', MetricType.COUNTER, 'Total requests'),
success: this.createTimeSeries('requests_success', MetricType.COUNTER, 'Successful requests'),
failure: this.createTimeSeries('requests_failure', MetricType.COUNTER, 'Failed requests'),
timeout: this.createTimeSeries('requests_timeout', MetricType.COUNTER, 'Timed out requests'),
rejected: this.createTimeSeries('requests_rejected', MetricType.COUNTER, 'Rejected requests'),
cacheHit: this.createTimeSeries('cache_hits', MetricType.COUNTER, 'Cache hits'),
cacheMiss: this.createTimeSeries('cache_misses', MetricType.COUNTER, 'Cache misses'),
},
latency: {
histogram: this.createTimeSeries('latency_ms', MetricType.HISTOGRAM, 'Request latency in milliseconds'),
percentiles: {
p50: 0,
p90: 0,
p95: 0,
p99: 0,
},
},
rates: {
successRate: this.createTimeSeries('success_rate', MetricType.GAUGE, 'Success rate percentage'),
failureRate: this.createTimeSeries('failure_rate', MetricType.GAUGE, 'Failure rate percentage'),
errorRate: this.createTimeSeries('error_rate', MetricType.GAUGE, 'Error rate percentage'),
},
rateLimits: {
occurrences: this.createTimeSeries('rate_limit_occurrences', MetricType.COUNTER, 'Rate limit occurrences'),
remaining: this.createTimeSeries('rate_limit_remaining', MetricType.GAUGE, 'Rate limit remaining'),
},
};
}
createTimeSeries(name, type, help) {
return {
name,
type,
help,
points: [],
};
}
recordStateChange(circuitName, from, to, stats) {
const circuit = this.circuits.get(circuitName);
if (!circuit)
return;
circuit.metrics.state = to;
const stateValue = to === CircuitState.CLOSED ? 0 : to === CircuitState.HALF_OPEN ? 1 : 2;
this.addMetricPoint(circuit.metrics.stateChanges, stateValue);
// Check alert conditions
this.checkAlerts(circuitName, stats);
// Emit monitoring event
this.emit('stateChange', {
circuit: circuitName,
from,
to,
timestamp: Date.now(),
});
}
recordSuccess(circuitName, latency) {
const circuit = this.circuits.get(circuitName);
if (!circuit)
return;
// Update counters
this.incrementCounter(circuit.metrics.requests.total);
this.incrementCounter(circuit.metrics.requests.success);
// Update latency
this.addMetricPoint(circuit.metrics.latency.histogram, latency);
this.updateLatencyPercentiles(circuit.metrics);
// Update rates
this.updateRates(circuit.metrics);
// Cache miss (successful request means cache wasn't used)
this.recordCacheMiss(circuitName);
}
recordFailure(circuitName, error, latency) {
const circuit = this.circuits.get(circuitName);
if (!circuit)
return;
// Check if it's a rate limit error
const isRateLimit = error.status === 429;
if (isRateLimit) {
this.incrementCounter(circuit.metrics.rateLimits.occurrences);
// Extract rate limit info if available
const headers = error.headers;
if (headers?.['x-rate-limit-remaining']) {
this.addMetricPoint(circuit.metrics.rateLimits.remaining, parseInt(headers['x-rate-limit-remaining']));
}
}
// Update counters
this.incrementCounter(circuit.metrics.requests.total);
this.incrementCounter(circuit.metrics.requests.failure);
// Update latency
this.addMetricPoint(circuit.metrics.latency.histogram, latency);
this.updateLatencyPercentiles(circuit.metrics);
// Update rates
this.updateRates(circuit.metrics);
}
recordTimeout(circuitName, latency) {
const circuit = this.circuits.get(circuitName);
if (!circuit)
return;
this.incrementCounter(circuit.metrics.requests.total);
this.incrementCounter(circuit.metrics.requests.timeout);
this.addMetricPoint(circuit.metrics.latency.histogram, latency);
this.updateLatencyPercentiles(circuit.metrics);
this.updateRates(circuit.metrics);
}
recordRejection(circuitName, reason) {
const circuit = this.circuits.get(circuitName);
if (!circuit)
return;
this.incrementCounter(circuit.metrics.requests.total);
this.incrementCounter(circuit.metrics.requests.rejected);
this.updateRates(circuit.metrics);
}
incrementCounter(timeSeries) {
const lastValue = this.getLatestValue(timeSeries);
this.addMetricPoint(timeSeries, lastValue + 1);
}
addMetricPoint(timeSeries, value) {
timeSeries.points.push({
timestamp: Date.now(),
value,
});
}
getLatestValue(timeSeries) {
if (timeSeries.points.length === 0)
return 0;
return timeSeries.points[timeSeries.points.length - 1].value;
}
updateRates(metrics) {
const total = this.getLatestValue(metrics.requests.total);
if (total === 0)
return;
const success = this.getLatestValue(metrics.requests.success);
const failure = this.getLatestValue(metrics.requests.failure);
const timeout = this.getLatestValue(metrics.requests.timeout);
this.addMetricPoint(metrics.rates.successRate, (success / total) * 100);
this.addMetricPoint(metrics.rates.failureRate, (failure / total) * 100);
this.addMetricPoint(metrics.rates.errorRate, ((failure + timeout) / total) * 100);
}
updateLatencyPercentiles(metrics) {
const latencies = metrics.latency.histogram.points
.slice(-1000) // Last 1000 samples
.map((p) => p.value)
.sort((a, b) => a - b);
if (latencies.length === 0)
return;
metrics.latency.percentiles = {
p50: this.percentile(latencies, 0.5),
p90: this.percentile(latencies, 0.9),
p95: this.percentile(latencies, 0.95),
p99: this.percentile(latencies, 0.99),
};
}
percentile(sorted, p) {
const index = Math.ceil(sorted.length * p) - 1;
return sorted[Math.max(0, Math.min(index, sorted.length - 1))];
}
checkAlerts(circuitName, stats) {
const conditions = this.alertConditions.get(circuitName);
if (!conditions)
return;
const circuit = this.circuits.get(circuitName);
if (!circuit)
return;
conditions.forEach((condition) => {
const value = this.getMetricValue(circuit.metrics, condition.metric);
if (value === undefined)
return;
const triggered = this.evaluateCondition(value, condition);
if (triggered) {
this.triggerAlert(circuitName, condition, value);
}
});
}
getMetricValue(metrics, metricPath) {
// Simple metric path evaluation (e.g., "rates.errorRate")
const parts = metricPath.split('.');
let current = metrics;
for (const part of parts) {
if (current[part] === undefined)
return undefined;
current = current[part];
}
if (current instanceof Object && 'points' in current) {
return this.getLatestValue(current);
}
return typeof current === 'number' ? current : undefined;
}
evaluateCondition(value, condition) {
switch (condition.operator) {
case 'gt':
return value > condition.threshold;
case 'lt':
return value < condition.threshold;
case 'gte':
return value >= condition.threshold;
case 'lte':
return value <= condition.threshold;
case 'eq':
return value === condition.threshold;
default:
return false;
}
}
triggerAlert(circuitName, condition, value) {
// Check cooldown
const cooldownKey = `${circuitName}:${condition.name}`;
const lastAlert = this.alertCooldowns.get(cooldownKey) || 0;
if (Date.now() - lastAlert < condition.cooldown) {
return;
}
const alert = {
id: `${circuitName}:${condition.name}:${Date.now()}`,
condition,
timestamp: Date.now(),
value,
message: `Alert: ${condition.name} - ${condition.metric} is ${value} (threshold: ${condition.operator} ${condition.threshold})`,
severity: this.getAlertSeverity(condition, value),
};
// Store alert
const alerts = this.activeAlerts.get(circuitName) || [];
alerts.push(alert);
this.activeAlerts.set(circuitName, alerts);
// Update cooldown
this.alertCooldowns.set(cooldownKey, Date.now());
// Emit alert event
this.emit('alert', alert);
}
getAlertSeverity(condition, value) {
// Simple severity calculation based on how far the value exceeds the threshold
const ratio = value / condition.threshold;
if (ratio > 2)
return 'critical';
if (ratio > 1.5)
return 'error';
if (ratio > 1.2)
return 'warning';
return 'info';
}
calculateHealth(stats, alerts) {
// Circuit open = unhealthy
if (stats.state === CircuitState.OPEN)
return 'unhealthy';
// Critical alerts = unhealthy
if (alerts.some((a) => a.severity === 'critical'))
return 'unhealthy';
// High failure rate or multiple alerts = degraded
if (stats.failureRate > 50 || alerts.length > 2)
return 'degraded';
return 'healthy';
}
startCleanupTimer() {
this.cleanupTimer = setInterval(() => {
const cutoff = Date.now() - this.metricsRetention;
this.circuits.forEach((circuit) => {
// Clean up old metric points
this.cleanupTimeSeries(circuit.metrics.stateChanges, cutoff);
Object.values(circuit.metrics.requests).forEach((ts) => {
this.cleanupTimeSeries(ts, cutoff);
});
this.cleanupTimeSeries(circuit.metrics.latency.histogram, cutoff);
Object.values(circuit.metrics.rates).forEach((ts) => {
this.cleanupTimeSeries(ts, cutoff);
});
Object.values(circuit.metrics.rateLimits).forEach((ts) => {
this.cleanupTimeSeries(ts, cutoff);
});
});
// Clean up old alerts
this.activeAlerts.forEach((alerts, circuitName) => {
const filtered = alerts.filter((a) => a.timestamp > cutoff);
this.activeAlerts.set(circuitName, filtered);
});
}, this.metricsInterval);
}
cleanupTimeSeries(timeSeries, cutoff) {
timeSeries.points = timeSeries.points.filter((p) => p.timestamp > cutoff);
}
}
/**
* Create a new circuit breaker monitor instance
*/
export function createCircuitBreakerMonitor(options) {
return new CircuitBreakerMonitor(options);
}
/**
* Express/HTTP endpoint for Prometheus metrics
*/
export function createMetricsEndpoint(monitor) {
return (req, res) => {
res.set('Content-Type', 'text/plain; version=0.0.4');
res.send(monitor.exportPrometheus());
};
}
/**
* WebSocket handler for real-time dashboard
*/
export function createDashboardWebSocket(monitor) {
return async (ws) => {
// Send initial dashboard data
ws.send(JSON.stringify({
type: 'dashboard',
data: monitor.getDashboard(),
}));
// Subscribe to alerts
const alertHandler = (alert) => {
ws.send(JSON.stringify({
type: 'alert',
data: alert,
}));
};
monitor.on('alert', alertHandler);
// Send periodic updates
const interval = setInterval(() => {
if (ws.readyState === ws.OPEN) {
ws.send(JSON.stringify({
type: 'dashboard',
data: monitor.getDashboard(),
}));
}
}, 1000);
// Cleanup on disconnect
ws.on('close', () => {
clearInterval(interval);
monitor.off('alert', alertHandler);
});
};
}
//# sourceMappingURL=monitoring.js.map