@re-shell/cli
Version:
Full-stack development platform uniting microservices and microfrontends. Build complete applications with .NET (ASP.NET Core Web API, Minimal API), Java (Spring Boot, Quarkus, Micronaut, Vert.x), Rust (Actix-Web, Warp, Rocket, Axum), Python (FastAPI, Dja
450 lines (449 loc) • 15.6 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RetryManager = exports.CircuitBreaker = exports.RetryExecutor = exports.CircuitState = void 0;
exports.createRetryManager = createRetryManager;
exports.getGlobalRetryManager = getGlobalRetryManager;
exports.setGlobalRetryManager = setGlobalRetryManager;
const events_1 = require("events");
var CircuitState;
(function (CircuitState) {
CircuitState["CLOSED"] = "closed";
CircuitState["OPEN"] = "open";
CircuitState["HALF_OPEN"] = "half-open";
})(CircuitState || (exports.CircuitState = CircuitState = {}));
class RetryExecutor extends events_1.EventEmitter {
constructor(options = {}) {
super();
this.options = options;
this.defaultOptions = {
maxAttempts: 3,
baseDelay: 1000,
maxDelay: 30000,
backoffMultiplier: 2,
jitter: true,
timeout: 60000
};
this.options = { ...this.defaultOptions, ...options };
}
async execute(operation, customOptions) {
const opts = { ...this.defaultOptions, ...this.options, ...customOptions };
const startTime = Date.now();
let lastError;
let lastAttemptDuration = 0;
for (let attempt = 1; attempt <= opts.maxAttempts; attempt++) {
const attemptStartTime = Date.now();
try {
this.emit('attempt:start', { attempt, operation: operation.name });
let result;
if (opts.timeout) {
result = await this.withTimeout(operation(), opts.timeout);
}
else {
result = await operation();
}
lastAttemptDuration = Date.now() - attemptStartTime;
const totalDuration = Date.now() - startTime;
this.emit('attempt:success', {
attempt,
duration: lastAttemptDuration,
totalDuration
});
return {
success: true,
result,
attempts: attempt,
totalDuration,
lastAttemptDuration
};
}
catch (error) {
lastError = error;
lastAttemptDuration = Date.now() - attemptStartTime;
this.emit('attempt:failure', {
attempt,
error,
duration: lastAttemptDuration
});
// Check if we should retry
if (attempt === opts.maxAttempts ||
(opts.retryCondition && !opts.retryCondition(error))) {
break;
}
// Calculate delay
const delay = this.calculateDelay(attempt, opts);
if (opts.onRetry) {
opts.onRetry(attempt, error, delay);
}
this.emit('retry:delay', { attempt, delay, error });
// Wait before retry
await this.sleep(delay);
}
}
const totalDuration = Date.now() - startTime;
this.emit('retry:exhausted', {
attempts: opts.maxAttempts,
error: lastError,
totalDuration
});
return {
success: false,
error: lastError,
attempts: opts.maxAttempts,
totalDuration,
lastAttemptDuration
};
}
calculateDelay(attempt, options) {
const exponentialDelay = options.baseDelay * Math.pow(options.backoffMultiplier, attempt - 1);
let delay = Math.min(exponentialDelay, options.maxDelay);
// Add jitter to prevent thundering herd
if (options.jitter) {
delay = delay * (0.5 + Math.random() * 0.5);
}
return Math.floor(delay);
}
async withTimeout(promise, timeoutMs) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
reject(new Error(`Operation timed out after ${timeoutMs}ms`));
}, timeoutMs);
promise
.then(result => {
clearTimeout(timer);
resolve(result);
})
.catch(error => {
clearTimeout(timer);
reject(error);
});
});
}
sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
// Static convenience methods
static async withRetry(operation, options) {
const executor = new RetryExecutor(options);
const result = await executor.execute(operation);
if (result.success) {
return result.result;
}
else {
throw result.error;
}
}
static isRetryableError(error) {
// Network errors
if (error.code === 'ECONNRESET' ||
error.code === 'ENOTFOUND' ||
error.code === 'ECONNREFUSED' ||
error.code === 'ETIMEDOUT') {
return true;
}
// HTTP errors (5xx)
if (error.response && error.response.status >= 500) {
return true;
}
// Rate limiting
if (error.response && error.response.status === 429) {
return true;
}
return false;
}
}
exports.RetryExecutor = RetryExecutor;
class CircuitBreaker extends events_1.EventEmitter {
constructor(name, options = {}) {
super();
this.name = name;
this.options = options;
this.state = CircuitState.CLOSED;
this.failureCount = 0;
this.successCount = 0;
this.totalCalls = 0;
this.stats = new Map();
this.defaultOptions = {
failureThreshold: 5,
recoveryTimeout: 60000,
monitoringPeriod: 60000,
volumeThreshold: 10,
errorThresholdPercentage: 50
};
this.options = { ...this.defaultOptions, ...options };
this.startMonitoring();
}
async execute(operation) {
if (this.state === CircuitState.OPEN) {
if (this.shouldAttemptReset()) {
this.setState(CircuitState.HALF_OPEN);
}
else {
throw new Error(`Circuit breaker '${this.name}' is OPEN. Next retry at ${this.nextRetryTime}`);
}
}
this.totalCalls++;
const startTime = Date.now();
try {
const result = await operation();
this.onSuccess(Date.now() - startTime);
return result;
}
catch (error) {
this.onFailure(error, Date.now() - startTime);
throw error;
}
}
onSuccess(duration) {
this.successCount++;
this.lastSuccessTime = new Date();
if (this.state === CircuitState.HALF_OPEN) {
this.setState(CircuitState.CLOSED);
this.failureCount = 0;
}
this.emit('success', {
duration,
state: this.state,
successCount: this.successCount
});
}
onFailure(error, duration) {
this.failureCount++;
this.lastFailureTime = new Date();
this.emit('failure', {
error,
duration,
state: this.state,
failureCount: this.failureCount
});
if (this.state === CircuitState.HALF_OPEN) {
this.setState(CircuitState.OPEN);
this.setNextRetryTime();
}
else if (this.shouldTrip()) {
this.setState(CircuitState.OPEN);
this.setNextRetryTime();
}
}
shouldTrip() {
if (this.totalCalls < this.options.volumeThreshold) {
return false;
}
const failureRate = (this.failureCount / this.totalCalls) * 100;
return failureRate >= this.options.errorThresholdPercentage;
}
shouldAttemptReset() {
return this.nextRetryTime ? new Date() >= this.nextRetryTime : false;
}
setNextRetryTime() {
this.nextRetryTime = new Date(Date.now() + this.options.recoveryTimeout);
}
setState(newState) {
const oldState = this.state;
this.state = newState;
this.emit('state:change', {
from: oldState,
to: newState,
timestamp: new Date()
});
if (this.options.onStateChange) {
this.options.onStateChange(newState);
}
}
startMonitoring() {
setInterval(() => {
this.resetCounters();
this.emit('monitoring:reset', this.getStats());
}, this.options.monitoringPeriod);
}
resetCounters() {
// Reset counters for monitoring period
if (this.state === CircuitState.CLOSED) {
this.failureCount = 0;
this.successCount = 0;
this.totalCalls = 0;
}
}
getStats() {
const failureRate = this.totalCalls > 0 ?
(this.failureCount / this.totalCalls) * 100 : 0;
return {
state: this.state,
failureCount: this.failureCount,
successCount: this.successCount,
totalCalls: this.totalCalls,
failureRate,
lastFailureTime: this.lastFailureTime,
lastSuccessTime: this.lastSuccessTime,
nextRetryTime: this.nextRetryTime
};
}
reset() {
this.setState(CircuitState.CLOSED);
this.failureCount = 0;
this.successCount = 0;
this.totalCalls = 0;
this.lastFailureTime = undefined;
this.lastSuccessTime = undefined;
this.nextRetryTime = undefined;
this.emit('reset', { timestamp: new Date() });
}
forceOpen() {
this.setState(CircuitState.OPEN);
this.setNextRetryTime();
this.emit('force:open', { timestamp: new Date() });
}
forceClose() {
this.setState(CircuitState.CLOSED);
this.failureCount = 0;
this.nextRetryTime = undefined;
this.emit('force:close', { timestamp: new Date() });
}
getName() {
return this.name;
}
getState() {
return this.state;
}
isOpen() {
return this.state === CircuitState.OPEN;
}
isClosed() {
return this.state === CircuitState.CLOSED;
}
isHalfOpen() {
return this.state === CircuitState.HALF_OPEN;
}
}
exports.CircuitBreaker = CircuitBreaker;
class RetryManager extends events_1.EventEmitter {
constructor() {
super(...arguments);
this.retryExecutors = new Map();
this.circuitBreakers = new Map();
}
createRetryExecutor(name, options) {
const executor = new RetryExecutor(options);
this.retryExecutors.set(name, executor);
executor.on('attempt:start', (data) => this.emit('executor:attempt:start', { name, ...data }));
executor.on('attempt:success', (data) => this.emit('executor:attempt:success', { name, ...data }));
executor.on('attempt:failure', (data) => this.emit('executor:attempt:failure', { name, ...data }));
executor.on('retry:exhausted', (data) => this.emit('executor:retry:exhausted', { name, ...data }));
return executor;
}
createCircuitBreaker(name, options) {
const breaker = new CircuitBreaker(name, options);
this.circuitBreakers.set(name, breaker);
breaker.on('success', (data) => this.emit('breaker:success', { name, ...data }));
breaker.on('failure', (data) => this.emit('breaker:failure', { name, ...data }));
breaker.on('state:change', (data) => this.emit('breaker:state:change', { name, ...data }));
return breaker;
}
async executeWithRetryAndCircuitBreaker(name, operation, retryOptions, circuitOptions) {
let executor = this.retryExecutors.get(name);
if (!executor) {
executor = this.createRetryExecutor(name, retryOptions);
}
let breaker = this.circuitBreakers.get(name);
if (!breaker) {
breaker = this.createCircuitBreaker(name, circuitOptions);
}
const result = await executor.execute(() => breaker.execute(operation));
return result.result;
}
getRetryExecutor(name) {
return this.retryExecutors.get(name);
}
getCircuitBreaker(name) {
return this.circuitBreakers.get(name);
}
getAllCircuitBreakers() {
return Array.from(this.circuitBreakers.values());
}
getAllRetryExecutors() {
return Array.from(this.retryExecutors.values());
}
getCircuitBreakerStats() {
return Array.from(this.circuitBreakers.entries()).map(([name, breaker]) => ({
name,
stats: breaker.getStats()
}));
}
resetAllCircuitBreakers() {
for (const breaker of this.circuitBreakers.values()) {
breaker.reset();
}
this.emit('reset:all');
}
getHealthStatus() {
const openCircuits = [];
const failedOperations = [];
for (const [name, breaker] of this.circuitBreakers) {
if (breaker.isOpen()) {
openCircuits.push(name);
}
const stats = breaker.getStats();
if (stats.failureRate > 25) { // 25% failure rate threshold
failedOperations.push(name);
}
}
return {
healthy: openCircuits.length === 0,
openCircuits,
totalCircuits: this.circuitBreakers.size,
failedOperations
};
}
// Convenience methods for common retry patterns
async retryNetworkOperation(operation, options) {
const defaultNetworkOptions = {
maxAttempts: 3,
baseDelay: 1000,
maxDelay: 10000,
retryCondition: RetryExecutor.isRetryableError
};
return RetryExecutor.withRetry(operation, { ...defaultNetworkOptions, ...options });
}
async retryFileOperation(operation, options) {
const defaultFileOptions = {
maxAttempts: 5,
baseDelay: 500,
maxDelay: 5000,
retryCondition: (error) => {
return error.code === 'EBUSY' ||
error.code === 'EMFILE' ||
error.code === 'ENFILE' ||
error.code === 'EAGAIN';
}
};
return RetryExecutor.withRetry(operation, { ...defaultFileOptions, ...options });
}
async retryDatabaseOperation(operation, options) {
const defaultDbOptions = {
maxAttempts: 3,
baseDelay: 2000,
maxDelay: 20000,
retryCondition: (error) => {
return error.code === 'ECONNRESET' ||
error.code === 'ECONNREFUSED' ||
error.message?.includes('timeout') ||
error.message?.includes('deadlock');
}
};
return RetryExecutor.withRetry(operation, { ...defaultDbOptions, ...options });
}
}
exports.RetryManager = RetryManager;
// Global retry manager
let globalRetryManager = null;
function createRetryManager() {
return new RetryManager();
}
function getGlobalRetryManager() {
if (!globalRetryManager) {
globalRetryManager = new RetryManager();
}
return globalRetryManager;
}
function setGlobalRetryManager(manager) {
globalRetryManager = manager;
}