hook-engine
Version:
Production-grade webhook engine with comprehensive adapter support, security, reliability, structured logging, and CLI tools.
268 lines (267 loc) • 10.2 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RetryEngine = void 0;
exports.retry = retry;
const webhook_errors_1 = require("../errors/webhook-errors");
const base_1 = require("../errors/base");
const timing_1 = require("../utils/timing");
/**
* Enhanced retry system with advanced features
*/
class RetryEngine {
constructor(config, deadLetterQueue) {
this.circuitBreakerState = 'closed';
this.failureCount = 0;
this.lastFailureTime = 0;
this.budgetResetTime = 0;
this.config = config;
this.deadLetterQueue = deadLetterQueue;
// Circuit breaker configuration
this.circuitBreakerConfig = {
failureThreshold: 5,
recoveryTimeout: 60000, // 1 minute
monitoringPeriod: 300000, // 5 minutes
};
// Initialize retry budget (max retries per hour)
this.retryBudget = 100;
this.budgetResetTime = Date.now() + 3600000; // 1 hour
}
/**
* Execute function with retry logic
*/
async execute(fn, event, policy = 'exponential') {
const timer = new timing_1.Timer();
let lastError;
let attempt = 0;
// Check circuit breaker
if (this.isCircuitOpen()) {
return {
success: false,
error: new webhook_errors_1.WebhookProcessingError('Circuit breaker is open', { eventId: event.id }),
attempts: 0,
totalTime: 0,
failureReason: 'circuit_breaker'
};
}
// Check retry budget
if (!this.hasRetryBudget()) {
return {
success: false,
error: new webhook_errors_1.WebhookProcessingError('Retry budget exhausted', { eventId: event.id }),
attempts: 0,
totalTime: 0,
failureReason: 'max_attempts'
};
}
while (attempt < this.config.maxAttempts) {
attempt++;
try {
const result = await (0, timing_1.withTimeout)(fn(), this.config.retryOn.includes('TIMEOUT_ERROR') ? 30000 : 60000, `Webhook processing timeout for event ${event.id}`);
// Success - record and return
this.recordSuccess();
return {
success: true,
result,
attempts: attempt,
totalTime: timer.elapsed()
};
}
catch (error) {
lastError = error;
// Check if error is retryable
if (!this.isRetryableError(lastError)) {
this.recordFailure();
await this.sendToDeadLetter(event, lastError, attempt);
return {
success: false,
error: lastError,
attempts: attempt,
totalTime: timer.elapsed(),
failureReason: 'non_retryable'
};
}
// Check if we should continue retrying
if (attempt >= this.config.maxAttempts) {
this.recordFailure();
await this.sendToDeadLetter(event, lastError, attempt);
return {
success: false,
error: lastError,
attempts: attempt,
totalTime: timer.elapsed(),
failureReason: 'max_attempts'
};
}
// Check circuit breaker after each failure
if (this.isCircuitOpen()) {
await this.sendToDeadLetter(event, lastError, attempt);
return {
success: false,
error: lastError,
attempts: attempt,
totalTime: timer.elapsed(),
failureReason: 'circuit_breaker'
};
}
// Calculate delay and wait
const delay = this.calculateDelay(attempt, policy);
console.warn(`🔄 Retry ${attempt}/${this.config.maxAttempts} for ${event.id} after ${delay}ms`);
// Consume retry budget
this.consumeRetryBudget();
await (0, timing_1.sleep)(delay);
}
}
// This should never be reached, but just in case
this.recordFailure();
await this.sendToDeadLetter(event, lastError, attempt);
return {
success: false,
error: lastError,
attempts: attempt,
totalTime: timer.elapsed(),
failureReason: 'max_attempts'
};
}
/**
* Batch retry execution
*/
async executeBatch(operations, policy = 'exponential') {
const results = await Promise.allSettled(operations.map(({ fn, event }) => this.execute(fn, event, policy)));
return results.map(result => result.status === 'fulfilled' ? result.value : {
success: false,
error: new Error('Batch operation failed'),
attempts: 0,
totalTime: 0,
failureReason: 'non_retryable'
});
}
/**
* Get retry statistics
*/
getStats() {
return {
circuitBreakerState: this.circuitBreakerState,
failureCount: this.failureCount,
retryBudget: this.retryBudget,
budgetResetTime: new Date(this.budgetResetTime)
};
}
/**
* Reset circuit breaker manually
*/
resetCircuitBreaker() {
this.circuitBreakerState = 'closed';
this.failureCount = 0;
this.lastFailureTime = 0;
}
/**
* Reset retry budget
*/
resetRetryBudget() {
this.retryBudget = 100;
this.budgetResetTime = Date.now() + 3600000;
}
isRetryableError(error) {
if (error instanceof base_1.HookEngineError) {
return error.retryable;
}
// Check if error code is in retryable list
const errorCode = error.code || 'UNKNOWN_ERROR';
return this.config.retryOn.includes(errorCode);
}
calculateDelay(attempt, policy) {
switch (policy) {
case 'exponential':
return (0, timing_1.calculateBackoffDelay)(attempt, this.config.initialDelayMs, this.config.backoffMultiplier, this.config.maxDelayMs, this.config.jitter);
case 'linear':
const linearDelay = this.config.initialDelayMs * attempt;
return Math.min(linearDelay, this.config.maxDelayMs);
case 'fixed':
return this.config.initialDelayMs;
case 'custom':
// Custom policy could be implemented here
return (0, timing_1.calculateBackoffDelay)(attempt, this.config.initialDelayMs, this.config.backoffMultiplier, this.config.maxDelayMs, this.config.jitter);
default:
return (0, timing_1.calculateBackoffDelay)(attempt, this.config.initialDelayMs, this.config.backoffMultiplier, this.config.maxDelayMs, this.config.jitter);
}
}
isCircuitOpen() {
const now = Date.now();
// Check if we need to reset the monitoring period
if (now - this.lastFailureTime > this.circuitBreakerConfig.monitoringPeriod) {
this.failureCount = 0;
this.circuitBreakerState = 'closed';
}
// Check if circuit should be opened
if (this.circuitBreakerState === 'closed' &&
this.failureCount >= this.circuitBreakerConfig.failureThreshold) {
this.circuitBreakerState = 'open';
console.warn(`🔴 Circuit breaker opened after ${this.failureCount} failures`);
}
// Check if circuit should move to half-open
if (this.circuitBreakerState === 'open' &&
now - this.lastFailureTime > this.circuitBreakerConfig.recoveryTimeout) {
this.circuitBreakerState = 'half-open';
console.info('🟡 Circuit breaker moved to half-open state');
}
return this.circuitBreakerState === 'open';
}
recordSuccess() {
if (this.circuitBreakerState === 'half-open') {
this.circuitBreakerState = 'closed';
this.failureCount = 0;
console.info('🟢 Circuit breaker closed after successful operation');
}
}
recordFailure() {
this.failureCount++;
this.lastFailureTime = Date.now();
if (this.circuitBreakerState === 'half-open') {
this.circuitBreakerState = 'open';
console.warn('🔴 Circuit breaker opened again after failure in half-open state');
}
}
hasRetryBudget() {
const now = Date.now();
// Reset budget if time window passed
if (now > this.budgetResetTime) {
this.resetRetryBudget();
}
return this.retryBudget > 0;
}
consumeRetryBudget() {
if (this.retryBudget > 0) {
this.retryBudget--;
}
}
async sendToDeadLetter(event, error, attempts) {
if (this.deadLetterQueue) {
try {
await this.deadLetterQueue.add(event, error, attempts);
console.error(`💀 Sent event ${event.id} to dead letter queue after ${attempts} attempts`);
}
catch (dlqError) {
console.error('Failed to send to dead letter queue:', dlqError);
}
}
}
}
exports.RetryEngine = RetryEngine;
/**
* Legacy retry function for backwards compatibility
*/
async function retry(event, fn, maxAttempts = 3) {
const config = {
maxAttempts,
initialDelayMs: 100,
maxDelayMs: 30000,
backoffMultiplier: 2,
jitter: true,
retryOn: ['WEBHOOK_PROCESSING_ERROR', 'NETWORK_ERROR', 'TIMEOUT_ERROR']
};
const retryEngine = new RetryEngine(config);
const result = await retryEngine.execute(fn, event);
if (!result.success) {
throw result.error;
}
}