@bcoders.gr/virtualpool
Version:
Advanced Ethereum testing framework with provider pooling, load balancing, and comprehensive monitoring
906 lines (793 loc) • 33.8 kB
JavaScript
import { spawn, exec } from 'child_process';
import { promisify } from 'util';
import net from 'net';
import { EventEmitter } from 'events';
import { IPCProvider } from '@bcoders.gr/provider';
const execAsync = promisify(exec);
class AnvilProviderPool extends EventEmitter {
/**
* @param {number} portStart - Starting port
* @param {number} portEnd - Ending port
* @param {object} options - Pool options
* @param {function} [options.logger] - Optional custom logger
* @param {object} [options.providerOptions] - Options for IPCProvider
*/
constructor(portStart, portEnd, options = {}) {
super(); // Initialize EventEmitter
// Validate inputs
if (typeof portStart !== 'number' || typeof portEnd !== 'number') {
throw new Error('Port start and end must be numbers');
}
if (portStart < 1 || portStart > 65535 || portEnd < 1 || portEnd > 65535) {
throw new Error('Ports must be between 1 and 65535');
}
if (portStart > portEnd) {
throw new Error('Port start cannot be greater than port end');
}
this.validateOptions(options);
this.portStart = portStart;
this.portEnd = portEnd;
this.providers = new Map();
this.processes = new Map();
this.busyStatus = new Map();
this.providerHealth = new Map();
this.restartAttempts = new Map();
this.isShuttingDown = false;
this.options = {
timeout: 30000,
maxRetries: 5,
maxRetriesPerPort: 3,
healthCheckInterval: 30000,
anvilPath: '/root/.foundry/bin/anvil',
maxParallelStarts: 5,
waitInterval: 10,
gracefulShutdownTimeout: 5000,
providerOptions: {}, // Allow provider options override
...options
};
this.log = this.options.logger || ((msg) => console.log(`[AnvilProviderPool] ${msg}`));
this.pLimit = this.createPLimit(this.options.maxParallelStarts);
this.log(`Initializing pool for ports ${portStart} to ${portEnd}.`);
this.noisyLogRegex = /^(eth_|evm_|anvil_|Transaction)/;
if (this.options.healthCheckInterval > 0) {
this.healthCheckInterval = setInterval(() => {
this.performHealthCheck().catch(err =>
this.log(`Health check error: ${err.message}`)
);
}, this.options.healthCheckInterval);
}
// Initialize metrics
this.initializeMetrics();
}
/**
* Initialize metrics collection
*/
initializeMetrics() {
this.metrics = {
totalRequests: 0,
successfulRequests: 0,
failedRequests: 0,
averageResponseTime: 0,
responseTimes: [],
providerUsage: new Map(),
startTime: Date.now(),
lastHealthCheck: null,
restartHistory: []
};
// Initialize provider usage counters
for (let port = this.portStart; port <= this.portEnd; port++) {
this.metrics.providerUsage.set(port, {
requestCount: 0,
totalResponseTime: 0,
errors: 0,
lastUsed: null
});
}
}
/**
* Record request metrics
*/
recordRequest(port, responseTime, success = true) {
this.metrics.totalRequests++;
if (success) {
this.metrics.successfulRequests++;
} else {
this.metrics.failedRequests++;
}
// Update response times (keep last 1000 for rolling average)
this.metrics.responseTimes.push(responseTime);
if (this.metrics.responseTimes.length > 1000) {
this.metrics.responseTimes.shift();
}
// Calculate average response time
this.metrics.averageResponseTime =
this.metrics.responseTimes.reduce((a, b) => a + b, 0) / this.metrics.responseTimes.length;
// Update provider-specific metrics
const providerMetrics = this.metrics.providerUsage.get(port);
if (providerMetrics) {
providerMetrics.requestCount++;
providerMetrics.totalResponseTime += responseTime;
providerMetrics.lastUsed = Date.now();
if (!success) {
providerMetrics.errors++;
}
}
this.emit('requestCompleted', {
port,
responseTime,
success,
totalRequests: this.metrics.totalRequests
});
}
/**
* Get comprehensive metrics
*/
getMetrics() {
const uptime = Date.now() - this.metrics.startTime;
const requestsPerSecond = this.metrics.totalRequests / (uptime / 1000);
return {
uptime: Math.round(uptime / 1000), // seconds
totalRequests: this.metrics.totalRequests,
successfulRequests: this.metrics.successfulRequests,
failedRequests: this.metrics.failedRequests,
successRate: this.metrics.totalRequests > 0
? ((this.metrics.successfulRequests / this.metrics.totalRequests) * 100).toFixed(2) + '%'
: '0%',
averageResponseTime: Math.round(this.metrics.averageResponseTime * 100) / 100,
requestsPerSecond: Math.round(requestsPerSecond * 100) / 100,
providerUsage: Object.fromEntries(
Array.from(this.metrics.providerUsage.entries()).map(([port, usage]) => [
port,
{
...usage,
averageResponseTime: usage.requestCount > 0
? Math.round((usage.totalResponseTime / usage.requestCount) * 100) / 100
: 0,
errorRate: usage.requestCount > 0
? ((usage.errors / usage.requestCount) * 100).toFixed(2) + '%'
: '0%'
}
])
),
restartHistory: this.metrics.restartHistory.slice(-10) // Last 10 restarts
};
}
/**
* Simple p-limit implementation for controlling concurrency
*/
createPLimit(concurrency) {
const queue = [];
let activeCount = 0;
const next = () => {
activeCount--;
if (queue.length > 0) {
queue.shift()();
}
};
return (fn) => {
return new Promise((resolve, reject) => {
const run = async () => {
activeCount++;
try {
const result = await fn();
resolve(result);
} catch (error) {
reject(error);
} finally {
next();
}
};
if (activeCount < concurrency) {
run();
} else {
queue.push(run);
}
});
};
}
/**
* Calculate exponential backoff delay
*/
getBackoffDelay(attempt) {
const baseDelay = 1000; // 1 second
const maxDelay = 30000; // 30 seconds
return Math.min(maxDelay, baseDelay * Math.pow(2, attempt));
}
/**
* Get an available provider with enhanced selection strategy
*/
getAvailableProvider() {
// First, try to get a healthy provider
const healthyProviders = Array.from(this.providers.entries())
.filter(([port]) => this.providerHealth.get(port) === true && !this.busyStatus.get(port))
.sort(() => Math.random() - 0.5);
if (healthyProviders.length > 0) {
const [port, provider] = healthyProviders[0];
this.busyStatus.set(port, true);
this.log(`Provider on port ${port} allocated (${this.getAvailableCount()} remaining).`);
return { provider, port };
}
// Fallback to any available provider if no healthy ones
const shuffledProviders = Array.from(this.providers.entries())
.sort(() => Math.random() - 0.5);
for (const [port, provider] of shuffledProviders) {
if (!this.busyStatus.get(port)) {
this.busyStatus.set(port, true);
this.log(`Provider on port ${port} allocated (fallback, ${this.getAvailableCount()} remaining).`);
return { provider, port };
}
}
this.log(`No available providers out of ${this.providers.size} total.`);
return null;
}
/**
* Circuit breaker for provider health management
*/
isProviderCircuitOpen(port) {
const attempts = this.restartAttempts.get(port) || 0;
const isHealthy = this.providerHealth.get(port);
// Open circuit if too many restart attempts or provider is unhealthy
return attempts >= this.options.maxRetriesPerPort || isHealthy === false;
}
/**
* Enhanced provider selection with circuit breaker logic
*/
getAvailableProviderWithCircuitBreaker() {
// Filter out providers with open circuits
const availableProviders = Array.from(this.providers.entries())
.filter(([port]) =>
!this.busyStatus.get(port) &&
!this.isProviderCircuitOpen(port) &&
this.providerHealth.get(port) === true
)
.sort(() => Math.random() - 0.5);
if (availableProviders.length > 0) {
const [port, provider] = availableProviders[0];
this.busyStatus.set(port, true);
this.log(`Provider on port ${port} allocated with circuit breaker (${this.getAvailableCount()} remaining).`);
return { provider, port };
}
// Fallback to regular selection if no circuit-safe providers
return this.getAvailableProvider();
}
/**
* Unmark a provider as busy with validation.
*/
releaseProvider(port) {
if (typeof port !== 'number' || port < 1 || port > 65535) {
this.log(`Invalid port number: ${port}`);
return false;
}
if (this.busyStatus.has(port)) {
this.busyStatus.set(port, false);
this.log(`Provider on port ${port} released (${this.getAvailableCount()} available).`);
return true;
} else {
this.log(`Port ${port} is not tracked or already available.`);
return false;
}
}
/**
* Get the count of available (not busy) providers
*/
getAvailableCount() {
return Array.from(this.busyStatus.values()).filter(busy => !busy).length;
}
/**
* Get the count of busy providers
*/
getBusyCount() {
return Array.from(this.busyStatus.values()).filter(busy => busy).length;
}
/**
* Get pool statistics
*/
getStats() {
const total = this.providers.size;
const available = this.getAvailableCount();
const busy = this.getBusyCount();
const healthy = Array.from(this.providerHealth.values()).filter(health => health === true).length;
const restartCounts = Array.from(this.restartAttempts.values());
const totalRestarts = restartCounts.reduce((sum, count) => sum + count, 0);
// Get metrics data
const metrics = this.getMetrics();
return {
// Pool status
total,
available,
busy,
healthy,
unhealthy: total - healthy,
utilizationRate: total > 0 ? (busy / total * 100).toFixed(1) + '%' : '0%',
// Restart information
totalRestarts,
restartHistory: metrics.restartHistory,
// Performance metrics
uptime: metrics.uptime,
totalRequests: metrics.totalRequests,
successRate: metrics.successRate,
averageResponseTime: metrics.averageResponseTime,
requestsPerSecond: metrics.requestsPerSecond,
// Provider details
ports: Array.from(this.providers.keys()),
providerDetails: Object.fromEntries(
Array.from(this.providers.keys()).map(port => [
port,
{
...this.getProviderInfo(port),
metrics: metrics.providerUsage[port] || {}
}
])
)
};
}
/**
* Perform health check on all providers
*/
async performHealthCheck() {
const healthPromises = Array.from(this.providers.entries()).map(async ([port, provider]) => {
try {
// Simple health check - get block number with timeout
const timeoutPromise = new Promise((_, reject) =>
setTimeout(() => reject(new Error('Health check timeout')), 5000)
);
await Promise.race([
provider.send('eth_blockNumber', []),
timeoutPromise
]);
const wasUnhealthy = this.providerHealth.get(port) === false;
this.providerHealth.set(port, true);
if (wasUnhealthy) {
this.emit('providerRecovered', { port });
this.log(`Provider on port ${port} recovered`);
}
} catch (error) {
const wasHealthy = this.providerHealth.get(port) === true;
this.providerHealth.set(port, false);
this.log(`Health check failed for port ${port}: ${error.message}`);
if (wasHealthy) {
this.emit('providerFailed', { port, error: error.message });
}
// Consider restarting unhealthy providers
if (!this.isShuttingDown) {
this.restartProvider(port);
}
}
});
await Promise.allSettled(healthPromises);
this.emit('healthCheckCompleted', this.getStats());
}
/**
* Restart a specific provider with exponential backoff
*/
async restartProvider(port) {
const attempts = this.restartAttempts.get(port) || 0;
if (attempts >= this.options.maxRetriesPerPort) {
this.log(`Max restart attempts (${this.options.maxRetriesPerPort}) reached for port ${port}`);
return;
}
this.restartAttempts.set(port, attempts + 1);
const delay = this.getBackoffDelay(attempts);
// Record restart attempt in metrics
this.metrics.restartHistory.push({
port,
timestamp: Date.now(),
attempt: attempts + 1,
reason: 'health_check_failed'
});
this.log(`Restarting provider on port ${port} (attempt ${attempts + 1}/${this.options.maxRetriesPerPort}) after ${delay}ms delay`);
this.emit('providerRestarting', { port, attempt: attempts + 1, delay });
setTimeout(async () => {
try {
await this.killProcessOnPort(port);
await this.startAnvil(port);
this.log(`Successfully restarted provider on port ${port}`);
// Reset restart attempts on successful restart
this.restartAttempts.set(port, 0);
this.emit('providerRestarted', { port, successful: true });
} catch (error) {
this.log(`Failed to restart provider on port ${port}: ${error.message}`);
this.emit('providerRestarted', { port, successful: false, error: error.message });
}
}, delay);
}
/**
* Execute a function with an automatically managed provider
*/
async withProvider(fn) {
const providerInfo = this.getAvailableProvider();
if (!providerInfo) {
throw new Error('No available providers in the pool');
}
const { provider, port } = providerInfo;
const startTime = Date.now();
let success = true;
try {
const result = await fn(provider);
return result;
} catch (error) {
success = false;
throw error;
} finally {
const responseTime = Date.now() - startTime;
this.recordRequest(port, responseTime, success);
this.releaseProvider(port);
}
}
/**
* Wait for an available provider with configurable timeout and interval
*/
async waitForProvider(timeoutMs = 10000) {
const startTime = Date.now();
while (Date.now() - startTime < timeoutMs) {
const providerInfo = this.getAvailableProvider();
if (providerInfo) {
return providerInfo;
}
// Use configurable wait interval
await new Promise(resolve => setTimeout(resolve, this.options.waitInterval));
}
throw new Error(`No providers became available within ${timeoutMs}ms`);
}
/**
* Execute a function with an automatically managed provider (with wait)
*/
async withProviderWait(fn, timeoutMs = 10000) {
const providerInfo = await this.waitForProvider(timeoutMs);
const { provider, port } = providerInfo;
const startTime = Date.now();
let success = true;
try {
const result = await fn(provider);
return result;
} catch (error) {
success = false;
throw error;
} finally {
const responseTime = Date.now() - startTime;
this.recordRequest(port, responseTime, success);
this.releaseProvider(port);
}
}
/**
* Check if a port is in use.
* @param {number} port
* @returns {Promise<boolean>}
*/
async isPortInUse(port) {
return new Promise((resolve) => {
const server = net.createServer();
server.once('error', (err) => resolve(err.code === 'EADDRINUSE'));
server.once('listening', () => server.close(() => resolve(false)));
server.listen(port);
});
}
/**
* Kill any process running on the specified port with graceful shutdown and IPC cleanup
* @param {number} port
*/
async killProcessOnPort(port) {
try {
const { stdout } = await execAsync(`lsof -i :${port} -t`);
if (!stdout.trim()) {
// Clean up orphaned IPC file if exists
const ipcPath = `/tmp/anvil${port}.ipc`;
await execAsync(`rm -f ${ipcPath}`);
return;
}
const pid = stdout.trim();
this.log(`Found process with PID ${pid} on port ${port}. Attempting graceful shutdown.`);
try {
await execAsync(`kill -15 ${pid}`);
await new Promise(resolve => setTimeout(resolve, this.options.gracefulShutdownTimeout));
try {
await execAsync(`kill -0 ${pid}`);
this.log(`Process ${pid} didn't shutdown gracefully, force killing.`);
await execAsync(`kill -9 ${pid}`);
} catch {
// Process already terminated
}
} catch (error) {
this.log(`Failed to kill process on port ${port}: ${error.message}`);
}
// Clean up IPC file after killing
const ipcPath = `/tmp/anvil${port}.ipc`;
await execAsync(`rm -f ${ipcPath}`);
} catch (error) {
// No process found or lsof failed
// Clean up orphaned IPC file if exists
const ipcPath = `/tmp/anvil${port}.ipc`;
await execAsync(`rm -f ${ipcPath}`);
}
}
/**
* Start an Anvil instance on the specified port with enhanced error handling
* @param {number} port
* @returns {Promise<IPCProvider>}
*/
async startAnvil(port) {
return this.pLimit(async () => {
this.log(`Starting Anvil instance on port ${port}.`);
await this.killProcessOnPort(port);
const ipcPath = `/tmp/anvil${port}.ipc`;
const args = [
"--auto-impersonate",
"--no-mining",
"--base-fee",
"0",
"--ipc",
ipcPath,
"--port",
port.toString(),
];
return new Promise((resolve, reject) => {
const anvilProcess = spawn(this.options.anvilPath, args);
const timeout = setTimeout(() => {
anvilProcess.kill('SIGTERM');
setTimeout(() => anvilProcess.kill('SIGKILL'), 2000);
reject(new Error(`[Anvil ${port}] Process timeout exceeded.`));
}, this.options.timeout);
anvilProcess.stdout.on('data', (data) => {
const message = data.toString().trim();
if (!this.noisyLogRegex.test(message) && message.includes("Listening on")) {
clearTimeout(timeout);
this.log(`[Anvil ${port}] is ready and listening.`);
setTimeout(async () => {
try {
const provider = new IPCProvider(ipcPath, {
cacheEnabled: true,
batchRequests: false,
autoReconnect: true,
requestTimeout: 10000,
...this.options.providerOptions
});
await provider.connect();
await provider.send('eth_blockNumber', []);
this.providers.set(port, provider);
this.processes.set(port, anvilProcess);
this.busyStatus.set(port, false);
this.providerHealth.set(port, true);
this.restartAttempts.set(port, 0);
resolve(provider);
} catch (error) {
this.log(`[Anvil ${port}] Provider connection failed: ${error.message}`);
anvilProcess.kill('SIGTERM');
await execAsync(`rm -f ${ipcPath}`); // Clean up IPC file
reject(error);
}
}, 1000);
}
});
anvilProcess.stderr.on('data', (data) => {
const message = data.toString().trim();
if (!this.noisyLogRegex.test(message)) {
this.log(`[Anvil ${port}] STDERR: ${message}`);
}
});
anvilProcess.on('error', (err) => {
clearTimeout(timeout);
this.log(`[Anvil ${port}] Process error: ${err.message}`);
reject(err);
});
anvilProcess.on('close', async (code) => {
clearTimeout(timeout);
this.log(`[Anvil ${port}] Process exited with code ${code}.`);
const provider = this.providers.get(port);
if (provider && typeof provider.disconnect === 'function') {
try {
await provider.disconnect();
} catch (error) {
// Ignore disconnect errors during cleanup
}
}
this.providers.delete(port);
this.processes.delete(port);
this.providerHealth.set(port, false);
await execAsync(`rm -f ${ipcPath}`); // Clean up IPC file
if (!this.isShuttingDown) {
this.restartProvider(port);
}
});
});
});
}
/**
* Initialize the provider pool by starting all instances
*/
async initialize() {
if (this.portStart > this.portEnd) {
throw new Error('Port start cannot be greater than port end');
}
const ports = [];
for (let port = this.portStart; port <= this.portEnd; port++) {
ports.push(port);
}
this.log(`Initializing ${ports.length} providers...`);
const initPromises = ports.map(port =>
this.startAnvil(port).catch(error => {
this.log(`Failed to initialize provider on port ${port}: ${error.message}`);
return null;
})
);
const results = await Promise.allSettled(initPromises);
const successful = results.filter(result => result.status === 'fulfilled' && result.value !== null).length;
this.log(`Pool initialization completed: ${successful}/${ports.length} providers started successfully`);
if (successful === 0) {
throw new Error('Failed to initialize any providers');
}
return this.getStats();
}
/**
* Shutdown the provider pool and all managed providers
*/
async shutdown(forceTimeout = 10000) {
this.log(`Shutting down provider pool.`);
this.isShuttingDown = true;
// Clear health check interval
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval);
}
this.emit('shuttingDown');
// Wait for all providers to become idle with timeout
const waitForIdle = new Promise((resolve) => {
const checkIdle = setInterval(() => {
if (this.getBusyCount() === 0) {
clearInterval(checkIdle);
resolve();
}
}, 100);
});
const forceShutdown = new Promise(resolve =>
setTimeout(resolve, forceTimeout)
);
await Promise.race([waitForIdle, forceShutdown]);
// Cleanup all ports
const cleanupPromises = Array.from(this.providers.keys()).map(port =>
this.cleanupPort(port)
);
await Promise.allSettled(cleanupPromises);
this.emit('shutdown');
this.log(`Provider pool shutdown completed.`);
}
/**
* Cleanup resources for a specific port
*/
async cleanupPort(port) {
try {
const provider = this.providers.get(port);
if (provider && typeof provider.disconnect === 'function') {
await provider.disconnect();
}
const process = this.processes.get(port);
if (process) {
process.kill('SIGTERM');
setTimeout(() => process.kill('SIGKILL'), 2000);
}
// Clean up IPC file
const ipcPath = `/tmp/anvil${port}.ipc`;
await execAsync(`rm -f ${ipcPath}`).catch(() => {
// Ignore cleanup errors
});
// Clean up maps
this.providers.delete(port);
this.processes.delete(port);
this.busyStatus.delete(port);
this.providerHealth.delete(port);
this.emit('portCleaned', { port });
} catch (error) {
this.log(`Error cleaning up port ${port}: ${error.message}`);
}
}
/**
* Get detailed provider information
*/
getProviderInfo(port) {
return {
port,
hasProvider: this.providers.has(port),
hasProcess: this.processes.has(port),
isBusy: this.busyStatus.get(port) || false,
isHealthy: this.providerHealth.get(port) || false,
restartAttempts: this.restartAttempts.get(port) || 0,
circuitOpen: this.isProviderCircuitOpen(port)
};
}
/**
* Validate configuration options
*/
validateOptions(options) {
const errors = [];
if (options.timeout && (typeof options.timeout !== 'number' || options.timeout < 1000)) {
errors.push('Timeout must be a number >= 1000ms');
}
if (options.maxRetries && (typeof options.maxRetries !== 'number' || options.maxRetries < 1)) {
errors.push('maxRetries must be a positive number');
}
if (options.maxRetriesPerPort && (typeof options.maxRetriesPerPort !== 'number' || options.maxRetriesPerPort < 1)) {
errors.push('maxRetriesPerPort must be a positive number');
}
if (options.healthCheckInterval && typeof options.healthCheckInterval !== 'number') {
errors.push('healthCheckInterval must be a number');
}
if (options.maxParallelStarts && (typeof options.maxParallelStarts !== 'number' || options.maxParallelStarts < 1)) {
errors.push('maxParallelStarts must be a positive number');
}
if (errors.length > 0) {
throw new Error(`Configuration validation failed: ${errors.join(', ')}`);
}
}
/**
* Load balancing strategies
*/
getProviderByStrategy(strategy = 'random') {
const availableProviders = Array.from(this.providers.entries())
.filter(([port]) =>
!this.busyStatus.get(port) &&
this.providerHealth.get(port) === true
);
if (availableProviders.length === 0) {
return null;
}
let selectedProvider;
switch (strategy) {
case 'roundRobin':
if (!this.roundRobinIndex) this.roundRobinIndex = 0;
selectedProvider = availableProviders[this.roundRobinIndex % availableProviders.length];
this.roundRobinIndex++;
break;
case 'leastUsed':
selectedProvider = availableProviders.reduce((least, current) => {
const leastUsage = this.metrics.providerUsage.get(least[0]) || { requestCount: 0 };
const currentUsage = this.metrics.providerUsage.get(current[0]) || { requestCount: 0 };
return leastUsage.requestCount <= currentUsage.requestCount ? least : current;
});
break;
case 'fastest':
selectedProvider = availableProviders.reduce((fastest, current) => {
const fastestUsage = this.metrics.providerUsage.get(fastest[0]) || { totalResponseTime: 0, requestCount: 1 };
const currentUsage = this.metrics.providerUsage.get(current[0]) || { totalResponseTime: 0, requestCount: 1 };
const fastestAvg = fastestUsage.totalResponseTime / fastestUsage.requestCount;
const currentAvg = currentUsage.totalResponseTime / currentUsage.requestCount;
return fastestAvg <= currentAvg ? fastest : current;
});
break;
case 'random':
default:
const randomIndex = Math.floor(Math.random() * availableProviders.length);
selectedProvider = availableProviders[randomIndex];
break;
}
if (selectedProvider) {
const [port, provider] = selectedProvider;
this.busyStatus.set(port, true);
this.log(`Provider on port ${port} allocated using ${strategy} strategy (${this.getAvailableCount()} remaining).`);
return { provider, port };
}
return null;
}
/**
* Execute a function with load balancing strategy
*/
async withProviderStrategy(fn, strategy = 'random', timeoutMs = 10000) {
let providerInfo = this.getProviderByStrategy(strategy);
// Fallback to waiting if no provider available
if (!providerInfo) {
providerInfo = await this.waitForProvider(timeoutMs);
}
const { provider, port } = providerInfo;
const startTime = Date.now();
let success = true;
try {
const result = await fn(provider);
return result;
} catch (error) {
success = false;
throw error;
} finally {
const responseTime = Date.now() - startTime;
this.recordRequest(port, responseTime, success);
this.releaseProvider(port);
}
}
}
export default AnvilProviderPool;