@bcoders.gr/virtualpool/src/lib/AnvilProviderPool.js

High-performance Ethereum testing framework with advanced provider pooling, adaptive load balancing, and comprehensive monitoring

import { spawn, exec } from 'child_process';
import { promisify } from 'util';
import net from 'net';
import EventEmitter from 'eventemitter3';
import PQueue from 'p-queue';
import NodeCache from 'node-cache';
import debug from 'debug';
import { IPCProvider } from '@bcoders.gr/eth-provider';

const execAsync = promisify(exec);
const log = debug('virtualpool:AnvilProviderPool');

class AnvilProviderPool extends EventEmitter {
    constructor(portStart, portEnd, options = {}) {
        super();

        if (typeof portStart !== 'number' || typeof portEnd !== 'number') {
            throw new Error('Port start and end must be numbers');
        }
        if (portStart < 1 || portStart > 65535 || portEnd < 1 || portEnd > 65535) {
            throw new Error('Ports must be between 1 and 65535');
        }
        if (portStart > portEnd) {
            throw new Error('Port start cannot be greater than port end');
        }

        this.options = {
            timeout: 30000,
            maxRetries: 5,
            maxRetriesPerPort: 3,
            healthCheckInterval: 30000,
            anvilPath: '/root/.foundry/bin/anvil',
            maxParallelStarts: 5,
            waitInterval: 10,
            gracefulShutdownTimeout: 5000,
            providerOptions: {},
            prewarmCount: 2,
            warmupPeriod: 60000,
            adaptiveLoadBalancing: true,
            ...options
        };

        this.portStart = portStart;
        this.portEnd = portEnd;

        // Avoid geth default port 8545, use 8550-8560 for Anvil
        if (this.portStart <= 8545 && this.portEnd >= 8545) {
            this.portStart = 8550;
            this.portEnd = 8560;
        }

        this.providers = new Map();
        this.processes = new Map();
        this.busyStatus = new Map();
        this.providerHealth = new Map();
        this.restartAttempts = new Map();
        this.warmupTimers = new Map();

        this.isShuttingDown = false;

        this.healthCheckCache = new NodeCache({ stdTTL: 30, checkperiod: 60 });

        this.pQueue = new PQueue({ concurrency: this.options.maxParallelStarts });

        this.log = this.options.logger || ((msg) => log(msg));

        this.noisyLogRegex = /^(eth_|evm_|anvil_|Transaction)/;

        if (this.options.healthCheckInterval > 0) {
            this.healthCheckInterval = setInterval(() => {
                this.performHealthCheck().catch(err => this.log(`Health check error: ${err.message}`));
            }, this.options.healthCheckInterval);
        }
    }

    async initialize() {
        const ports = [];
        for (let port = this.portStart; port <= this.portEnd; port++) {
            ports.push(port);
        }

        this.log(`Initializing ${ports.length} providers...`);

        const initPromises = ports.map(port => this.pQueue.add(() => this.startAnvil(port).catch(error => {
            this.log(`Failed to initialize provider on port ${port}: ${error.message}`);
            return null;
        })));

        const results = await Promise.allSettled(initPromises);
        const successful = results.filter(r => r.status === 'fulfilled' && r.value !== null).length;

        this.log(`Pool initialization completed: ${successful}/${ports.length} providers started successfully`);

        if (successful === 0) {
            throw new Error('Failed to initialize any providers');
        }

        await this.prewarmProviders();

        return this.getStats();
    }

    async prewarmProviders() {
        const availablePorts = Array.from(this.providers.keys()).slice(0, this.options.prewarmCount);
        this.log(`Prewarming ${availablePorts.length} providers...`);

        await Promise.all(availablePorts.map(async (port) => {
            try {
                await this.withProvider(async (provider) => {
                    await provider.getBlockNumber();
                });
                this.log(`Provider on port ${port} prewarmed`);
            } catch (error) {
                this.log(`Prewarm failed for provider on port ${port}: ${error.message}`);
            }
        }));
    }

    async startAnvil(port) {
        return this.pQueue.add(async () => {
            this.log(`Starting Anvil instance on port ${port}.`);
            await this.killProcessOnPort(port);
            const ipcPath = `/tmp/anvil${port}.ipc`;
            const args = [
                "--auto-impersonate",
                "--no-mining",
                "--base-fee",
                "0",
                "--ipc",
                ipcPath,
                "--port",
                port.toString(),
            ];

            return new Promise((resolve, reject) => {
                const anvilProcess = spawn(this.options.anvilPath, args);
                const timeout = setTimeout(() => {
                    anvilProcess.kill('SIGTERM');
                    setTimeout(() => anvilProcess.kill('SIGKILL'), 2000);
                    reject(new Error(`[Anvil ${port}] Process timeout exceeded.`));
                }, this.options.timeout);

                anvilProcess.stdout.on('data', (data) => {
                    const message = data.toString().trim();
                    if (!this.noisyLogRegex.test(message) && message.includes("Listening on")) {
                        clearTimeout(timeout);
                        this.log(`[Anvil ${port}] is ready and listening.`);
                        setTimeout(async () => {
                            try {
                                const provider = new IPCProvider(ipcPath, {
                                    cacheEnabled: false,
                                    batchRequests: false,
                                    autoReconnect: true,
                                    requestTimeout: 30000,
                                    reconnectDelay: 1000,
                                    maxReconnects: 5,
                                    ...this.options.providerOptions
                                });
                                // Removed await provider.connect(); because connect method does not exist
                                await new Promise(resolve => setTimeout(resolve, 1000));
                                this.providers.set(port, provider);
                                this.processes.set(port, anvilProcess);
                                this.busyStatus.set(port, false);
                                this.providerHealth.set(port, true);
                                this.restartAttempts.set(port, 0);
                                this.setWarmupTimer(port);
                                resolve(provider);
                            } catch (error) {
                                this.log(`[Anvil ${port}] Provider connection failed: ${error.message}`);
                                anvilProcess.kill('SIGTERM');
                                await execAsync(`rm -f ${ipcPath}`);
                                reject(error);
                            }
                        }, 2000);
                    }
                });

                anvilProcess.stderr.on('data', (data) => {
                    const message = data.toString().trim();
                    if (!this.noisyLogRegex.test(message)) {
                        this.log(`[Anvil ${port}] STDERR: ${message}`);
                    }
                });

                anvilProcess.on('error', (err) => {
                    clearTimeout(timeout);
                    this.log(`[Anvil ${port}] Process error: ${err.message}`);
                    reject(err);
                });

                anvilProcess.on('close', async (code) => {
                    clearTimeout(timeout);
                    this.log(`[Anvil ${port}] Process exited with code ${code}.`);
                    const provider = this.providers.get(port);
                    if (provider && typeof provider.disconnect === 'function') {
                        try {
                            await provider.disconnect();
                        } catch {}
                    }
                    this.providers.delete(port);
                    this.processes.delete(port);
                    this.providerHealth.set(port, false);
                    await execAsync(`rm -f ${ipcPath}`);
                    if (!this.isShuttingDown) {
                        this.restartProvider(port);
                    }
                });
            });
        });
    }

    setWarmupTimer(port) {
        if (this.warmupTimers.has(port)) {
            clearTimeout(this.warmupTimers.get(port));
        }
        const timer = setTimeout(() => {
            this.log(`Provider on port ${port} warmup period ended.`);
            this.warmupTimers.delete(port);
        }, this.options.warmupPeriod);
        this.warmupTimers.set(port, timer);
    }

    async killProcessOnPort(port) {
        try {
            const { stdout } = await execAsync(`lsof -i :${port} -t`);
            if (!stdout.trim()) {
                const ipcPath = `/tmp/anvil${port}.ipc`;
                await execAsync(`rm -f ${ipcPath}`);
                return;
            }
            const pid = stdout.trim();
            this.log(`Found process with PID ${pid} on port ${port}. Attempting graceful shutdown.`);
            try {
                await execAsync(`kill -15 ${pid}`);
                await new Promise(resolve => setTimeout(resolve, this.options.gracefulShutdownTimeout));
                try {
                    await execAsync(`kill -0 ${pid}`);
                    this.log(`Process ${pid} didn't shutdown gracefully, force killing.`);
                    await execAsync(`kill -9 ${pid}`);
                } catch {}
            } catch (error) {
                this.log(`Failed to kill process on port ${port}: ${error.message}`);
            }
            const ipcPath = `/tmp/anvil${port}.ipc`;
            await execAsync(`rm -f ${ipcPath}`);
        } catch {
            const ipcPath = `/tmp/anvil${port}.ipc`;
            await execAsync(`rm -f ${ipcPath}`);
        }
    }

    async performHealthCheck() {
        const healthPromises = Array.from(this.providers.entries()).map(async ([port, provider]) => {
            if (this.healthCheckCache.get(port)) {
                this.log(`Health check cache hit for port ${port}`);
                return;
            }
            try {
                const timeoutPromise = new Promise((_, reject) =>
                    setTimeout(() => reject(new Error('Health check timeout')), 5000)
                );
                await Promise.race([
                    provider.getBlockNumber(),
                    timeoutPromise
                ]);
                this.providerHealth.set(port, true);
                this.healthCheckCache.set(port, true);
                this.emit('providerRecovered', { port });
                this.log(`Provider on port ${port} is healthy.`);
            } catch (error) {
                this.providerHealth.set(port, false);
                this.healthCheckCache.del(port);
                this.log(`Health check failed for port ${port}: ${error.message}`);
                this.emit('providerFailed', { port, error: error.message });
                if (!this.isShuttingDown) {
                    this.restartProvider(port);
                }
            }
        });
        await Promise.allSettled(healthPromises);
        this.emit('healthCheckCompleted', this.getStats());
    }

    async restartProvider(port) {
        const attempts = this.restartAttempts.get(port) || 0;
        if (attempts >= this.options.maxRetriesPerPort) {
            this.log(`Max restart attempts reached for port ${port}`);
            return;
        }
        this.restartAttempts.set(port, attempts + 1);
        const delay = Math.min(30000, 1000 * 2 ** attempts);
        this.log(`Restarting provider on port ${port} in ${delay}ms (attempt ${attempts + 1})`);
        setTimeout(async () => {
            try {
                await this.killProcessOnPort(port);
                await this.startAnvil(port);
                this.restartAttempts.set(port, 0);
                this.emit('providerRestarted', { port, successful: true });
            } catch (error) {
                this.log(`Failed to restart provider on port ${port}: ${error.message}`);
                this.emit('providerRestarted', { port, successful: false, error: error.message });
            }
        }, delay);
    }

    getAvailableProvider() {
        const healthyProviders = Array.from(this.providers.entries())
            .filter(([port]) => this.providerHealth.get(port) === true && !this.busyStatus.get(port))
            .sort(() => Math.random() - 0.5);
        if (healthyProviders.length > 0) {
            const [port, provider] = healthyProviders[0];
            this.busyStatus.set(port, true);
            this.log(`Provider on port ${port} allocated (${this.getAvailableCount()} remaining).`);
            return { provider, port };
        }
        const shuffledProviders = Array.from(this.providers.entries())
            .sort(() => Math.random() - 0.5);
        for (const [port, provider] of shuffledProviders) {
            if (!this.busyStatus.get(port)) {
                this.busyStatus.set(port, true);
                this.log(`Provider on port ${port} allocated (fallback, ${this.getAvailableCount()} remaining).`);
                return { provider, port };
            }
        }
        this.log(`No available providers out of ${this.providers.size} total.`);
        return null;
    }

    releaseProvider(port) {
        if (typeof port !== 'number' || port < 1 || port > 65535) {
            this.log(`Invalid port number: ${port}`);
            return false;
        }
        if (this.busyStatus.has(port)) {
            this.busyStatus.set(port, false);
            this.log(`Provider on port ${port} released (${this.getAvailableCount()} available).`);
            return true;
        } else {
            this.log(`Port ${port} is not tracked or already available.`);
            return false;
        }
    }

    getAvailableCount() {
        return Array.from(this.busyStatus.values()).filter(busy => !busy).length;
    }

    getBusyCount() {
        return Array.from(this.busyStatus.values()).filter(busy => busy).length;
    }

    getStats() {
        const total = this.providers.size;
        const available = this.getAvailableCount();
        const busy = this.getBusyCount();
        const healthy = Array.from(this.providerHealth.values()).filter(health => health === true).length;
        const totalRestarts = Array.from(this.restartAttempts.values()).reduce((sum, count) => sum + count, 0);

        return {
            total,
            available,
            busy,
            healthy,
            unhealthy: total - healthy,
            utilizationRate: total > 0 ? ((busy / total) * 100).toFixed(1) + '%' : '0%',
            totalRestarts,
            uptime: 0,
            totalRequests: 0,
            successRate: 'N/A',
            averageResponseTime: 0,
            requestsPerSecond: 0,
            ports: Array.from(this.providers.keys()),
            providerDetails: Object.fromEntries(
                Array.from(this.providers.keys()).map(port => [
                    port,
                    this.getProviderInfo(port)
                ])
            )
        };
    }

    getProviderInfo(port) {
        return {
            port,
            hasProvider: this.providers.has(port),
            hasProcess: this.processes.has(port),
            isBusy: this.busyStatus.get(port) || false,
            isHealthy: this.providerHealth.get(port) || false,
            restartAttempts: this.restartAttempts.get(port) || 0,
            circuitOpen: this.restartAttempts.get(port) >= this.options.maxRetriesPerPort
        };
    }

    async withProvider(fn) {
        const providerInfo = this.getAvailableProvider();
        if (!providerInfo) {
            throw new Error('No available providers in the pool');
        }
        const { provider, port } = providerInfo;
        const startTime = Date.now();
        let success = true;
        try {
            const result = await fn(provider);
            return result;
        } catch (error) {
            success = false;
            throw error;
        } finally {
            const responseTime = Date.now() - startTime;
            this.releaseProvider(port);
        }
    }

    async waitForProvider(timeoutMs = 10000) {
        const startTime = Date.now();
        while (Date.now() - startTime < timeoutMs) {
            const providerInfo = this.getAvailableProvider();
            if (providerInfo) {
                return providerInfo;
            }
            await new Promise(resolve => setTimeout(resolve, this.options.waitInterval));
        }
        throw new Error(`No providers became available within ${timeoutMs}ms`);
    }

    async shutdown(forceTimeout = 10000) {
        this.log(`Shutting down provider pool.`);
        this.isShuttingDown = true;

        if (this.healthCheckInterval) {
            clearInterval(this.healthCheckInterval);
        }

        this.emit('shuttingDown');

        const waitForIdle = new Promise((resolve) => {
            const checkIdle = setInterval(() => {
                if (this.getBusyCount() === 0) {
                    clearInterval(checkIdle);
                    resolve();
                }
            }, 100);
        });

        const forceShutdown = new Promise(resolve => setTimeout(resolve, forceTimeout));

        await Promise.race([waitForIdle, forceShutdown]);

        const cleanupPromises = Array.from(this.providers.keys()).map(port => this.cleanupPort(port));

        await Promise.allSettled(cleanupPromises);

        this.emit('shutdown');
        this.log(`Provider pool shutdown completed.`);
    }

    async cleanupPort(port) {
        try {
            const provider = this.providers.get(port);
            if (provider && typeof provider.disconnect === 'function') {
                await provider.disconnect();
            }

            const process = this.processes.get(port);
            if (process) {
                process.kill('SIGTERM');
                setTimeout(() => process.kill('SIGKILL'), 2000);
            }

            const ipcPath = `/tmp/anvil${port}.ipc`;
            await execAsync(`rm -f ${ipcPath}`).catch(() => {});

            this.providers.delete(port);
            this.processes.delete(port);
            this.busyStatus.delete(port);
            this.providerHealth.delete(port);

            this.emit('portCleaned', { port });
        } catch (error) {
            this.log(`Error cleaning up port ${port}: ${error.message}`);
        }
    }
}

export default AnvilProviderPool;