@fanboynz/network-scanner/lib/nettools.js

A Puppeteer-based network scanner for analyzing web traffic, generating adblock filter rules, and identifying third-party requests. Features include fingerprint spoofing, Cloudflare bypass, content analysis with curl/grep, and multiple output formats.

/**
 * Network tools module for whois and dig lookups - COMPLETE FIXED VERSION
 * Provides domain analysis capabilities with proper timeout handling, custom whois servers, and retry logic
 */

const { exec, execSync } = require('child_process');
const fs = require('fs');
const path = require('path');
const { formatLogMessage, messageColors } = require('./colorize');
const ANSI_REGEX = /\x1b\[[0-9;]*m/g;

// Cycling index for whois server rotation
let whoisServerCycleIndex = 0;

// Global dig result cache — shared across ALL handler instances and processUrl calls
// Key: `${domain}-${recordType}`, Value: { result, timestamp }
// DNS records don't change based on what terms you're searching for,
// so we cache the raw dig output and let each handler check its own terms against it
const globalDigResultCache = new Map();
const GLOBAL_DIG_CACHE_TTL = 50400000; // 14 hours (persisted to disk between runs)
const GLOBAL_DIG_CACHE_MAX = 1000;

// Global whois result cache — shared across ALL handler instances and processUrl calls
// Whois data is per root domain and doesn't change based on search terms
const globalWhoisResultCache = new Map();
const GLOBAL_WHOIS_CACHE_TTL = 50400000; // 14 hours (persisted to disk between runs)
const GLOBAL_WHOIS_CACHE_MAX = 1000;

// Persistent disk cache file paths
const DIG_CACHE_FILE = path.join(__dirname, '..', '.digcache');
const WHOIS_CACHE_FILE = path.join(__dirname, '..', '.whoiscache');

/**
 * Load persistent cache from disk into in-memory Map
 * Skips expired entries and enforces max size
 * @param {string} filePath - Path to cache file
 * @param {Map} cache - In-memory cache Map to populate
 * @param {number} ttl - TTL in milliseconds
 * @param {number} maxSize - Maximum cache entries
 */
function loadDiskCache(filePath, cache, ttl, maxSize) {
  try {
    if (!fs.existsSync(filePath)) return;
    const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
    const now = Date.now();
    let loaded = 0;
    for (const [key, entry] of Object.entries(data)) {
      if (loaded >= maxSize) break;
      if (now - entry.timestamp < ttl) {
        cache.set(key, entry);
        loaded++;
      }
    }
  } catch {
    // Corrupt or unreadable cache file — delete and start fresh
    try { fs.unlinkSync(filePath); } catch {}
  }
}

/**
 * Save in-memory cache to disk, evicting oldest entries if over max size
 * @param {string} filePath - Path to cache file
 * @param {Map} cache - In-memory cache Map to persist
 * @param {number} ttl - TTL in milliseconds
 * @param {number} maxSize - Maximum cache entries
 */
function saveDiskCache(filePath, cache, ttl, maxSize) {
  try {
    const now = Date.now();
    const entries = {};
    let count = 0;

    // Collect valid entries, skip expired
    for (const [key, entry] of cache) {
      if (now - entry.timestamp < ttl) {
        entries[key] = entry;
        count++;
      }
    }

    // If over max, keep only the newest entries. Drop the pretty-print —
    // saveDiskCache runs on the synchronous 'exit' handler when --dns-cache
    // is set, so any work here directly delays scan exit. Compact JSON is
    // several times faster on multi-megabyte caches and the file is not
    // intended for human reading.
    if (count > maxSize) {
      const sorted = Object.entries(entries)
        .sort((a, b) => b[1].timestamp - a[1].timestamp)
        .slice(0, maxSize);
      const trimmed = {};
      for (const [key, entry] of sorted) {
        trimmed[key] = entry;
      }
      fs.writeFileSync(filePath, JSON.stringify(trimmed));
    } else {
      fs.writeFileSync(filePath, JSON.stringify(entries));
    }
  } catch {
    // Disk write failed — non-fatal, in-memory cache still works
  }
}

// Track in-flight lookups to prevent duplicate concurrent requests
const pendingDigLookups = new Map();
const pendingWhoisLookups = new Map();

// DNS cache statistics
const dnsCacheStats = { digHits: 0, digMisses: 0, whoisHits: 0, whoisMisses: 0, freshDig: [], freshWhois: [] };

/**
 * Get DNS cache statistics for end-of-scan reporting
 * @returns {Object} Cache hit/miss counts and fresh domain lists
 */
function getDnsCacheStats() {
  return { ...dnsCacheStats };
}

// Disk cache is opt-in via --dns-cache flag
let diskCacheEnabled = false;

/**
 * Enable persistent disk caching for dig/whois results
 * Call this when --dns-cache flag is set
 */
function enableDiskCache() {
  diskCacheEnabled = true;
  loadDiskCache(DIG_CACHE_FILE, globalDigResultCache, GLOBAL_DIG_CACHE_TTL, GLOBAL_DIG_CACHE_MAX);
  loadDiskCache(WHOIS_CACHE_FILE, globalWhoisResultCache, GLOBAL_WHOIS_CACHE_TTL, GLOBAL_WHOIS_CACHE_MAX);

  // Save caches to disk once on process exit instead of per-lookup. The
  // 'exit' handler fires synchronously regardless of how the process exits
  // (normal completion, signal, uncaught exception), so a separate signal
  // handler is redundant. We deliberately do NOT install SIGINT/SIGTERM
  // handlers here — nwss.js installs its own async ones that perform
  // browser/VPN cleanup, and a sync handler here would call process.exit(0)
  // first and skip that cleanup entirely.
  const flushCaches = () => {
    saveDiskCache(DIG_CACHE_FILE, globalDigResultCache, GLOBAL_DIG_CACHE_TTL, GLOBAL_DIG_CACHE_MAX);
    saveDiskCache(WHOIS_CACHE_FILE, globalWhoisResultCache, GLOBAL_WHOIS_CACHE_TTL, GLOBAL_WHOIS_CACHE_MAX);
  };
  process.on('exit', flushCaches);
}

/**
 * Strips ANSI color codes from a string for clean file logging
 * @param {string} text - Text that may contain ANSI codes
 * @returns {string} Text with ANSI codes removed
 */
function stripAnsiColors(text) {
  // Remove ANSI escape sequences (color codes)
  ANSI_REGEX.lastIndex = 0;
  return text.replace(ANSI_REGEX, '');
}

/**
 * Validates if whois command is available on the system
 * @returns {Object} Object with isAvailable boolean and version/error info
 */
function validateWhoisAvailability() {
  if (validateWhoisAvailability._cached) return validateWhoisAvailability._cached;
  try {
    const result = execSync('whois --version 2>&1', { encoding: 'utf8' });
    validateWhoisAvailability._cached = {
      isAvailable: true,
      version: result.trim()
    };
  } catch (error) {
    try {
      execSync('which whois', { encoding: 'utf8' });
      validateWhoisAvailability._cached = {
        isAvailable: true,
        version: 'whois (version unknown)'
      };
    } catch (e) {
      validateWhoisAvailability._cached = {
        isAvailable: false,
        error: 'whois command not found'
      };
    }
  }
  return validateWhoisAvailability._cached;
}

/**
 * Validates if dig command is available on the system
 * @returns {Object} Object with isAvailable boolean and version/error info
 */
function validateDigAvailability() {
  if (validateDigAvailability._cached) return validateDigAvailability._cached;
  try {
    const result = execSync('dig -v 2>&1', { encoding: 'utf8' });
    validateDigAvailability._cached = {
      isAvailable: true,
      version: result.split('\n')[0].trim()
    };
  } catch (error) {
    validateDigAvailability._cached = {
      isAvailable: false,
      error: 'dig command not found'
    };
  }
  return validateDigAvailability._cached;
}

/**
 * Executes a command with proper timeout handling
 * @param {string} command - Command to execute
 * @param {number} timeout - Timeout in milliseconds
 * @returns {Promise<Object>} Promise that resolves with stdout/stderr or rejects on timeout/error
 */
function execWithTimeout(command, timeout = 10000) {
  return new Promise((resolve, reject) => {
    const child = exec(command, { encoding: 'utf8' }, (error, stdout, stderr) => {
      if (timer) clearTimeout(timer);
      
      if (error) {
        reject(error);
      } else {
        resolve({ stdout, stderr });
      }
    });
    
    // Set up timeout
    const timer = setTimeout(() => {
      child.kill('SIGTERM');

      // Force kill after 2 seconds if SIGTERM doesn't work. unref() so this
      // tail timer doesn't keep the event loop alive past scan completion —
      // a dig that times out near the end of a scan would otherwise delay
      // exit by ~2 seconds.
      const killTimer = setTimeout(() => {
        if (!child.killed) {
          child.kill('SIGKILL');
        }
      }, 2000);
      killTimer.unref();

      reject(new Error(`Command timeout after ${timeout}ms: ${command}`));
    }, timeout);
    
    // Handle child process errors
    child.on('error', (err) => {
      if (timer) clearTimeout(timer);
      reject(err);
    });
  });
}

/**
 * Selects a whois server from the configuration
 * @param {string|Array<string>} whoisServer - Single server string or array of servers
 * @param {string} mode - Selection mode: 'random' (default) or 'cycle'
 * @returns {string|null} Selected whois server or null if none specified
 */
function selectWhoisServer(whoisServer = '', mode = 'random'){
  if (!whoisServer) {
    return null; // Use default whois behavior
  }
  
  if (typeof whoisServer === 'string') {
    return whoisServer;
  }
  
  if (Array.isArray(whoisServer) && whoisServer.length > 0) {
    if (mode === 'cycle') {
      const selectedServer = whoisServer[whoisServerCycleIndex % whoisServer.length];
      whoisServerCycleIndex = (whoisServerCycleIndex + 1) % whoisServer.length;
      
      return selectedServer;
    } else {
      // Random selection (default behavior)
      const randomIndex = Math.floor(Math.random() * whoisServer.length);
      return whoisServer[randomIndex];
    }
  }
  
  return null;
}

/**
 * Gets common whois servers for debugging/fallback suggestions
 * @returns {Array<string>} List of common whois servers
 */
function getCommonWhoisServers() {
  return [
    'whois.iana.org',
    'whois.internic.net', 
    'whois.verisign-grs.com',
    'whois.markmonitor.com',
    'whois.godaddy.com',
    'whois.namecheap.com',
    'whois.1and1.com'
  ];
}

/**
 * Suggests alternative whois servers based on domain TLD
 * @param {string} domain - Domain to get suggestions for
 * @param {string} failedServer - Server that failed (to exclude from suggestions)
 * @returns {Array<string>} Suggested whois servers
 */
function suggestWhoisServers(domain, failedServer = null) {
  const tld = domain.split('.').pop().toLowerCase();
  const suggestions = [];
  
  // TLD-specific servers
  const tldServers = {
    'com': ['whois.verisign-grs.com', 'whois.internic.net'],
    'net': ['whois.verisign-grs.com', 'whois.internic.net'],
    'org': ['whois.pir.org'],
    'info': ['whois.afilias.net'],
    'biz': ['whois.neulevel.biz'],
    'uk': ['whois.nominet.uk'],
    'de': ['whois.denic.de'],
    'fr': ['whois.afnic.fr'],
    'it': ['whois.nic.it'],
    'nl': ['whois.domain-registry.nl']
  };
  
  if (tldServers[tld]) {
    suggestions.push(...tldServers[tld]);
  }
  
  // Add common servers
  suggestions.push(...getCommonWhoisServers());
  
  // Remove duplicates and failed server
  const uniqueSuggestions = [...new Set(suggestions)];
  return failedServer ? uniqueSuggestions.filter(s => s !== failedServer) : uniqueSuggestions;
}

/**
 * Performs a whois lookup on a domain with proper timeout handling and custom server support (basic version)
 * @param {string} domain - Domain to lookup
 * @param {number} timeout - Timeout in milliseconds (default: 10000)
 * @param {string|Array<string>} whoisServer - Custom whois server(s) to use
 * @param {boolean} debugMode - Enable debug logging (default: false)
 * @returns {Promise<Object>} Object with success status and output/error
 */
async function whoisLookup(domain = '', timeout = 10000, whoisServer = '', debugMode = false, logFunc = null) {
  const startTime = Date.now();
  let cleanDomain, selectedServer, whoisCommand;
  
  try {
    // Clean domain (remove protocol, path, etc)
    cleanDomain = domain.replace(/^https?:\/\//, '').replace(/\/.*$/, '').replace(/:\d+$/, '');
    
    // Select whois server if provided
    selectedServer = selectWhoisServer(whoisServer);
    
    // Build whois command
    if (selectedServer) {
      // Use custom whois server with -h flag
      whoisCommand = `whois -h "${selectedServer}" -- "${cleanDomain}"`;
    } else {
      // Use default whois behavior
      whoisCommand = `whois -- "${cleanDomain}"`;
    }
       
    if (debugMode) {
      if (logFunc) {
        logFunc(`${messageColors.highlight('[whois]')} Starting lookup for ${cleanDomain} (timeout: ${timeout}ms)`);
        logFunc(`${messageColors.highlight('[whois]')} Command: ${whoisCommand}`);
      } else {
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Starting lookup for ${cleanDomain} (timeout: ${timeout}ms)`));
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Command: ${whoisCommand}`));
      }
    }
    
    const { stdout, stderr } = await execWithTimeout(whoisCommand, timeout);
    const duration = Date.now() - startTime;
    
    if (stderr && stderr.trim()) {
      if (debugMode) {
        if (logFunc) {
          logFunc(`${messageColors.highlight('[whois]')} Lookup failed for ${cleanDomain} after ${duration}ms`);
          logFunc(`${messageColors.highlight('[whois]')} Server: ${selectedServer || 'default'}`);
          logFunc(`${messageColors.highlight('[whois]')} Error: ${stderr.trim()}`);
          logFunc(`${messageColors.highlight('[whois]')} Command executed: ${whoisCommand}`);
        } else {
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Lookup failed for ${cleanDomain} after ${duration}ms`));
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Server: ${selectedServer || 'default'}`));
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Error: ${stderr.trim()}`));
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Command executed: ${whoisCommand}`));
        }
      if (selectedServer) {
          if (logFunc) {
            logFunc(`${messageColors.highlight('[whois]')} Custom server used: ${selectedServer}`);
          } else {
            console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Custom server used: ${selectedServer}`));
          }
        }
      }
      
      return {
        success: false,
        error: stderr.trim(),
        domain: cleanDomain,
        whoisServer: selectedServer,
        duration: duration,
        command: whoisCommand
      };
    }
    
    if (debugMode) {
      if (logFunc) {
        logFunc(`${messageColors.highlight('[whois]')} Lookup successful for ${cleanDomain} after ${duration}ms`);
        logFunc(`${messageColors.highlight('[whois]')} Server: ${selectedServer || 'default'}`);
        logFunc(`${messageColors.highlight('[whois]')} Output length: ${stdout.length} characters`);
      } else {
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Lookup successful for ${cleanDomain} after ${duration}ms`));
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Server: ${selectedServer || 'default'}`));
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Output length: ${stdout.length} characters`));
      }
    }
    
    return {
      success: true,
      output: stdout,
      domain: cleanDomain,
      whoisServer: selectedServer,
      duration: duration,
      command: whoisCommand
    };
  } catch (error) {
    const duration = Date.now() - startTime;
    const isTimeout = error.message.includes('timeout') || error.message.includes('Command timeout');
    const errorType = isTimeout ? 'timeout' : 'error';
    
    if (debugMode) {
      if (logFunc) {
        logFunc(`${messageColors.highlight('[whois]')} Lookup ${errorType} for ${cleanDomain || domain} after ${duration}ms`);
        logFunc(`${messageColors.highlight('[whois]')} Server: ${selectedServer || 'default'}`);
        logFunc(`${messageColors.highlight('[whois]')} Command: ${whoisCommand || 'command not built'}`);
        logFunc(`${messageColors.highlight('[whois]')} ${errorType === 'timeout' ? 'Timeout' : 'Error'}: ${error.message}`);
      } else {
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Lookup ${errorType} for ${cleanDomain || domain} after ${duration}ms`));
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Server: ${selectedServer || 'default'}`));
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Command: ${whoisCommand || 'command not built'}`));
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} ${errorType === 'timeout' ? 'Timeout' : 'Error'}: ${error.message}`));
      }
      
       if (selectedServer) {
        if (logFunc) {
          logFunc(`${messageColors.highlight('[whois]')} Failed server: ${selectedServer} (custom)`);
        } else {
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Failed server: ${selectedServer} (custom)`));
        }
      } else {
        if (logFunc) {
          logFunc(`${messageColors.highlight('[whois]')} Failed server: system default whois server`);
        } else {
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Failed server: system default whois server`));
        }
      }
      
      if (isTimeout) {
        if (logFunc) {
          logFunc(`${messageColors.highlight('[whois]')} Timeout exceeded ${timeout}ms limit`);
        } else {
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Timeout exceeded ${timeout}ms limit`));
        }
        if (selectedServer) {
          if (logFunc) {
            logFunc(`${messageColors.highlight('[whois]')} Consider using a different whois server or increasing timeout`);
          } else {
            console.log(formatLogMessage('debug', `${messageColors.highlight('[whois]')} Consider using a different whois server or increasing timeout`));
          }
        }
      }
    }
    
    return {
      success: false,
      error: error.message,
      domain: cleanDomain || domain,
      whoisServer: selectedServer,
      duration: duration,
      command: whoisCommand,
      isTimeout: isTimeout,
      errorType: errorType
    };
  }
}

/**
 * Performs a whois lookup with retry logic and fallback servers
 * @param {string} domain - Domain to lookup
 * @param {number} timeout - Timeout in milliseconds (default: 10000)
 * @param {string|Array<string>} whoisServer - Custom whois server(s) to use
 * @param {boolean} debugMode - Enable debug logging (default: false)
 * @param {Object} retryOptions - Retry configuration options
 * @param {number} whoisDelay - Delay in milliseconds before whois requests (default: 2000)
 * @returns {Promise<Object>} Object with success status and output/error
 */
async function whoisLookupWithRetry(domain = '', timeout = 10000, whoisServer = '', debugMode = false, retryOptions = {}, whoisDelay = 8000, logFunc = null) {
  const {
    maxRetries = 3,
    timeoutMultiplier = 1.5,
    useFallbackServers = true,
    retryOnTimeout = true,
    retryOnError = true
  } = retryOptions;

  let serversToTry = [];
  
  // Build list of servers to try
  if (whoisServer && whoisServer !== '') {
    if (Array.isArray(whoisServer)) {
      serversToTry = [...whoisServer]; // Copy array to avoid modifying original
    } else {
      serversToTry = [whoisServer];
    }
  } else {
    serversToTry = ['']; // Default server (empty string instead of null)
  }
  
  // Add fallback servers if enabled and we have custom servers
  if (useFallbackServers && whoisServer && whoisServer !== '') {
    const fallbacks = suggestWhoisServers(domain).slice(0, 3);
    // Only add fallbacks that aren't already in our list
    const existingServers = serversToTry.filter(s => s !== '');
        const existingServerCount = existingServers.length;
        const newFallbacks = fallbacks.filter(fb => {
          for (let i = 0; i < existingServerCount; i++) {
            if (existingServers[i] === fb) return false;
          }
          return true;
        });
    serversToTry.push(...newFallbacks);
  }
  
  let lastError = null;
  let totalAttempts = 0;
  let serversAttempted = [];
  
  if (debugMode) {
      const totalServers = serversToTry.length;
    if (logFunc) {
        logFunc(`${messageColors.highlight('[whois-retry]')} Starting whois lookup for ${domain} with ${totalServers} server(s) to try`);
      logFunc(`${messageColors.highlight('[whois-retry]')} Servers: [${serversToTry.map(s => s || 'default').join(', ')}]`);
      logFunc(`${messageColors.highlight('[whois-retry]')} Retry settings: maxRetries=${maxRetries} per server, timeoutMultiplier=${timeoutMultiplier}, retryOnTimeout=${retryOnTimeout}, retryOnError=${retryOnError}`);
    } else {
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Starting whois lookup for ${domain} with ${totalServers} server(s) to try`));
      console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Servers: [${serversToTry.map(s => s || 'default').join(', ')}]`));
      console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Retry settings: maxRetries=${maxRetries} per server, timeoutMultiplier=${timeoutMultiplier}, retryOnTimeout=${retryOnTimeout}, retryOnError=${retryOnError}`));
    }
  }
  
  // Try each server with retry logic
  const serverCount = serversToTry.length;
  for (let serverIndex = 0; serverIndex < serverCount; serverIndex++) {
    const server = serversToTry[serverIndex];
    let currentTimeout = timeout;
    let retryCount = 0;
    serversAttempted.push(server);
    
    if (debugMode) {
      const serverName = (server && server !== '') ? server : 'default';
      if (logFunc) {
        logFunc(`${messageColors.highlight('[whois-retry]')} Server ${serverIndex + 1}/${serverCount}: ${serverName} (max ${maxRetries} attempts)`);
      } else {
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Server ${serverIndex + 1}/${serverCount}: ${serverName} (max ${maxRetries} attempts)`));
      }
    }
    
    // Retry this server up to maxRetries times
    while (retryCount < maxRetries) {
      totalAttempts++;
      const attemptNum = retryCount + 1;
      
      if (debugMode) {
        const serverName = (server && server !== '') ? server : 'default';
        if (logFunc) {
          logFunc(`${messageColors.highlight('[whois-retry]')} Attempt ${attemptNum}/${maxRetries} on server ${serverName} (timeout: ${currentTimeout}ms)`);
        } else {
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Attempt ${attemptNum}/${maxRetries} on server ${serverName} (timeout: ${currentTimeout}ms)`));
        }
      }
      
      // Add progressive delay between retries (but not before first attempt on any server)
      if (retryCount > 0 && whoisDelay > 0) {
        // Progressive delay: base delay * retry attempt number + extra delay
        // Attempt 2: base delay * 1 + 4000ms = 8000ms + 4000ms = 12000ms
        // Attempt 3: base delay * 2 + 6000ms = 16000ms + 6000ms = 22000ms
        // Attempt 4+: base delay * 3 + 6000ms = 24000ms + 6000ms = 30000ms (if maxRetries > 3)
        const delayMultiplier = Math.min(retryCount, 3);
        const baseDelay = whoisDelay * delayMultiplier;
        
        // Add extra delay based on retry attempt
        let extraDelay = 0;
        if (retryCount === 1) {
          extraDelay = 4000; // Extra 4 seconds for 2nd attempt
        } else if (retryCount >= 2) {
          extraDelay = 6000; // Extra 6 seconds for 3rd+ attempts
        }
        
        const actualDelay = baseDelay + extraDelay;
        
        if (debugMode) {
          if (logFunc) {
            logFunc(`${messageColors.highlight('[whois-retry]')} Adding ${actualDelay}ms progressive delay before retry ${retryCount + 1} (base: ${baseDelay}ms + extra: ${extraDelay}ms)...`);
          } else {
            console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Adding ${actualDelay}ms progressive delay before retry ${retryCount + 1} (base: ${baseDelay}ms + extra: ${extraDelay}ms)...`));
          }
        }
        await new Promise(resolve => setTimeout(resolve, actualDelay));
      } else if (serverIndex > 0 && retryCount === 0 && whoisDelay > 0) {
        // Add delay before trying a new server (but not the very first server)
        if (debugMode) {
          if (logFunc) {
            logFunc(`${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before trying new server...`);
          } else {
            console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Adding ${whoisDelay}ms delay before trying new server...`));
          }
        }
        await new Promise(resolve => setTimeout(resolve, whoisDelay));
      } else if (debugMode && whoisDelay === 0) {
        // Log when delay is skipped due to whoisDelay being 0
        if (logFunc) {
          logFunc(`${messageColors.highlight('[whois-retry]')} Skipping delay (whoisDelay: ${whoisDelay}ms)`);
        } else {
          console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Skipping delay (whoisDelay: ${whoisDelay}ms)`));
        }
      }
      
      try {
        const result = await whoisLookup(domain, currentTimeout, server || '', debugMode, logFunc);
        
        if (result.success) {
          if (debugMode) {
            if (logFunc) {
              logFunc(`${messageColors.highlight('[whois-retry]')} SUCCESS on attempt ${attemptNum}/${maxRetries} for server ${result.whoisServer || 'default'} (total attempts: ${totalAttempts})`);
            } else {
              console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} SUCCESS on attempt ${attemptNum}/${maxRetries} for server ${result.whoisServer || 'default'} (total attempts: ${totalAttempts})`));
            }
          }
          
          // Add retry info to result
          // V8 Optimized: Object.assign performs better than spread
          return Object.assign({}, result, {
            retryInfo: {
              totalAttempts: totalAttempts,
              maxAttempts: serverCount * maxRetries,
              serversAttempted: serversAttempted,
              finalServer: result.whoisServer,
              retriedAfterFailure: totalAttempts > 1,
              serverRetries: retryCount,
              serverIndex: serverIndex
            }
          });
        }
        
        // Determine if we should retry based on error type
        const shouldRetry = (result.isTimeout && retryOnTimeout) || (!result.isTimeout && retryOnError);
        
        if (debugMode) {
          const serverName = (result.whoisServer && result.whoisServer !== '') ? result.whoisServer : 'default';
          const errorType = result.isTimeout ? 'TIMEOUT' : 'ERROR';
          if (logFunc) {
            logFunc(`${messageColors.highlight('[whois-retry]')} ${errorType} on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${result.error}`);
          } else {
            console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} ${errorType} on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${result.error}`));
          }
          
          if (retryCount < maxRetries - 1) {
            if (shouldRetry) {
              if (logFunc) {
                logFunc(`${messageColors.highlight('[whois-retry]')} Will retry attempt ${attemptNum + 1}/${maxRetries} on same server...`);
              } else {
                console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Will retry attempt ${attemptNum + 1}/${maxRetries} on same server...`));
              }
            } else {
              if (logFunc) {
                logFunc(`${messageColors.highlight('[whois-retry]')} Skipping retry on same server (retryOn${result.isTimeout ? 'Timeout' : 'Error'}=${shouldRetry})`);
              } else {
                console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Skipping retry on same server (retryOn${result.isTimeout ? 'Timeout' : 'Error'}=${shouldRetry})`));
              }
            }
          } else if (serverIndex < serverCount - 1) {
            if (logFunc) {
              logFunc(`${messageColors.highlight('[whois-retry]')} Max retries reached for server${serverIndex < serverCount - 1 ? ', will try next server...' : ', no more servers to try'}`);
            } else {
              console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Max retries reached for server${serverIndex < serverCount - 1 ? ', will try next server...' : ', no more servers to try'}`));            }
          }
        }
        
        lastError = result;
        
        // If this is the last retry for this server or we shouldn't retry this error type, break to next server
        if (retryCount >= maxRetries - 1 || !shouldRetry) {
          break;
        }
        
        // Increase timeout for next retry attempt on same server
        retryCount++;
        currentTimeout = Math.round(currentTimeout * timeoutMultiplier);
        
      } catch (error) {
        if (debugMode) {
          const serverName = (server && server !== '') ? server : 'default'
          if (logFunc) {
            logFunc(`${messageColors.highlight('[whois-retry]')} EXCEPTION on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${error.message}`);
          } else {
            console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} EXCEPTION on attempt ${attemptNum}/${maxRetries} with server ${serverName}: ${error.message}`));
          }
        }
        
        lastError = {
          success: false,
          error: error.message,
          domain: domain,
          whoisServer: server || '',
          isTimeout: error.message.includes('timeout'),
          duration: 0
        };
        
        // For exceptions, only retry if it's a retryable error type
        const isRetryableException = error.message.includes('timeout') || 
                                   error.message.includes('ECONNRESET') || 
                                   error.message.includes('ENOTFOUND');
        
        if (retryCount >= maxRetries - 1 || !isRetryableException) {
          break;
        }
        
        retryCount++;
        currentTimeout = Math.round(currentTimeout * timeoutMultiplier);
      }
    }
  }
  
  // All attempts failed
  if (debugMode) {
    const attemptedServerCount = serversAttempted.length;
    if (logFunc) {
      logFunc(`${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${totalAttempts} attempts failed for ${domain} across ${attemptedServerCount} server(s)`);
    } else {
      console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} FINAL FAILURE: All ${totalAttempts} attempts failed for ${domain} across ${attemptedServerCount} server(s)`));
    }
    if (lastError) {
      if (logFunc) {
        logFunc(`${messageColors.highlight('[whois-retry]')} Last error: ${lastError.error} (${lastError.isTimeout ? 'timeout' : 'error'})`);
      } else {
        console.log(formatLogMessage('debug', `${messageColors.highlight('[whois-retry]')} Last error: ${lastError.error} (${lastError.isTimeout ? 'timeout' : 'error'})`));
      }
    }
  }
  
  // Return the last error with retry info
  // V8 Optimized: Object.assign instead of spread operator
  return Object.assign({}, lastError, {
    retryInfo: {
      totalAttempts: totalAttempts,
      maxAttempts: serverCount * maxRetries,
      serversAttempted: serversAttempted,
      finalServer: lastError?.whoisServer || '',
      retriedAfterFailure: totalAttempts > 1,
      allAttemptsFailed: true
    }
  });
}

/**
 * Performs a dig lookup on a domain with proper timeout handling
 * @param {string} domain - Domain to lookup
 * @param {string} recordType - DNS record type (A, AAAA, MX, TXT, etc.) default: 'A'
 * @param {number} timeout - Timeout in milliseconds (default: 5000)
 * @returns {Promise<Object>} Object with success status and output/error
 */
async function digLookup(domain = '', recordType = 'A', timeout = 5000) {
  try {
    // Clean domain
    const cleanDomain = domain.replace(/^https?:\/\//, '').replace(/\/.*$/, '').replace(/:\d+$/, '');
    
    // Single dig command — full output contains everything including the short answers
    const { stdout: fullOutput, stderr } = await execWithTimeout(`dig "${cleanDomain}" ${recordType}`, timeout);
    
    if (stderr && stderr.trim()) {
      return {
        success: false,
        error: stderr.trim(),
        domain: cleanDomain,
        recordType
      };
    }
    
    // Extract short output from ANSWER SECTION of full dig output
    const answerMatch = fullOutput.match(/;; ANSWER SECTION:\n([\s\S]*?)(?:\n;;|\n*$)/);
    let shortOutput = '';
    if (answerMatch) {
      shortOutput = answerMatch[1]
        .split('\n')
        .map(line => line.split(/\s+/).pop())
        .filter(Boolean)
        .join('\n');
    }
    
    return {
      success: true,
      output: fullOutput,
      shortOutput,
      domain: cleanDomain,
      recordType
    };
  } catch (error) {
    return {
      success: false,
      error: error.message,
      domain: domain,
      recordType
    };
  }
}

/**
 * Checks if whois output contains all specified search terms (AND logic)
 * @param {string} whoisOutput - The whois lookup output
 * @param {Array<string>} searchTerms - Array of terms that must all be present
 * @returns {boolean} True if all terms are found
 */
function checkWhoisTerms(whoisOutput, searchTerms) {
  if (!searchTerms || !Array.isArray(searchTerms) || searchTerms.length === 0) {
    return false;
  }
  
  const lowerOutput = whoisOutput.toLowerCase();
  return searchTerms.every(term => lowerOutput.includes(term.toLowerCase()));
}

/**
 * Checks if whois output contains any of the specified search terms (OR logic)
 * @param {string} whoisOutput - The whois lookup output
 * @param {Array<string>} searchTerms - Array of terms where at least one must be present
 * @returns {boolean} True if any term is found
 */
function checkWhoisTermsOr(whoisOutput, searchTerms) {
  if (!searchTerms || !Array.isArray(searchTerms) || searchTerms.length === 0) {
    return false;
  }
  
  const lowerOutput = whoisOutput.toLowerCase();
  return searchTerms.some(term => lowerOutput.includes(term.toLowerCase()));
}

/**
 * Checks if dig output contains all specified search terms (AND logic)
 * @param {string} digOutput - The dig lookup output
 * @param {Array<string>} searchTerms - Array of terms that must all be present
 * @returns {boolean} True if all terms are found
 */
function checkDigTerms(digOutput, searchTerms) {
  if (!searchTerms || !Array.isArray(searchTerms) || searchTerms.length === 0) {
    return false;
  }
  
  const lowerOutput = digOutput.toLowerCase();
  return searchTerms.every(term => lowerOutput.includes(term.toLowerCase()));
}

/**
 * Checks if dig output contains any of the specified search terms (OR logic)
 * @param {string} digOutput - The dig lookup output
 * @param {Array<string>} searchTerms - Array of terms where at least one must be present
 * @returns {boolean} True if any term is found
 */
function checkDigTermsOr(digOutput, searchTerms) {
  if (!searchTerms || !Array.isArray(searchTerms) || searchTerms.length === 0) {
    return false;
  }
  
  const lowerOutput = digOutput.toLowerCase();
  return searchTerms.some(term => lowerOutput.includes(term.toLowerCase()));
}

/**
 * Enhanced dry run callback factory for better nettools reporting
 * @param {Map} matchedDomains - The matched domains collection
 * @param {boolean} forceDebug - Debug logging flag
 * @returns {Function} Enhanced dry run callback
 */
function createEnhancedDryRunCallback(matchedDomains, forceDebug) {
  return (domain, tool, matchType, matchedTerm, details, additionalInfo = {}) => {
    const result = { 
      domain, 
      tool, 
      matchType, 
      matchedTerm, 
      details, 
      ...additionalInfo 
    };
    
    matchedDomains.get('dryRunNetTools').push(result);
    
    if (forceDebug) {
      const serverInfo = additionalInfo.server ? ` (server: ${additionalInfo.server})` : '';
      const timingInfo = additionalInfo.duration ? ` [${additionalInfo.duration}ms]` : '';
      console.log(formatLogMessage('debug', `[DRY RUN] NetTools match: ${domain} via ${tool.toUpperCase()} (${matchType})${serverInfo}${timingInfo}`));
    }
  };
}

/**
 * Creates a handler for network tools checks with enhanced error handling
 * @param {Object} config - Configuration object
 * @returns {Function} Async function that handles network tool lookups
 */
function createNetToolsHandler(config) {
  const {
    whoisTerms,
    whoisOrTerms,
    whoisDelay = 4000,
    whoisServer,
    whoisServerMode = 'random',
    debugLogFile = null,
    digTerms,
    digOrTerms,
    digRecordType = 'A',
    digSubdomain = false,
    dryRunCallback = null,
    matchedDomains,
    addMatchedDomain,
    isDomainAlreadyDetected,
    getRootDomain,
    siteConfig,
    processedWhoisDomains = new Set(), // Accept global sets, fallback to new for backward compatibility
    processedDigDomains = new Set(),
    dumpUrls,
    matchedUrlsLogFile,
    forceDebug,
    fs
  } = config;
  
  const hasWhois = whoisTerms && Array.isArray(whoisTerms) && whoisTerms.length > 0;
  const hasWhoisOr = whoisOrTerms && Array.isArray(whoisOrTerms) && whoisOrTerms.length > 0;
  const hasDig = digTerms && Array.isArray(digTerms) && digTerms.length > 0;
  const hasDigOr = digOrTerms && Array.isArray(digOrTerms) && digOrTerms.length > 0;

  // Pre-lowercase search terms once per handler so the per-domain check loop
  // doesn't re-lowercase the same constants for every output it scans.
  const whoisTermsLower   = hasWhois   ? whoisTerms.map(t => t.toLowerCase())   : null;
  const whoisOrTermsLower = hasWhoisOr ? whoisOrTerms.map(t => t.toLowerCase()) : null;
  const digTermsLower     = hasDig     ? digTerms.map(t => t.toLowerCase())     : null;
  const digOrTermsLower   = hasDigOr   ? digOrTerms.map(t => t.toLowerCase())   : null;

  // Hoisted out of handleNetToolsCheck so the closure is constructed once per
  // handler rather than once per invocation. References forceDebug, debugLogFile,
  // and fs from the destructured config above.
  function logToConsoleAndFile(message) {
    if (forceDebug) {
      console.log(formatLogMessage('debug', message));
    }
    if (debugLogFile && fs) {
      try {
        const timestamp = new Date().toISOString();
        const cleanMessage = stripAnsiColors(message);
        fs.appendFileSync(debugLogFile, `${timestamp} [debug nettools] ${cleanMessage}\n`);
      } catch (_) {
        // Silently fail file logging to avoid disrupting whois operations
      }
    }
  }
  
  // Create config-aware cache keys for deduplication
  // Whois: Only include search terms + server (domain registry data is consistent across subdomains)
  const whoisConfigKey = JSON.stringify({
    terms: whoisTerms || [],
    orTerms: whoisOrTerms || [],
    server: whoisServer || 'default',
    serverMode: whoisServerMode || 'random'
  });
  // Dig: Include all config (DNS records can vary by specific subdomain)
  const digConfigKey = JSON.stringify({
    terms: digTerms || [],
    orTerms: digOrTerms || [],
    recordType: digRecordType,
    subdomain: digSubdomain
  });

  // Whois cache is global (globalWhoisResultCache) — shared across all handler instances
  // Whois data is per root domain and doesn't change based on search terms
  // Dig cache is global (globalDigResultCache) — shared across all handler instances
  // DNS results are the same regardless of search terms
  
  return async function handleNetToolsCheck(domain, fullSubdomain) {
    const originalDomain = fullSubdomain;

    // Check if domain was already detected (skip expensive operations)
    if (typeof isDomainAlreadyDetected === 'function' && isDomainAlreadyDetected(fullSubdomain)) {
      if (forceDebug) {
        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping already detected subdomain: ${fullSubdomain} (output domain: ${domain})`);
      }
      return;
    }

    // Determine which domain will be used for dig lookup
    const digDomain = digSubdomain && originalDomain ? originalDomain : domain;

    // For whois: use root domain only (whois data is consistent for entire domain)
    const whoisRootDomain = getRootDomain ? getRootDomain(`http://${domain}`) : domain;
    
    // Check if we need to perform any lookups with appropriate deduplication
    // Whois: root domain + config (whois data same for sub.example.com and example.com)
    const whoisDedupeKey = `${whoisRootDomain}:${whoisConfigKey}`;
    // Dig: specific subdomain + config (DNS records can differ between subdomains)
    const digDedupeKey = `${digDomain}:${digConfigKey}`;
    const needsWhoisLookup = (hasWhois || hasWhoisOr) && !processedWhoisDomains.has(whoisDedupeKey);
    const needsDigLookup = (hasDig || hasDigOr) && !processedDigDomains.has(digDedupeKey);
    
    // Skip if we don't need to perform any lookups
    if (!needsWhoisLookup && !needsDigLookup) {
      if (forceDebug) {
        const whoisSkipped = (hasWhois || hasWhoisOr) ? `cached(${whoisRootDomain})` : 'n/a';
        const digSkipped = (hasDig || hasDigOr) ? `cached(${digDomain})` : 'n/a';
        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Skipping duplicate lookups for ${domain} (whois: ${whoisSkipped}, dig: ${digSkipped})`);
      }
      return;
    }
    

    if (forceDebug) {
      const totalProcessed = processedWhoisDomains.size + processedDigDomains.size;
      logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Processing domain: ${domain} (whois: ${needsWhoisLookup ? whoisRootDomain : 'skip'}, dig: ${needsDigLookup ? digDomain : 'skip'}) (${totalProcessed} total processed)`);
    }

      // Log site-specific whois delay if different from default
      if (forceDebug && whoisDelay !== 3000) {
        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Using site-specific whois delay: ${whoisDelay}ms`);
      }

    // Wrap entire function in timeout protection (single timer)
    let overallTimeoutId;
    return Promise.race([
      (async () => {
        try { return await executeNetToolsLookup(); }
        finally { clearTimeout(overallTimeoutId); }
      })(),
      new Promise((_, reject) => {
        overallTimeoutId = setTimeout(() => reject(new Error('NetTools overall timeout')), 65000);
      })
    ]).catch(err => {
      if (forceDebug) {
        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} ${err.message} for ${domain}, continuing...`);
      }
    });
    
    async function executeNetToolsLookup() {
    
    try {
      let whoisMatched = false;
      let whoisOrMatched = false;
      let digMatched = false;
      let digOrMatched = false;
      
      // Debug logging for digSubdomain logic
      if (forceDebug) {
        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} digSubdomain setting: ${digSubdomain}`);
        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} domain parameter: ${domain}`);
        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} originalDomain parameter: ${originalDomain}`);
        logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Final digDomain will be: ${digDomain}`);
        if (whoisServer) {
          const serverInfo = Array.isArray(whoisServer) 
            ? `randomized from [${whoisServer.join(', ')}]` 
            : whoisServer;
          logToConsoleAndFile(`${messageColors.highlight('[nettools]')} Custom whois server: ${serverInfo}`);
        }
      }
      
      // Enhanced dry run logging
      if (dryRunCallback && forceDebug) {
        logToConsoleAndFile(`${messageColors.highlight('[nettools-dryrun]')} Processing ${domain} (original: ${originalDomain})`);

        // Show what checks will be performed
        const checksToPerform = [];
        if (hasWhois) checksToPerform.push('whois-and');
        if (hasWhoisOr) checksToPerform.push('whois-or');
        if (hasDig) checksToPerform.push('dig-and');
        if (hasDigOr) checksToPerform.push('dig-or');
        logToConsoleAndFile(`${messageColors.highlight('[nettools-dryrun]')} Will perform: ${checksToPerform.join(', ')}`);
        
        // Show which domain will be used for dig
        if (hasDig || hasDigOr) {
          logToConsoleAndFile(`${messageColors.highlight('[dig-dryrun]')} Will check ${digDomain} (${digSubdomain ? 'subdomain mode' : 'root domain mode'})`);
        }
        
        // Show whois server selection
        if (hasWhois || hasWhoisOr) {
          const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
          const serverInfo = selectedServer ? selectedServer : 'system default';
          logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Will use server: ${serverInfo}`);
        }
        
        // Show retry configuration in dry-run
        if (hasWhois || hasWhoisOr) {
          const maxRetries = siteConfig.whois_max_retries || 2;
          logToConsoleAndFile(`${messageColors.highlight('[whois-dryrun]')} Max retries: ${maxRetries}, timeout multiplier: ${siteConfig.whois_timeout_multiplier || 1.5}`);
        }
      }
      
      // Perform whois lookup if either whois or whois-or is configured
      if (needsWhoisLookup) {
        // Mark whois root domain+config as being processed
        processedWhoisDomains.add(whoisDedupeKey);
        
        // Check whois cache first - cache key includes server for accuracy
        const selectedServer = selectWhoisServer(whoisServer, whoisServerMode);
        const whoisCacheKey = `${whoisRootDomain}-${(selectedServer && selectedServer !== '') ? selectedServer : 'default'}`;
        const now = Date.now();
        let whoisResult = null;
        
        if (globalWhoisResultCache.has(whoisCacheKey)) {
          const cachedEntry = globalWhoisResultCache.get(whoisCacheKey);
          if (now - cachedEntry.timestamp < GLOBAL_WHOIS_CACHE_TTL) {
            if (forceDebug) {
              const age = Math.round((now - cachedEntry.timestamp) / 1000);
              const serverInfo = (selectedServer && selectedServer !== '') ? ` (server: ${selectedServer})` : ' (default server)';
              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Using cached result for ${whoisRootDomain}${serverInfo} [age: ${age}s]`);
            }
            // V8 Optimized: Object.assign is faster than spread for object merging
            whoisResult = Object.assign({}, cachedEntry.result, {
              fromCache: true,
              cacheAge: now - cachedEntry.timestamp,
              originalTimestamp: cachedEntry.timestamp
            });
            dnsCacheStats.whoisHits++;
          } else {
            // Cache expired, remove it
            globalWhoisResultCache.delete(whoisCacheKey);
            if (forceDebug) {
              logToConsoleAndFile(`${messageColors.highlight('[whois-cache]')} Cache expired for ${whoisRootDomain}, performing fresh lookup`);
            }
          }
        }
        
        // Perform fresh lookup if not cached
        if (!whoisResult) {
          // Deduplicate concurrent lookups — wait for in-flight request instead of starting a new one
          if (pendingWhoisLookups.has(whoisCacheKey)) {
            whoisResult = await pendingWhoisLookups.get(whoisCacheKey);
          } else {
            if (forceDebug) {
              const serverInfo = (selectedServer && selectedServer !== '') ? ` using server ${selectedServer}` : ' using default server';
              logToConsoleAndFile(`${messageColors.highlight('[whois]')} Performing fresh whois lookup for ${whoisRootDomain}${serverInfo}`);
            }

            // Configure retry options based on site config or use defaults
            const retryOptions = {
              maxRetries: siteConfig.whois_max_retries || 3,
              timeoutMultiplier: siteConfig.whois_timeout_multiplier || 1.5,
              useFallbackServers: siteConfig.whois_use_fallback !== false, // Default true
              retryOnTimeout: siteConfig.whois_retry_on_timeout !== false, // Default true
              retryOnError: siteConfig.whois_retry_on_error === true // Default false
            };

            try {
              const lookupPromise = whoisLookupWithRetry(whoisRootDomain, 8000, whoisServer, forceDebug, retryOptions, whoisDelay, logToConsoleAndFile);
              pendingWhoisLookups.set(whoisCacheKey, lookupPromise);
              // try/finally so a rejected lookup still clears the pending
              // entry — see matching comment on pendingDigLookups below.
              try {
                whoisResult = await lookupPromise;
              } finally {
                pendingWhoisLookups.delete(whoisCacheKey);
              }

              // Cache successful results (and certain types of failures)
              if (whoisResult.success ||
                  (whoisResult.error && !whoisResult.isTimeout &&
                   !whoisResult.error.toLowerCase().includes('connection') &&
                   !whoisResult.error.toLowerCase().includes('network'))) {

                globalWhoisResultCache.set(whoisCacheKey, {
                  result: whoisResult,
                timestamp: now
              });
              dnsCacheStats.whoisMisses++;
              dnsCacheStats.freshW