UNPKG

loggregate

Version:

A tool for aggregating and analyzing EVM blockchain events

905 lines (802 loc) 28.1 kB
/** * Core scanner module for logtui * Handles connecting to Hypersync and displaying the TUI */ import { keccak256, toHex, decodeEventLog, parseAbi, toEventSignature, } from "viem"; import { HypersyncClient, Decoder, LogField, JoinMode, } from "@envio-dev/hypersync-client"; import blessed from "blessed"; import contrib from "blessed-contrib"; import chalk from "chalk"; import figlet from "figlet"; import supportsColor from "supports-color"; let onlyOnce = true; const logOnlyOnce = (toLogs) => { if (onlyOnce) { console.log("only-once"); toLogs.forEach((toLog) => { console.log(toLog); }); onlyOnce = false; } }; // Force terminal compatibility mode - stronger settings // process.env.FORCE_COLOR = "3"; // Force full true color support - REMOVING THIS process.env.NCURSES_NO_UTF8_ACS = "1"; process.env.TERM = "xterm-256color"; // Use more compatible terminal type // Ensure chalk uses normal level for auto-detection // chalk.level = 3; // REMOVING FORCED LEVEL // Detect actual terminal color support const hasColorSupport = !!supportsColor.stdout; const has256ColorSupport = !!( supportsColor.stdout && supportsColor.stdout.has256 ); const hasTrueColorSupport = !!( supportsColor.stdout && supportsColor.stdout.has16m ); // Log color support detection for verbose mode if (process.env.DEBUG) { console.log(`Terminal color support detected: - Basic colors: ${hasColorSupport} - 256 colors: ${has256ColorSupport} - True colors: ${hasTrueColorSupport} `); } // Apply completely silent error handling for Blessed/Terminal issues const originalConsoleError = console.error; console.error = function (...args) { // Check if this is a terminal capability error if (args.length > 0 && typeof args[0] === "string") { const errorMsg = args[0]; if ( errorMsg.includes("Error on xterm") || errorMsg.includes("Setulc") || errorMsg.includes("stack") || errorMsg.includes("term") || errorMsg.includes("escape sequence") ) { return; // Silently ignore these errors } } originalConsoleError.apply(console, args); }; // Apply monkey patch to process.stderr.write to catch any remaining errors const originalStderrWrite = process.stderr.write; process.stderr.write = function (buffer, encoding, fd) { const str = buffer.toString(); if ( str.includes("Error on xterm") || str.includes("Setulc") || str.includes("stack") || str.includes("var v") || str.includes("terminal capability") || str.includes("xterm-256color") || str.toLowerCase().includes("setulc") ) { return true; // Pretend we wrote it but don't actually write } return originalStderrWrite.apply(process.stderr, arguments); }; /** * Format numbers with commas * @param {number|string} num - Number to format * @returns {string} Formatted number */ const formatNumber = (num) => { if (num === null || num === undefined) return "0"; return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); }; /** * Safe JSON stringify that handles circular references * @param {Object} obj - Object to stringify * @param {number} maxLength - Maximum length before truncating * @returns {string} Stringified object */ const safeStringify = (obj, maxLength = 100) => { try { if (!obj) return "null"; const str = JSON.stringify(obj); if (str.length <= maxLength) return str; return str.substring(0, maxLength) + "..."; } catch (err) { return `[Object: stringify failed]`; } }; /** * Create and run the scanner with TUI * @param {Object} options - Scanner options * @param {string} options.networkUrl - Hypersync network URL * @param {string} options.eventSignature - Event signature to scan for * @param {string} options.contractAddress - Contract address to monitor (optional) * @param {string} options.title - Title for the TUI * @param {number} options.decimals - Number of decimals for formatting * @param {number} options.fromBlock - Starting block number (default: 0) * @returns {Promise<void>} */ export async function createScanner({ networkUrl, eventSignatureHuman, param, contractAddress, title = "Event Scanner", decimals = 0, fromBlock = 0, }) { const eventSignature = toEventSignature(eventSignatureHuman); // Initialize Hypersync client const client = HypersyncClient.new({ url: networkUrl, bearerToken: process.env.HYPERSYNC_BEARER_TOKEN || "35aca157-42fa-413f-a3cd-fb0261b21ee2", // Default token if not provided }); // Create topic0 hash from event signature const topic0 = keccak256(toHex(eventSignature)); // Define the Hypersync query to get events we're interested in let query = { fromBlock: fromBlock, logs: [ { // Get all events that match the topic0 value we want topics: [[topic0]], // Add contract address filter if provided ...(contractAddress && { address: [contractAddress] }), }, ], fieldSelection: { log: [LogField.Topic0, LogField.Topic1, LogField.Topic2, LogField.Data], }, joinMode: JoinMode.JoinTransactions, }; // Track event counts - will be populated dynamically const eventAggregates = { count: 0, Unknown: 0, sum: BigInt(0), sumSquares: BigInt(0), // Add sum of squares for standard deviation min: BigInt(0), max: BigInt(0), average: BigInt(0), variance: BigInt(0), // Add variance field stdDev: BigInt(0), // Add standard deviation field median: BigInt(0), // Add median field // Efficient median tracking using two heaps maxHeap: [], // Contains the smaller half of values minHeap: [], // Contains the larger half of values }; // Get event name from signature const eventName = eventSignature.split("(")[0]; eventAggregates[eventName] = 0; // Create a mapping of topic0 hash to event name const topic0ToName = { [topic0]: eventName, }; // Efficient heap operations for median calculation const heapify = (heap, compare) => { for (let i = Math.floor(heap.length / 2) - 1; i >= 0; i--) { heapifyDown(heap, i, compare); } }; const heapifyDown = (heap, index, compare) => { while (true) { let largest = index; const left = 2 * index + 1; const right = 2 * index + 2; if (left < heap.length && compare(heap[left], heap[largest]) > 0) { largest = left; } if (right < heap.length && compare(heap[right], heap[largest]) > 0) { largest = right; } if (largest === index) break; [heap[index], heap[largest]] = [heap[largest], heap[index]]; index = largest; } }; const heapifyUp = (heap, index, compare) => { while (index > 0) { const parent = Math.floor((index - 1) / 2); if (compare(heap[index], heap[parent]) <= 0) break; [heap[index], heap[parent]] = [heap[parent], heap[index]]; index = parent; } }; const insertToHeap = (heap, value, compare) => { heap.push(value); heapifyUp(heap, heap.length - 1, compare); }; const extractFromHeap = (heap, compare) => { if (heap.length === 0) return null; const result = heap[0]; heap[0] = heap[heap.length - 1]; heap.pop(); if (heap.length > 0) { heapifyDown(heap, 0, compare); } return result; }; const peekHeap = (heap) => { return heap.length > 0 ? heap[0] : null; }; // Efficient median calculation using two heaps const updateMedian = (value) => { const { maxHeap, minHeap } = eventAggregates; // Compare function for max heap (larger values have higher priority) const maxHeapCompare = (a, b) => { if (a < b) return -1; if (a > b) return 1; return 0; }; // Compare function for min heap (smaller values have higher priority) const minHeapCompare = (a, b) => { if (a > b) return -1; if (a < b) return 1; return 0; }; // Insert the new value if (maxHeap.length === 0 || value <= peekHeap(maxHeap)) { insertToHeap(maxHeap, value, maxHeapCompare); } else { insertToHeap(minHeap, value, minHeapCompare); } // Rebalance heaps to maintain the median property if (maxHeap.length > minHeap.length + 1) { const moved = extractFromHeap(maxHeap, maxHeapCompare); insertToHeap(minHeap, moved, minHeapCompare); } else if (minHeap.length > maxHeap.length) { const moved = extractFromHeap(minHeap, minHeapCompare); insertToHeap(maxHeap, moved, maxHeapCompare); } // Calculate median if (maxHeap.length > minHeap.length) { eventAggregates.median = peekHeap(maxHeap); } else { const left = peekHeap(maxHeap); const right = peekHeap(minHeap); eventAggregates.median = (left + right) / BigInt(2); } }; // Function to format BigInt with commas and decimals const formatBigInt = (value) => { try { // Convert to string and handle decimals let strValue = value.toString(); if (decimals > 0) { // Pad with zeros if needed while (strValue.length <= decimals) { strValue = "0" + strValue; } // Only keep the whole number part strValue = strValue.slice(0, -decimals); } // Add commas for thousands return strValue.replace(/\B(?=(\d{3})+(?!\d))/g, ","); } catch (error) { console.error("Error formatting BigInt:", error); return value.toString(); } }; //============================================================================= // TUI SETUP //============================================================================= // Create blessed screen with improved compatibility settings const screen = blessed.screen({ smartCSR: true, title, dockBorders: true, fullUnicode: true, forceUnicode: true, autoPadding: true, terminal: "xterm-color", // Use simpler terminal type fastCSR: true, useBCE: true, // Use Background Color Erase for better rendering }); // Define UI color scheme with multiple fallback options tailored to detected capabilities // A selection of beautiful baby blue options const colorOptions = { trueColor: { primary: "#FFFFFF", // Pure white for logo secondary: "#00BFFF", // Keep baby blue for borders and other elements tertiary: "#87CEFA", // Light sky blue as another option }, ansi256: { // These are the closest ANSI 256 color codes primary: 15, // White secondary: 39, // Closest to deep sky blue tertiary: 45, // A lighter baby blue }, basic: { // When only basic ANSI colors are available primary: "white", // Simple white secondary: "cyanBright", // For non-logo elements fallback: "cyan", }, }; // Select the best color scheme based on terminal capabilities const getOptimalColorScheme = () => { if (hasTrueColorSupport) { return { type: "hex", primary: colorOptions.trueColor.primary, secondary: colorOptions.trueColor.secondary, }; } else if (has256ColorSupport) { return { type: "ansi256", primary: colorOptions.ansi256.primary, secondary: colorOptions.ansi256.secondary, }; } else { return { type: "basic", primary: colorOptions.basic.primary, secondary: colorOptions.basic.secondary, }; } }; // Get the best color scheme for this terminal const colorScheme = getOptimalColorScheme(); // Set the active color for border references - still using baby blue for borders let uiColor = colorScheme.type === "hex" ? colorOptions.trueColor.secondary : colorOptions.trueColor.secondary; // Create enhanced color function with multiple fallbacks based on terminal capabilities const safeHexColor = (text) => { try { if (colorScheme.type === "hex") { return chalk.hex(colorScheme.primary)(text); } else if (colorScheme.type === "ansi256") { return chalk.ansi256(colorScheme.primary)(text); } else { // Basic color mode return chalk[colorScheme.primary](text); } } catch (e) { // Ultimate fallback - use white which should be available everywhere return chalk.white(text); } }; // Create specific color function for borders and other non-logo elements const borderColor = (text) => { try { if (colorScheme.type === "hex") { return chalk.hex(colorScheme.secondary)(text); } else if (colorScheme.type === "ansi256") { return chalk.ansi256(colorScheme.secondary)(text); } else { return chalk[colorScheme.secondary](text); } } catch (e) { return chalk.cyanBright(text); } }; // Create a grid layout const grid = new contrib.grid({ rows: 12, cols: 12, screen: screen, }); // Create ASCII logo box const logo = grid.set(0, 0, 3, 12, blessed.box, { tags: true, align: "center", valign: "middle", border: { type: "line", fg: uiColor, }, }); // Define a direct neon cyan color using ANSI escape sequences // This should be universally visible in all terminals const NEON_CYAN = "\x1b[38;5;51m"; // Bright neon cyan (ANSI 256 color) const NEON_CYAN_BG = "\x1b[48;5;51m"; // Bright neon cyan background const RESET = "\x1b[0m"; // Reset to default color // Create a function to apply the neon cyan color to text - for consistency const neonCyanText = (text) => `${NEON_CYAN}${text}${RESET}`; // Create ASCII logo with direct approach try { // Generate the text first const logoText = figlet.textSync("ENVIO.DEV", { font: "ANSI Shadow", horizontalLayout: "full", }); // Color it directly with neon cyan - should work universally const coloredLogo = NEON_CYAN + logoText + RESET; // Set the content directly logo.setContent(coloredLogo); } catch (error) { // Emergency fallback to plain text if all else fails logo.setContent(chalk.cyan.bold("ENVIO.DEV")); } // Create subtitle const subtitle = grid.set(3, 0, 1, 12, blessed.box, { content: chalk.yellow(` ${title} - Powered by Envio `), tags: true, align: "center", valign: "middle", style: { fg: "yellow", bold: true, }, }); // Create a custom progress bar const progressBox = grid.set(4, 0, 1, 12, blessed.box, { label: " Scanning Progress ", tags: true, border: { type: "line", fg: uiColor, }, style: { fg: "white", }, }); // Create stats display const stats = grid.set(5, 0, 3, 6, blessed.box, { label: "Stats", tags: true, border: { type: "line", fg: uiColor, }, style: { fg: "white", }, }); // Create event distribution display const eventAggregateDataDisplay = grid.set(5, 6, 3, 6, blessed.box, { label: `${eventName} - ${param}: Aggregate stats`, tags: true, border: { type: "line", fg: uiColor, }, style: { fg: "white", }, }); // Create log window const logWindow = grid.set(8, 0, 4, 12, contrib.log, { label: "Event Log", tags: true, border: { type: "line", fg: uiColor, }, style: { fg: "green", }, bufferLength: 30, }); // Exit on Escape, q, or Ctrl+C screen.key(["escape", "q", "C-c"], function (ch, key) { return process.exit(0); }); // Custom function to update the progress bar display const updateProgressBar = (progress, label = "") => { try { // Calculate the width of the progress bar (accounting for borders and label) const width = progressBox.width - 4; const filledWidth = Math.floor(width * progress); const emptyWidth = width - filledWidth; // Create the progress bar with our neon cyan color const filledBar = NEON_CYAN_BG + " ".repeat(filledWidth) + RESET; const emptyBar = chalk.bgBlack(" ".repeat(emptyWidth)); // Update the progress box content progressBox.setContent( `${filledBar}${emptyBar} ${(progress * 100).toFixed(2)}% ${label}` ); } catch (err) { // Silently handle errors } }; // Function to update event distribution display using ASCII bars const updateDataAggregationDisplay = (eventAggregates) => { try { // Add extra spacing for better readability const labelWidth = 12; // Set the content with each stat on its own line const content = `${neonCyanText("Count".padEnd(labelWidth))} ${formatNumber( eventAggregates.count )}\n` + `${neonCyanText("Sum".padEnd(labelWidth))} ${formatBigInt( eventAggregates.sum )}\n` + `${neonCyanText("Min".padEnd(labelWidth))} ${formatBigInt( eventAggregates.min )}\n` + `${neonCyanText("Max".padEnd(labelWidth))} ${formatBigInt( eventAggregates.max )}\n` + `${neonCyanText("Avg".padEnd(labelWidth))} ${formatBigInt( eventAggregates.average )}\n` + `${neonCyanText("Median".padEnd(labelWidth))} ${formatBigInt( eventAggregates.median )}\n` + `${neonCyanText("Variance".padEnd(labelWidth))} ${formatBigInt( eventAggregates.variance )}\n` + `${neonCyanText("StdDev".padEnd(labelWidth))} ${formatBigInt( eventAggregates.stdDev )}`; eventAggregateDataDisplay.setContent(content); } catch (err) { // Silently handle errors } }; // Render the screen screen.render(); //============================================================================= // MAIN FUNCTION //============================================================================= const startTime = performance.now(); // Log startup logWindow.log(chalk.yellow(`Initializing Event Scanner...`)); screen.render(); try { //========================================================================= // STEP 1: Get blockchain height using Hypersync //========================================================================= const height = await client.getHeight(); logWindow.log( `Starting scan from block ${safeHexColor( formatNumber(fromBlock) )} to ${safeHexColor(formatNumber(height))}` ); screen.render(); //========================================================================= // STEP 2: Create a decoder for the event signature //========================================================================= const decoder = Decoder.fromSignatures([eventSignature]); logWindow.log("Event decoder initialized"); screen.render(); //========================================================================= // STEP 3: Stream events from Hypersync //========================================================================= logWindow.log(chalk.green("Starting event stream...")); screen.render(); const stream = await client.stream(query, {}); // Update subtitle to show network subtitle.setContent( chalk.yellow(` ${title} - Block Height: ${formatNumber(height)} `) ); screen.render(); //========================================================================= // STEP 4: Process streaming data //========================================================================= let lastLogUpdate = 0; let lastDistributionUpdate = 0; // Initialize progress bar updateProgressBar( 0, `Block: ${formatNumber(fromBlock)}/${formatNumber(height)}` ); // Initialize distribution display updateDataAggregationDisplay(eventAggregates); screen.render(); while (true) { // Get the next batch of data from Hypersync const res = await stream.recv(); // Quit if we reached the tip of the blockchain if (res === null) { logWindow.log(chalk.green("✓ Reached the tip of the blockchain!")); updateProgressBar( 1, `Block: ${formatNumber(height)}/${formatNumber(height)}` ); screen.render(); break; } // Make sure we have a nextBlock value if (!res.nextBlock) { logWindow.log(chalk.yellow("Warning: Missing nextBlock in response")); continue; } // Process logs if any exist in this batch if ( res.data && res.data.logs && Array.isArray(res.data.logs) && res.data.logs.length > 0 ) { // Process logs based on their topic0 value res.data.logs.forEach((log) => { if (!log) return; // Skip if log is null eventAggregates.count++; if (!log.topics || !Array.isArray(log.topics) || !log.topics[0]) { eventAggregates.Unknown++; return; } const topic0 = log.topics[0]; const eventName = topic0ToName[topic0] || "Unknown"; if (eventName === "Unknown") { eventAggregates.Unknown++; } else { eventAggregates[eventName] = (eventAggregates[eventName] || 0) + 1; // Process the data field if it exists if (log.data) { try { // should use the hypersync decoder let decodedEvent = decodeEventLog({ abi: parseAbi([eventSignatureHuman]), data: log.data, topics: log.topics, strict: false, }); const value = decodedEvent.args[param]; eventAggregates.sum = eventAggregates.sum + value; eventAggregates.sumSquares = eventAggregates.sumSquares + value * value; // Update min and max on first value or if current value is smaller/larger if (eventAggregates.count === 1) { eventAggregates.min = value; eventAggregates.max = value; } else { if (value < eventAggregates.min) eventAggregates.min = value; if (value > eventAggregates.max) eventAggregates.max = value; } // Update median efficiently using heaps updateMedian(value); // Update average eventAggregates.average = eventAggregates.sum / BigInt(eventAggregates.count); // Calculate standard deviation with decimal place adjustment const mean = eventAggregates.average; // Adjust for decimals by dividing by 10^decimals before calculating variance const adjustedSumSquares = eventAggregates.sumSquares / BigInt(10) ** BigInt(decimals * 2); const adjustedMean = mean / BigInt(10) ** BigInt(decimals); eventAggregates.variance = BigInt( Math.floor( Number( adjustedSumSquares / BigInt(eventAggregates.count) - adjustedMean * adjustedMean ) ) ); // Convert to regular number for sqrt, floor it, then back to BigInt eventAggregates.stdDev = BigInt( Math.floor(Math.sqrt(Number(eventAggregates.variance))) ) * BigInt(10) ** BigInt(decimals); } catch (error) { console.error("Error processing log data:", error); } } } }); // Log a decoded event sample occasionally try { if (eventAggregates.count % 1000 === 0 && res.data.logs[0]) { const decodedLogs = await decoder.decodeLogs([res.data.logs[0]]); if ( decodedLogs && Array.isArray(decodedLogs) && decodedLogs.length > 0 && decodedLogs[0] ) { const eventInfo = decodedLogs[0].event ? safeStringify(decodedLogs[0].event) : "No event data"; logWindow.log( neonCyanText( `Sample event at block ${res.nextBlock}: ${eventInfo}` ) ); screen.render(); } } } catch (decodeError) { logWindow.log(chalk.yellow(`Decode warning: ${decodeError.message}`)); } } // Update the fromBlock for the next iteration if (res.nextBlock) { query.fromBlock = res.nextBlock; } // Calculate time stats const currentTime = performance.now(); const seconds = Math.max((currentTime - startTime) / 1000, 0.1); // Avoid division by zero const eventsPerSecond = (eventAggregates.count / seconds).toFixed(1); // Calculate progress const progress = Math.min( (res.nextBlock - fromBlock) / (height - fromBlock), 1 ); // Update the progress bar updateProgressBar( progress, `Block: ${formatNumber(res.nextBlock)}/${formatNumber(height)}` ); // Update stats display try { stats.setContent( `${neonCyanText("Current Block")}: ${formatNumber(res.nextBlock)}\n` + `${neonCyanText("Progress")}: ${(progress * 100).toFixed(2)}%\n` + `${neonCyanText("Total Events")}: ${formatNumber( eventAggregates.count )}\n` + `${neonCyanText("Elapsed Time")}: ${seconds.toFixed(1)}s\n` + `${neonCyanText("Speed")}: ${formatNumber( eventsPerSecond )} events/s` ); } catch (statsError) { // Silently handle errors } // Update event distribution periodically if (res.nextBlock - lastDistributionUpdate >= 10000) { updateDataAggregationDisplay(eventAggregates); lastDistributionUpdate = res.nextBlock; } // Log progress periodically to avoid too many updates if (res.nextBlock - lastLogUpdate >= 50000) { logWindow.log( `${neonCyanText("Block")} ${formatNumber( res.nextBlock )} | ${formatNumber( eventAggregates.count )} events | ${eventsPerSecond} events/s` ); lastLogUpdate = res.nextBlock; } // Render the updated screen screen.render(); } //========================================================================= // Final summary //========================================================================= const totalTime = Math.max((performance.now() - startTime) / 1000, 0.1); // Avoid division by zero // Update final stats stats.setContent( `${neonCyanText("Blocks Scanned")}: ${formatNumber(height)}\n` + `${neonCyanText("Total Events")}: ${formatNumber( eventAggregates.count )}\n` + `${neonCyanText("Elapsed Time")}: ${totalTime.toFixed(1)}s\n` + `${neonCyanText("Avg Speed")}: ${formatNumber( Math.round(eventAggregates.count / totalTime) )} events/s` ); // Final distribution update updateDataAggregationDisplay(eventAggregates); // Log completion logWindow.log(chalk.green("✓ Scan complete!")); logWindow.log( chalk.yellow(`Total processing time: ${totalTime.toFixed(2)} seconds`) ); logWindow.log( chalk.yellow( `Average speed: ${formatNumber( Math.round(eventAggregates.count / totalTime) )} events/second` ) ); // Bold final message subtitle.setContent(chalk.green.bold(" Scan Complete - Press Q to Exit ")); // Render final screen screen.render(); // Wait for user to exit await new Promise((resolve) => setTimeout(resolve, 1000000000)); } catch (error) { logWindow.log(chalk.red(`Error: ${error.message}`)); screen.render(); await new Promise((resolve) => setTimeout(resolve, 5000)); process.exit(1); } }