UNPKG

@nori-zk/mina-token-bridge

Version:

Nori ethereum state settelment and nETH token bridge zkApp

1,083 lines 78.5 kB
/** * loadRunner.ts — Continuous Nori bridge load generator * * Long-running script that mimics N users repeatedly exercising the * bridge (Ethereum lock → Mina mint) to stress-test WSS, worker * lifecycle, and the 32-root deposit window on mesa-testnet. * * Per flow: * - Fresh TokenBridgeWorker (spawned and signalTerminate'd each run). * - Dedicated WSS connection per user (not shared). * - Full flow mirrors minimal-client/src/index.spec.ts. * - Randomised post-canMint delay (see pickClaimDelayUpdates). * * Minimal env: * ETH_RPC_URL=https://sepolia.infura.io/v3/<key> * LOAD_USER_ETH_PRIV_KEYS=0x<key1>,0x<key2>,0x<key3> * LOAD_USER_MINA_PRIV_KEYS=EKE<key1>,EKE<key2>,EKE<key3> * * Common optional env: * LOAD_USER_LABELS=alice,bob,carol * LOAD_LOCK_AMOUNTS_ETH=0.0001 * LOAD_BASE_TICK_MINUTES=2 * LOAD_MAX_CONCURRENT=2 * LOAD_MAX_CONCURRENT_COMPILES=5 * LOAD_PER_USER_COOLDOWN_MINUTES=5 * LOAD_MINT_GATE_TIMEOUT_MINUTES=120 * LOAD_LOG_DIR=./logs/loadRunner * * See parseEnv() for the full list and defaults. * * Logs under LOAD_LOG_DIR: * loadRunner.log — scheduler decisions + user state transitions * loadRunner.<label>.log — per-user stage log * loadRunner.summary.jsonl — one JSON line per completed flow * stdout — everything + live bridge observer * * Shutdown: SIGINT/SIGTERM = immediate exit (in-flight flows NOT drained). */ import 'dotenv/config'; import { appendFileSync, mkdirSync, existsSync } from 'node:fs'; import path from 'node:path'; import { Logger, LogPrinter } from 'esm-iso-logger'; import { Field, Mina, PrivateKey, fetchAccount } from 'o1js'; import { ethers } from 'ethers'; import { NoriTokenBridge__factory } from '@nori-zk/ethereum-token-bridge'; import { share, Subject, Subscription, takeUntil, } from 'rxjs'; import { getReconnectingBridgeSocket$ } from '../rx/socket.js'; import { getBridgeStateTopic$, getBridgeTimingsTopic$, getEthStateTopic$, } from '../rx/topics.js'; import { bridgeStatusesKnownEnoughToLockUnsafe, canMint, getDepositProcessingStatus$, readyToComputeMintProof, } from '../rx/deposit.js'; import { getTokenBridgeWorker } from '../workers/tokenBridgeWorker/node/parent.js'; import { getStagingEnv } from '../tests/testUtils.js'; new LogPrinter('LoadRunner'); const logger = new Logger('LoadRunner'); // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- // Hardcoded ETH gas headroom above the lock amount. Sepolia is cheap but RPCs // sometimes mis-estimate — this keeps us from bouncing on tight balances. const ETH_GAS_BUFFER_ETH = 0.001; // LOAD_MINA_MIN_BALANCES — minimum MINA balance required before a flow // starts. Covers setup (1 MINA new-account fee) + mint + retry headroom. const MINA_MIN_BALANCE_DEFAULT = 2; // Contract retains 32 deposit roots. We never lag more than this to keep a // safety margin below eviction. const MAX_CLAIM_LAG_UPDATES = 28; // Stall watchdog for the claim-lag wait: we don't cap TOTAL time, we cap // silence between advances. A single mesa update can take 15–60min (avg ~30); // waiting N updates just means summing N of those. What's actually abnormal // is an extended gap with no advances at all, so we bail if no fresh slot // lands for this long. const CLAIM_LAG_STALL_TIMEOUT_MINUTES = 45; // Pause after signalTerminate so the worker child can exit cleanly. const WORKER_SETTLE_MS_DEFAULT = 5000; // Hard cap on readyToComputeMintProof / canMint waits. These gates have no // upstream timeout, so without a cap a stalled bridge can hang a flow // forever (holding a worker child + compiled circuits in RAM). const MINT_GATE_TIMEOUT_MINUTES_DEFAULT = 120; // Hard cap on the pre-lock `bridgeStatusesKnownEnoughToLockUnsafe` wait. // Same rationale — no upstream timeout in the helper. const BRIDGE_READY_TIMEOUT_MINUTES_DEFAULT = 30; // Mint proofs occasionally land stale (the embedded bridge state has rolled // past the on-chain window between proof build and tx inclusion). The fix is // to rebuild the whole proof+send pair and try again — three attempts is // enough to absorb one or two rolls without giving up on a flow. const MINT_RETRY_ATTEMPTS = 3; // ---- Env defaults (kept here so tuning is a one-file edit) ---- // LOAD_LOCK_AMOUNTS_ETH — ETH amount per lock. Sepolia is cheap; 0.0001 keeps // 1000s of runs affordable while staying above the contract's min unit. const LOCK_AMOUNT_ETH_DEFAULT = 0.0001; // LOAD_ETH_MIN_BALANCES — floor wallet balance before a flow is allowed to // run. Acts as operator-mandated headroom beyond a single lock+gas. const ETH_MIN_BALANCE_DEFAULT = 0.001; // LOAD_BASE_TICK_MINUTES — scheduler's base period between launch decisions. const BASE_TICK_MINUTES_DEFAULT = 2; // LOAD_TICK_JITTER_PCT — ±% random jitter around the base tick so the // scheduler doesn't align with bridge cadence. const TICK_JITTER_PCT_DEFAULT = 40; // LOAD_MAX_CONCURRENT — true global cap on flows in flight at once. const MAX_CONCURRENT_DEFAULT = 2; // LOAD_MAX_CONCURRENT_COMPILES — cap on parallel compileAll() invocations // (CPU-bound, 3–5min each). Independent of max concurrent flows so a flow // can lock ETH while queued for a compile slot. const MAX_CONCURRENT_COMPILES_DEFAULT = 5; // LOAD_PER_USER_COOLDOWN_MINUTES — post-flow ineligibility window per user. const PER_USER_COOLDOWN_MINUTES_DEFAULT = 5; // LOAD_MINA_TX_FEE_MINA — fee paid on every Mina tx this script sends. const MINA_TX_FEE_MINA_DEFAULT = 0.01; // LOAD_LOG_DIR — where the three log streams are written. const LOG_DIR_DEFAULT = './logs/loadRunner'; // --------------------------------------------------------------------------- // Env parsing // --------------------------------------------------------------------------- function parseList(envVar) { if (!envVar) return undefined; const parts = envVar.split(',').map((s) => s.trim()).filter(Boolean); return parts.length > 0 ? parts : undefined; } /** * Array-or-scalar rule: length 1 broadcasts to all users; length N zips * by index; any other length is a fatal config error. */ function broadcastOrZip(values, userCount, fallback, fieldName) { if (!values?.length) return new Array(userCount).fill(fallback); if (values.length === 1) return new Array(userCount).fill(values[0]); if (values.length !== userCount) { throw new Error(`${fieldName}: expected length 1 or ${userCount}, got ${values.length}`); } return values; } /** * Strict numeric env parser. Rejects NaN, infinity, and out-of-range values * at startup so the scheduler never silently inherits bad config (zero-delay * ticks, deadlocked semaphores, etc). */ function parseNumberEnv(raw, fallback, name, opts = {}) { if (raw == null || raw === '') return fallback; const v = Number(raw); if (!Number.isFinite(v)) { throw new Error(`${name} must be a finite number (got "${raw}")`); } if (opts.int && !Number.isInteger(v)) { throw new Error(`${name} must be an integer (got ${v})`); } if (opts.min !== undefined && v < opts.min) { throw new Error(`${name} must be >= ${opts.min} (got ${v})`); } if (opts.max !== undefined && v > opts.max) { throw new Error(`${name} must be <= ${opts.max} (got ${v})`); } return v; } /** * Formats an ETH amount as a fixed decimal string. `(1e-7).toString()` yields * `"1e-7"` which `ethers.parseEther` rejects; this expands to `"0.0000001"`. */ function formatEthAmount(n) { if (!Number.isFinite(n) || n <= 0) { throw new Error(`ETH amount must be positive and finite (got ${n})`); } if (n >= 1e-6 && !n.toString().includes('e')) return n.toString(); return n.toFixed(18).replace(/\.?0+$/, ''); } /** * Restricts labels to characters safe in filenames. Keeps the first 32 chars * of `[A-Za-z0-9_.-]+` and replaces everything else with `_`. */ function sanitizeLabel(label) { const safe = label.replace(/[^A-Za-z0-9_.-]/g, '_').slice(0, 32); return safe.length > 0 && safe !== '.' && safe !== '..' ? safe : 'user'; } function parseEnv() { const staging = getStagingEnv(); const ethRpcUrl = process.env.ETH_RPC_URL ?? ''; if (!/^https?:\/\//.test(ethRpcUrl)) { throw new Error(`ETH_RPC_URL missing or invalid (got "${ethRpcUrl}")`); } const ethPrivKeys = parseList(process.env.LOAD_USER_ETH_PRIV_KEYS); const minaPrivKeys = parseList(process.env.LOAD_USER_MINA_PRIV_KEYS); if (!ethPrivKeys?.length) { throw new Error('LOAD_USER_ETH_PRIV_KEYS is required'); } if (!minaPrivKeys?.length) { throw new Error('LOAD_USER_MINA_PRIV_KEYS is required'); } if (ethPrivKeys.length !== minaPrivKeys.length) { throw new Error(`LOAD_USER_ETH_PRIV_KEYS (${ethPrivKeys.length}) and LOAD_USER_MINA_PRIV_KEYS (${minaPrivKeys.length}) must have equal length`); } if (new Set(ethPrivKeys.map((k) => k.toLowerCase())).size !== ethPrivKeys.length) { throw new Error('LOAD_USER_ETH_PRIV_KEYS contains duplicate keys'); } if (new Set(minaPrivKeys).size !== minaPrivKeys.length) { throw new Error('LOAD_USER_MINA_PRIV_KEYS contains duplicate keys'); } const userCount = ethPrivKeys.length; const rawLabels = parseList(process.env.LOAD_USER_LABELS) ?? Array.from({ length: userCount }, (_, i) => `user${i}`); if (rawLabels.length !== userCount) { throw new Error(`LOAD_USER_LABELS: expected length ${userCount}, got ${rawLabels.length}`); } // Sanitize once here so filenames can embed the label unconditionally. const labels = rawLabels.map(sanitizeLabel); if (new Set(labels).size !== labels.length) { throw new Error(`LOAD_USER_LABELS yielded duplicates after sanitization: ${labels.join(', ')}`); } const scramMsgs = parseList(process.env.LOAD_USER_SCRAM_MSGS) ?? labels.map((l) => `NoriZK-${l}`); if (scramMsgs.length !== userCount) { throw new Error(`LOAD_USER_SCRAM_MSGS: expected length ${userCount}, got ${scramMsgs.length}`); } ethPrivKeys.forEach((k, i) => { if (!/^0x[a-fA-F0-9]{64}$/.test(k)) { throw new Error(`LOAD_USER_ETH_PRIV_KEYS[${i}] ("${labels[i]}") must be 0x-prefixed 64 hex chars`); } }); minaPrivKeys.forEach((k, i) => { try { PrivateKey.fromBase58(k); } catch { throw new Error(`LOAD_USER_MINA_PRIV_KEYS[${i}] ("${labels[i]}") is not a valid Mina base58 private key`); } }); const lockAmountsEth = broadcastOrZip(parseList(process.env.LOAD_LOCK_AMOUNTS_ETH)?.map(Number), userCount, LOCK_AMOUNT_ETH_DEFAULT, 'LOAD_LOCK_AMOUNTS_ETH'); lockAmountsEth.forEach((amt, i) => { if (!Number.isFinite(amt) || amt <= 0) { throw new Error(`LOAD_LOCK_AMOUNTS_ETH[${i}] ("${labels[i]}") must be positive (got ${amt})`); } try { ethers.parseEther(formatEthAmount(amt)); } catch (err) { throw new Error(`LOAD_LOCK_AMOUNTS_ETH[${i}] ("${labels[i]}") is not a valid ETH amount: ${String(err)}`); } }); const ethMinBalances = broadcastOrZip(parseList(process.env.LOAD_ETH_MIN_BALANCES)?.map(Number), userCount, ETH_MIN_BALANCE_DEFAULT, 'LOAD_ETH_MIN_BALANCES'); ethMinBalances.forEach((v, i) => { if (!Number.isFinite(v) || v < 0) { throw new Error(`LOAD_ETH_MIN_BALANCES[${i}] ("${labels[i]}") must be >= 0 (got ${v})`); } }); const minaMinBalances = broadcastOrZip(parseList(process.env.LOAD_MINA_MIN_BALANCES)?.map(Number), userCount, MINA_MIN_BALANCE_DEFAULT, 'LOAD_MINA_MIN_BALANCES'); minaMinBalances.forEach((v, i) => { if (!Number.isFinite(v) || v < 0) { throw new Error(`LOAD_MINA_MIN_BALANCES[${i}] ("${labels[i]}") must be >= 0 (got ${v})`); } }); const users = labels.map((label, i) => ({ label, ethPrivKeyHex: ethPrivKeys[i], minaPrivKeyBase58: minaPrivKeys[i], scramMsg: scramMsgs[i], lockAmountEth: lockAmountsEth[i], ethMinBalanceEth: ethMinBalances[i], minaMinBalance: minaMinBalances[i], })); // Timing env is in MINUTES — Mina is slow and this script runs for days. const baseTickMinutes = parseNumberEnv(process.env.LOAD_BASE_TICK_MINUTES, BASE_TICK_MINUTES_DEFAULT, 'LOAD_BASE_TICK_MINUTES', { min: 0.01 }); const perUserCooldownMinutes = parseNumberEnv(process.env.LOAD_PER_USER_COOLDOWN_MINUTES, PER_USER_COOLDOWN_MINUTES_DEFAULT, 'LOAD_PER_USER_COOLDOWN_MINUTES', { min: 0 }); const mintGateTimeoutMinutes = parseNumberEnv(process.env.LOAD_MINT_GATE_TIMEOUT_MINUTES, MINT_GATE_TIMEOUT_MINUTES_DEFAULT, 'LOAD_MINT_GATE_TIMEOUT_MINUTES', { min: 1 }); const bridgeReadyTimeoutMinutes = parseNumberEnv(process.env.LOAD_BRIDGE_READY_TIMEOUT_MINUTES, BRIDGE_READY_TIMEOUT_MINUTES_DEFAULT, 'LOAD_BRIDGE_READY_TIMEOUT_MINUTES', { min: 1 }); return { users, ethRpcUrl, noriEthBridgeAddressHex: process.env.NORI_ETH_TOKEN_BRIDGE_ADDRESS ?? staging.NORI_ETH_TOKEN_BRIDGE_ADDRESS, noriMinaBridgeAddressBase58: process.env.NORI_MINA_TOKEN_BRIDGE_ADDRESS ?? staging.NORI_MINA_TOKEN_BRIDGE_ADDRESS, noriTokenBaseAddressBase58: process.env.NORI_MINA_TOKEN_BASE_ADDRESS ?? staging.NORI_MINA_TOKEN_BASE_ADDRESS, noriWssUrl: process.env.NORI_WSS_URL ?? staging.NORI_WSS_URL, noriPcsUrl: process.env.NORI_PCS_URL ?? staging.NORI_PCS_URL, noriTokenBaseTokenId: process.env.NORI_MINA_TOKEN_BASE_TOKEN_ID ?? staging.NORI_MINA_TOKEN_BASE_TOKEN_ID, minaRpcUrl: process.env.MINA_RPC_NETWORK_URL ?? staging.MINA_RPC_NETWORK_URL, minaArchiveRpcUrl: process.env.MINA_ARCHIVE_RPC_URL ?? staging.MINA_ARCHIVE_RPC_URL, minaNetworkId: process.env.MINA_RPC_NETWORK_ID ?? staging.MINA_RPC_NETWORK_ID, baseTickMs: baseTickMinutes * 60_000, tickJitterPct: parseNumberEnv(process.env.LOAD_TICK_JITTER_PCT, TICK_JITTER_PCT_DEFAULT, 'LOAD_TICK_JITTER_PCT', { min: 0, max: 200 }), maxConcurrent: parseNumberEnv(process.env.LOAD_MAX_CONCURRENT, MAX_CONCURRENT_DEFAULT, 'LOAD_MAX_CONCURRENT', { min: 1, int: true }), maxConcurrentCompiles: parseNumberEnv(process.env.LOAD_MAX_CONCURRENT_COMPILES, MAX_CONCURRENT_COMPILES_DEFAULT, 'LOAD_MAX_CONCURRENT_COMPILES', { min: 1, int: true }), perUserCooldownMs: perUserCooldownMinutes * 60_000, workerSettleMs: parseNumberEnv(process.env.LOAD_WORKER_SETTLE_MS, WORKER_SETTLE_MS_DEFAULT, 'LOAD_WORKER_SETTLE_MS', { min: 0 }), mintGateTimeoutMs: mintGateTimeoutMinutes * 60_000, bridgeReadyTimeoutMs: bridgeReadyTimeoutMinutes * 60_000, minaTxFeeNanomina: parseNumberEnv(process.env.LOAD_MINA_TX_FEE_MINA, MINA_TX_FEE_MINA_DEFAULT, 'LOAD_MINA_TX_FEE_MINA', { min: 0 }) * 1e9, logDir: process.env.LOAD_LOG_DIR ?? LOG_DIR_DEFAULT, }; } // --------------------------------------------------------------------------- // Utilities // --------------------------------------------------------------------------- const tsLine = (msg) => `[${new Date().toISOString()}] ${msg}`; const appendLine = (file, line) => appendFileSync(file, line + '\n'); const formatMs = (ms) => ms < 1000 ? `${ms}ms` : ms < 60_000 ? `${(ms / 1000).toFixed(2)}s` : `${(ms / 60_000).toFixed(2)}m`; function randInt(n) { return Math.floor(Math.random() * n); } /** Fisher-Yates partial pick. */ function samplePick(arr, k) { const copy = arr.slice(); const out = []; for (let i = 0; i < k && copy.length > 0; i++) { const idx = randInt(copy.length); out.push(copy[idx]); copy.splice(idx, 1); } return out; } function jitter(baseMs, jitterPct) { return Math.max(0, baseMs + (Math.random() * 2 - 1) * baseMs * (jitterPct / 100)); } /** * Counting semaphore. Used to cap concurrent `compileAll()` invocations: * every flow allocates a worker, but compile is CPU-bound and running 20 in * parallel saturates the box. Release transfers the slot to the next waiter * without decrementing, so the invariant `active ≤ max` always holds. */ class Semaphore { constructor(max) { this.max = max; this.queue = []; this.active = 0; } acquire() { if (this.active < this.max) { this.active += 1; return Promise.resolve(); } return new Promise((resolve) => this.queue.push(resolve)); } release() { const next = this.queue.shift(); if (next) next(); else this.active -= 1; } async run(fn) { await this.acquire(); try { return await fn(); } finally { this.release(); } } get inFlight() { return this.active; } get waiting() { return this.queue.length; } } /** * Wraps a promise in a hard timeout. On fire, runs `onTimeout` (used to * cancel the upstream rxjs chain via a Subject) BEFORE rejecting, so the * underlying subscription is torn down instead of leaking a live WSS. */ async function withCancelableTimeout(promise, timeoutMs, label, onTimeout) { let handle; const timeoutPromise = new Promise((_, reject) => { handle = setTimeout(() => { onTimeout(); reject(new Error(`${label} timed out after ${formatMs(timeoutMs)}`)); }, timeoutMs); }); try { return await Promise.race([promise, timeoutPromise]); } finally { if (handle) clearTimeout(handle); } } // Shared split-screen buffers. The TTY UI (below) redraws these, and the // per-user + scheduler loggers push through them so the right pane reflects // all flow activity regardless of who emitted it. // eslint-disable-next-line no-control-regex const ANSI_ESCAPE_RE_INLINE = /\x1b\[[0-9;?]*[a-zA-Z]/g; const stripAnsi = (s) => s.replace(ANSI_ESCAPE_RE_INLINE, ''); class LogRing { constructor(capacity) { this.capacity = capacity; this.buf = []; } push(line) { this.buf.push(line); if (this.buf.length > this.capacity) { this.buf.splice(0, this.buf.length - this.capacity); } } tail(n) { return this.buf.slice(Math.max(0, this.buf.length - n)); } } const logRing = new LogRing(500); /** * Consecutive-duplicate suppressor. Only collapses the bridge's `[deposit]` * stream, which fires on every WSS frame during mint processing — keying on * the embedded `stage_name` field so we get one line per stage transition. * Every other log passes through unchanged so compile progress, lock/mint * timing, WS state transitions, etc. all appear verbatim. */ class StageDedup { constructor() { this.lastDepositStage = ''; } shouldEmit(msg) { const deposit = msg.match(/\[deposit\]\s*(.*)$/); if (!deposit) return true; const stageMatch = deposit[1].match(/"stage_name"\s*:\s*"([^"]+)"/); const key = stageMatch ? stageMatch[1] : deposit[1]; if (key === this.lastDepositStage) return false; this.lastDepositStage = key; return true; } } /** * Writes to the per-user log file, the aggregate scheduler log, AND mirrors * to the split-screen ring so the right pane reflects flow activity. Each * destination has its own dedup state to keep [deposit] chatter bounded * without hiding any other messages. */ class UserFileLogger { constructor(aggregatePath, userPath, label, aggregateDedup) { this.aggregatePath = aggregatePath; this.userPath = userPath; this.label = label; this.aggregateDedup = aggregateDedup; this.userDedup = new StageDedup(); this.ringDedup = new StageDedup(); } log(msg) { const prefixed = `[${this.label}] ${msg}`; const line = tsLine(prefixed); if (this.userDedup.shouldEmit(prefixed)) { appendLine(this.userPath, line); } if (this.aggregateDedup.shouldEmit(prefixed)) { appendLine(this.aggregatePath, line); } if (this.ringDedup.shouldEmit(prefixed)) { logRing.push(stripAnsi(line)); } } } /** * Weighted distribution for the randomised claim delay: * 70% mint immediately (common case) * 20% lag 1..5 updates ("user stepped away briefly") * 10% lag 5..MAX ("user claims much later") */ function pickClaimDelayUpdates() { const r = Math.random(); if (r < 0.7) return 0; if (r < 0.9) return 1 + randInt(5); return 5 + randInt(MAX_CLAIM_LAG_UPDATES - 5 + 1); } /** * Waits for N distinct bridge output_slot advances, with an inactivity * watchdog: bails if no new advance is seen for `stallTimeoutMs`. Never * throws — on stall we resolve with `completed=false` so the caller can * decide (typically still try to mint; the on-chain call is authoritative). * * The first emission of `bridgeStateTopic$` is the replayed current slot, not * a fresh advance — we capture it as a baseline and count only strictly * greater slots. */ async function waitForBridgeUpdatesOrStall(bridgeStateTopic$, updatesToWaitFor, stallTimeoutMs, onUpdate, abort$) { if (updatesToWaitFor <= 0) return { completed: true, received: 0 }; return new Promise((resolve) => { let baselineSlot; const seen = new Set(); let count = 0; let stallTimer; let abortSub; const done = (completed, reason) => { if (stallTimer) clearTimeout(stallTimer); abortSub?.unsubscribe(); sub.unsubscribe(); resolve({ completed, reason, received: count }); }; const armStall = () => { if (stallTimer) clearTimeout(stallTimer); stallTimer = setTimeout(() => done(false, `no advance in ${Math.round(stallTimeoutMs / 60_000)}min`), stallTimeoutMs); }; armStall(); if (abort$) { abortSub = abort$.subscribe({ next: () => done(false, 'aborted (drain)'), }); } const sub = bridgeStateTopic$.subscribe({ next: (s) => { const slot = s.output_slot; if (baselineSlot === undefined) { baselineSlot = slot; seen.add(slot); return; } if (slot <= baselineSlot || seen.has(slot)) return; seen.add(slot); count += 1; onUpdate(count, slot); if (count >= updatesToWaitFor) done(true); else armStall(); }, error: () => done(false, 'stream error'), complete: () => done(false, 'stream completed'), }); }); } /** * Re-read every flow (per directive): balances drift from locks, fees, and * external transfers. Requires `Mina.setActiveInstance()` done by caller. */ async function checkBalances(cfg, etherProvider) { const ethWallet = new ethers.Wallet(cfg.ethPrivKeyHex, etherProvider); const ethBalanceWei = await etherProvider.getBalance(await ethWallet.getAddress()); const ethBalanceEth = Number(ethers.formatEther(ethBalanceWei)); // Enforce the operator-configured floor AND the lock+gas requirement. // ethMinBalanceEth lets the operator mandate headroom beyond a single run. const requiredEth = Math.max(cfg.ethMinBalanceEth, cfg.lockAmountEth + ETH_GAS_BUFFER_ETH); if (ethBalanceEth < requiredEth) { return { ok: false, reason: `ETH ${ethBalanceEth.toFixed(6)} < required ${requiredEth.toFixed(6)}`, ethBalanceEth, minaBalance: -1, }; } const minaPubKey = PrivateKey.fromBase58(cfg.minaPrivKeyBase58).toPublicKey(); await fetchAccount({ publicKey: minaPubKey }); let minaBalance = 0; try { minaBalance = Number(Mina.getAccount(minaPubKey).balance.toBigInt()) / 1e9; } catch { // Account doesn't exist yet — treat as zero balance. } if (minaBalance < cfg.minaMinBalance) { return { ok: false, reason: `MINA ${minaBalance.toFixed(4)} < required ${cfg.minaMinBalance}`, ethBalanceEth, minaBalance, }; } return { ok: true, reason: '', ethBalanceEth, minaBalance }; } /** * Read ETH, MINA, and nETH balances for the TUI side panel. Never throws — * failures are recorded as NaN with an `error` note so the pane can still * render. Intentionally fetches without a worker: nETH is read via * `Mina.getAccount(..., tokenId)` directly so we don't spin up a worker for * every IDLE user. */ async function fetchPaneBalances(u, script, etherProvider) { u.balancesLoading = true; try { const ethWallet = new ethers.Wallet(u.cfg.ethPrivKeyHex, etherProvider); const ethAddr = await ethWallet.getAddress(); const ethWei = await etherProvider.getBalance(ethAddr); const eth = Number(ethers.formatEther(ethWei)); const minaPubKey = PrivateKey.fromBase58(u.cfg.minaPrivKeyBase58).toPublicKey(); await fetchAccount({ publicKey: minaPubKey }); let mina = 0; try { mina = Number(Mina.getAccount(minaPubKey).balance.toBigInt()) / 1e9; } catch { /* account doesn't exist yet */ } let nEth = null; try { const fetched = await fetchAccount({ publicKey: minaPubKey, tokenId: Field.fromValue(script.noriTokenBaseTokenId) }); nEth = Number(fetched.account.balance.toBigInt()) / 1e6; } catch { nEth = 0; } u.balances = { eth, mina, nEth, fetchedAt: Date.now() }; } catch (err) { u.balances = { eth: NaN, mina: NaN, nEth: null, fetchedAt: Date.now(), error: String(err).slice(0, 60), }; } finally { u.balancesLoading = false; } } // --------------------------------------------------------------------------- // Per-user flow // --------------------------------------------------------------------------- /** * Full lock → mint pipeline for one user, one run. Mirrors the 14 stages in * minimal-client/src/index.spec.ts but with: * - a fresh worker spawned + signalTerminate'd per call * - a per-user WSS (stress-test the socket layer) * - a randomised post-canMint delay * - graceful handling of missed mint windows (logs + returns failure) * * Ordering invariant: compileAll() must resolve before ANY Mina-touching * worker call. We overlap compile with the ETH lock (both are slow) and gate * with a single `await tokenBridgeWorkerReady` right after the lock receipt. */ async function runUserFlow(userState, script, uLog, etherProvider, compileSemaphore, drainSignal, onStageChange) { const cfg = userState.cfg; // Wall-clock start of the whole runUserFlow call; used internally for // measuring overall duration in FlowResult / log output. const flowStart = Date.now(); // userState.flowStartedAt is the *user-visible* timer and is intentionally // delayed until after lockTokens() returns — compile + bridge-state-wait // happen before the deposit is actually on-chain, and we don't want that // pre-amble to inflate the per-flow elapsed shown in the TUI. userState.flowStartedAt = undefined; userState.stages = []; userState.phaseEvents = []; userState.currentStage = undefined; userState.lastDepStatus = undefined; userState.lastSlots = undefined; let lockDurationMs = 0; let mintDurationMs = 0; const subs = new Subscription(); // Fires on flow teardown or mint-gate timeout. Any observable gated on // this via takeUntil completes cleanly, so firstValueFrom subscriptions // (inside readyToComputeMintProof / canMint) don't leak a live WSS. const cancelMintGate$ = new Subject(); let tokenBridgeWorker; try { uLog.log('Checking on-chain balances...'); const bal = await checkBalances(cfg, etherProvider); uLog.log(`eth=${bal.ethBalanceEth.toFixed(6)} mina=${bal.minaBalance.toFixed(4)}`); if (!bal.ok) { uLog.log(`SKIP: ${bal.reason}`); return { status: 'skipped', reason: bal.reason, totalDurationMs: Date.now() - flowStart, }; } uLog.log('Spawning TokenBridgeWorker + queueing compileAll (parallel with lock)...'); const TokenBridgeWorker = getTokenBridgeWorker(); tokenBridgeWorker = new TokenBridgeWorker(); const worker = tokenBridgeWorker; await worker.WALLET_setMinaPrivateKey(cfg.minaPrivKeyBase58); await worker.minaSetup({ networkId: script.minaNetworkId, mina: script.minaRpcUrl, archive: script.minaArchiveRpcUrl, }); uLog.log(`compile slots: ${compileSemaphore.inFlight} in-flight, ${compileSemaphore.waiting} waiting`); const compileStart = Date.now(); const tokenBridgeWorkerReady = compileSemaphore.run(() => worker.compileMinterDepsNoCache(true)); // Suppress unhandled-rejection if the flow returns (lock revert, // receipt missing, outer throw) before awaiting this promise. // signalTerminate in finally will propagate a rejection here. tokenBridgeWorkerReady.catch(() => undefined); uLog.log(`Opening WSS ${script.noriWssUrl}`); const { bridgeSocket$, bridgeSocketConnectionState$ } = getReconnectingBridgeSocket$(script.noriWssUrl); subs.add(bridgeSocketConnectionState$.subscribe({ next: (state) => uLog.log(`[WS] ${state}`), error: (err) => uLog.log(`[WS ERROR] ${String(err)}`), })); const ethStateTopic$ = getEthStateTopic$(bridgeSocket$); const bridgeStateTopic$ = getBridgeStateTopic$(bridgeSocket$); const bridgeTimingsTopic$ = getBridgeTimingsTopic$(bridgeSocket$); // SCRAM sign + codeChallenge don't need compiled circuits. const signatureSCRAMBase58 = await worker.MOCK_SCRAM_signMessage(cfg.scramMsg); const codeChallengeSCRAMStr = await worker.SCRAM_createCodeChallenge(signatureSCRAMBase58); const codeChallengeSCRAMBigInt = BigInt(codeChallengeSCRAMStr); uLog.log('Awaiting sufficient bridge state...'); // Local cancel subject piped into copies of the topics so the timeout // path completes the underlying firstValueFrom inside the helper // without affecting the shared topics used downstream. const cancelPreLock$ = new Subject(); try { await withCancelableTimeout(bridgeStatusesKnownEnoughToLockUnsafe(ethStateTopic$.pipe(takeUntil(cancelPreLock$)), bridgeStateTopic$.pipe(takeUntil(cancelPreLock$)), bridgeTimingsTopic$.pipe(takeUntil(cancelPreLock$))), script.bridgeReadyTimeoutMs, 'bridgeStatusesKnownEnoughToLockUnsafe', () => cancelPreLock$.next()); } finally { cancelPreLock$.complete(); } uLog.log(`Locking ${cfg.lockAmountEth} ETH...`); const lockStart = Date.now(); const ethWallet = new ethers.Wallet(cfg.ethPrivKeyHex, etherProvider); const contract = NoriTokenBridge__factory.connect(script.noriEthBridgeAddressHex, ethWallet); const credentialAttestation = codeChallengeSCRAMBigInt; const depositAmount = ethers.parseEther(formatEthAmount(cfg.lockAmountEth)); let txResp; try { txResp = await contract.lockTokens(credentialAttestation, { value: depositAmount, }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); uLog.log(`LOCK REVERTED: ${msg}`); return { status: 'failure', reason: `lockTokens reverted: ${msg}`, totalDurationMs: Date.now() - flowStart, }; } // Real timer starts here: the deposit has been broadcast. userState.flowStartedAt = Date.now(); uLog.log(`Lock tx ${txResp.hash} sent, awaiting receipt...`); const receipt = await txResp.wait(); if (!receipt) { return { status: 'failure', reason: 'No tx receipt returned', lockTxHash: txResp.hash, totalDurationMs: Date.now() - flowStart, }; } lockDurationMs = Date.now() - lockStart; uLog.log(`Lock confirmed block=${receipt.blockNumber} in ${formatMs(lockDurationMs)}`); // GATE: any Mina-touching worker call from here down requires compile. const { noriStorageInterfaceVerificationKeySafe } = await tokenBridgeWorkerReady; uLog.log(`compileAll finished in ${formatMs(Date.now() - compileStart)}`); // share() turns the cold pipeline hot so both the logging subscription // and the readyToComputeMintProof/canMint awaiters consume the same // stream. takeUntil(cancelMintGate$) ensures timeouts and flow // teardown complete the observable and free its upstream subscription. const depositProcessingStatus$ = getDepositProcessingStatus$(receipt.blockNumber, ethStateTopic$, bridgeStateTopic$, bridgeTimingsTopic$).pipe(takeUntil(cancelMintGate$), share()); subs.add(depositProcessingStatus$.subscribe({ next: (msg) => { uLog.log(`[deposit] ${JSON.stringify(msg)}`); // Mirror to UserState so the TUI can render pipeline // + balances without scraping the log ring. const m = msg; if (!m.stage_name) return; const now = Date.now(); const stageChanged = userState.currentStage !== m.stage_name; if (stageChanged) { const prev = userState.stages[userState.stages.length - 1]; if (prev && !prev.finishedAt) prev.finishedAt = now; userState.stages.push({ name: m.stage_name, startedAt: now, serverElapsedSec: m.elapsed_sec ?? 0, etaSec: m.time_remaining_sec, depStatus: m.deposit_processing_status, inSlot: m.input_slot, outSlot: m.output_slot, }); userState.currentStage = m.stage_name; } else { const cur = userState.stages[userState.stages.length - 1]; if (cur) { cur.serverElapsedSec = m.elapsed_sec ?? cur.serverElapsedSec; cur.etaSec = m.time_remaining_sec ?? cur.etaSec; cur.depStatus = m.deposit_processing_status; cur.inSlot = m.input_slot; cur.outSlot = m.output_slot; } } // Track high-level phase transitions independently of // bridge stages: WaitingForEthFinality → // WaitingForPreviousJobCompletion → // WaitingForCurrentJobCompletion → ReadyToMint. const newStatus = m.deposit_processing_status; if (newStatus && newStatus !== userState.lastDepStatus) { const prevPhase = userState.phaseEvents[userState.phaseEvents.length - 1]; if (prevPhase && !prevPhase.finishedAt) { prevPhase.finishedAt = now; } userState.phaseEvents.push({ status: newStatus, startedAt: now, }); } userState.lastDepStatus = newStatus; if (m.input_slot !== undefined && m.output_slot !== undefined) { userState.lastSlots = { inSlot: m.input_slot, outSlot: m.output_slot, }; } userState.lastDepositBlock = m.deposit_block_number; if (stageChanged) onStageChange?.(userState); }, error: (err) => uLog.log(`[deposit ERROR] ${String(err)}`), complete: () => { // The upstream observable completes on MissedMintingOpportunity // (a narrow-window heuristic) OR teardown via cancelMintGate$. // Neither implies the on-chain mint will fail, so don't // phrase this as terminal. uLog.log('[deposit] WSS stream ended (informational only)'); const last = userState.stages[userState.stages.length - 1]; if (last && !last.finishedAt) last.finishedAt = Date.now(); }, })); try { await withCancelableTimeout(readyToComputeMintProof(depositProcessingStatus$), script.mintGateTimeoutMs, 'readyToComputeMintProof', () => cancelMintGate$.next()); } catch (err) { const msg = err instanceof Error ? err.message : String(err); // Same rationale as the canMint catch below: the observable's // "missed" heuristic is unreliable. Only a true timeout (from // withCancelableTimeout) should bail — anything else, log and // fall through to setup + attempt mint. const isMissedHeuristic = /miss/i.test(msg) && !msg.includes('timed out'); if (!isMissedHeuristic) { uLog.log(`readyToComputeMintProof threw (timeout): ${msg}`); return { status: 'failure', reason: `missed mint window (pre-proof): ${msg}`, lockTxHash: txResp.hash, totalDurationMs: Date.now() - flowStart, }; } uLog.log(`readyToComputeMintProof reports missed (heuristic): ${msg} — attempting mint anyway`); } const minaPubKeyBase58 = PrivateKey.fromBase58(cfg.minaPrivKeyBase58) .toPublicKey() .toBase58(); const setupRequired = await worker.needsToSetupStorage(script.noriMinaBridgeAddressBase58, minaPubKeyBase58); if (setupRequired) { uLog.log('Running MOCK_setupStorage...'); const setupStart = Date.now(); const { txHash: setupTxHash } = await worker.MOCK_setupStorage(minaPubKeyBase58, script.noriMinaBridgeAddressBase58, script.minaTxFeeNanomina, noriStorageInterfaceVerificationKeySafe); uLog.log(`Storage setup tx ${setupTxHash} in ${formatMs(Date.now() - setupStart)}`); } uLog.log('Computing deposit attestation witness...'); const depositAttestationInput = await worker.computeDepositAttestationWitness(codeChallengeSCRAMStr, receipt.blockNumber, script.noriPcsUrl); try { await withCancelableTimeout(canMint(depositProcessingStatus$), script.mintGateTimeoutMs, 'canMint', () => cancelMintGate$.next()); } catch (err) { const msg = err instanceof Error ? err.message : String(err); // The observable's "MissedMintingOpportunity" classification uses // a narrow window heuristic that often fires before the on-chain // window actually closes. We've already paid for compile + setup // + attestation witness — let the on-chain mint call decide. // Real timeouts (cancelMintGate$ fired) still bail. const isMissedHeuristic = /miss/i.test(msg) && !msg.includes('timed out'); if (!isMissedHeuristic) { uLog.log(`canMint threw (timeout): ${msg}`); return { status: 'failure', reason: `missed mint window (pre-send): ${msg}`, lockTxHash: txResp.hash, totalDurationMs: Date.now() - flowStart, }; } uLog.log(`canMint reports missed (heuristic): ${msg} — attempting mint anyway`); } const claimDelayUpdates = pickClaimDelayUpdates(); if (claimDelayUpdates > 0) { if (drainSignal.requested) { uLog.log(`[claim-lag] drain requested — skipping ${claimDelayUpdates}-update lag, minting now`); } else { uLog.log(`Lagging claim: ${claimDelayUpdates} update(s) (stall watchdog: ${CLAIM_LAG_STALL_TIMEOUT_MINUTES}min)`); const lagResult = await waitForBridgeUpdatesOrStall(bridgeStateTopic$, claimDelayUpdates, CLAIM_LAG_STALL_TIMEOUT_MINUTES * 60_000, (count, slot) => uLog.log(`[claim-lag] ${count}/${claimDelayUpdates} (slot ${slot})`), drainSignal.signal$); if (!lagResult.completed) { if (lagResult.reason === 'aborted (drain)') { uLog.log(`[claim-lag] drain — minting immediately after ${lagResult.received}/${claimDelayUpdates}`); } else { uLog.log(`[claim-lag] giving up after ${lagResult.received}/${claimDelayUpdates} (${lagResult.reason}) — attempting mint anyway`); } } } } // Mint can fail if the proof embeds bridge state that has rolled // forward by the time the tx is included. Rebuild the whole // proof+send pair on each attempt — both `needsToFundAccount` and // the proof itself are re-derived from current state, so a stale // one-shot becomes a fresh retry. const mintStart = Date.now(); let mintTxHash; let lastMintErr; for (let attempt = 1; attempt <= MINT_RETRY_ATTEMPTS; attempt++) { try { const needsToFundNow = await worker.needsToFundAccount(script.noriTokenBaseAddressBase58, minaPubKeyBase58); uLog.log(`Mint attempt ${attempt}/${MINT_RETRY_ATTEMPTS}: building proof (needsToFund=${needsToFundNow})...`); await worker.MOCK_computeMintProofAndCache(minaPubKeyBase58, script.noriMinaBridgeAddressBase58, depositAttestationInput, cfg.scramMsg, signatureSCRAMBase58, script.minaTxFeeNanomina, needsToFundNow); const sent = await worker.WALLET_MOCK_signAndSendMintProofCache(); mintTxHash = sent.txHash; uLog.log(`Mint attempt ${attempt} succeeded: tx ${mintTxHash}`); break; } catch (err) { lastMintErr = err; const msg = err instanceof Error ? err.message : String(err); uLog.log(`Mint attempt ${attempt}/${MINT_RETRY_ATTEMPTS} failed: ${msg}`); } } if (!mintTxHash) { const reason = lastMintErr instanceof Error ? lastMintErr.message : String(lastMintErr); return { status: 'failure', reason: `mint failed after ${MINT_RETRY_ATTEMPTS} attempts: ${reason}`, lockTxHash: txResp.hash, totalDurationMs: Date.now() - flowStart, }; } mintDurationMs = Date.now() - mintStart; uLog.log(`Mint tx ${mintTxHash} finalized in ${formatMs(mintDurationMs)}`); const mintedSoFar = await worker.mintedSoFar(script.noriMinaBridgeAddressBase58, minaPubKeyBase58); const balanceOfUser = await worker.getBalanceOf(script.noriTokenBaseAddressBase58, minaPubKeyBase58); uLog.log(`mintedSoFar=${mintedSoFar} balance=${balanceOfUser}`); return { status: 'success', lockTxHash: txResp.hash, mintTxHash, lockedEth: cfg.lockAmountEth, mintedBU: String(mintedSoFar), lockDurationMs, mintDurationMs, totalDurationMs: Date.now() - flowStart, claimDelayUpdates, }; } catch (err) { const msg = err instanceof Error ? err.stack ?? err.message : String(err); uLog.log(`UNCAUGHT: ${msg}`); return { status: 'failure', reason: `uncaught: ${msg.split('\n')[0]}`, totalDurationMs: Date.now() - flowStart, }; } finally { // Cancel any in-flight mint-gate observable chain before tearing down // subscriptions, so orphaned firstValueFrom promises (inside // readyToComputeMintProof / canMint) complete instead of leaking. cancelMintGate$.next(); cancelMintGate$.complete(); subs.unsubscribe(); try { tokenBridgeWorker?.signalTerminate(); } catch (err) { uLog.log(`signalTerminate ignored: ${String(err)}`); } await new Promise((resolve) => setTimeout(resolve, script.workerSettleMs)); uLog.log('Teardown complete.'); } } /** * Dedicated WSS that tracks live bridge state. Still logs every emission so a * tailing operator sees raw events, but also exposes a getter so the scheduler * can redraw a live status banner on a fixed cadence (between bridge frames). */ function startBridgeObserver(wssUrl) { logger.log(`[observer] connecting to ${wssUrl}`); const { bridgeSocket$, bridgeSocketConnectionState$ } = getReconnectingBridgeSocket$(wssUrl); const subs = new Subscription(); const snap = { wsState: 'connecting', latestBridge: null, latestEth: null, lastBridgeAt: 0, lastEthAt: 0, }; subs.add(bridgeSocketConnectionState$.subscribe({ next: (state) => { snap.wsState = state; }, error: (err) => logger.error(`[observer WS ERROR] ${String(err)}`), })); subs.add(getBridgeStateTopic$(bridgeSocket$).subscribe({ next: (s) => { snap.latestBridge = s; snap.lastBridgeAt = Date.now(); }, })); subs.add(getEthStateTopic$(bridgeSocket$).subscribe({ next: (s) => { snap.latestEth = s; snap.lastEthAt = Date.now(); }, })); return { stop: () => subs.unsubscribe(), snapshot: () => snap, }; } // --------------------------------------------------------------------------- // Main / scheduler // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // Split-screen terminal UI: stdout hijack + ring buffer + full redraw // --------------------------------------------------------------------------- const visibleLen = (s) => stripAnsi(s).length; const padRightVisible = (s, w) => { const v = visibleLen(s); return v >= w ? s : s + ' '.repeat(w - v); }; const originalStdoutWrite = process.stdout.write.bind(process.stdout); let bypassStdout = false; let stdoutLineBuffer = ''; function installStdoutHijack() { process.stdout.write = (chunk, ...rest) => { if (bypassStdout) { // eslint-disable-next-line @typescript-eslint/no-explicit-any return originalStdoutWrite(chunk, ...rest); } const str = typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'); stdoutLineBuffer += str; let nl = stdoutLineBuffer.indexOf('\n'); while (nl >= 0) { const line = stdoutLineBuffer.slice(0, nl); stdoutLineBuffer = stdoutLineBuffer.slice(nl + 1); if (line.length > 0) logRing.push(stripAnsi(line)); nl = stdoutLineBuffer.indexOf('\n'); } return true; }; } function restoreStdout() { process.stdout.write = originalStdoutWrite; } async function main() { const script = parseEnv(); if (!existsSync(script.logDir)) mkdirSync(script.logDir, { recursive: true }); const aggregatePath = path.join(script.logDir, 'loadRunner.log'); const summaryPath = path.join(script.logDir, 'loadRunner.summary.jsonl'); Mina.setActiveInstance(Mina.Network({ networkId: script.minaNetworkId, mina: script.minaRpcUrl, archive: script.minaArchiveRpcUrl, })); const etherProvider = new ethers.JsonRpcProvider(script.ethRpcUrl); const compileSemaphore = new Semaphore(script.maxConcurrentCompiles); const aggregateDedup = new StageDedup(); const userStates = script.users.map((cfg) => ({ cfg, status: 'IDLE', nextEligibleAt: 0, stats: { runs: 0, successes: 0, failures: 0, skipped: 0 }, stages: [], phaseEvents: [], })); const banner = [ '─'.repeat(60), `LoadRunner starting @ ${new Date().toISOString()}`, `users : ${userStates.map((u) => u.cfg.label).join(', ')}`, `tick : ${(script.baseTickMs / 60_000).toFixed(1)}min ±${script.tickJitterPct}%`, `max concurrent : ${script.maxConcurrent}`, `max concurrent cc : ${script.maxConcurrentCompiles}`, `mint gate timeout : ${(script.mintGateTimeoutMs / 60_000).toFixed(1)}min`, `per-user cooldown : ${(script.perUserCooldownMs / 60_000).toFixed(1)}min`, `eth rpc : ${script.ethRpcUrl}`, `mina rpc : ${script.minaRpcUrl}`, `wss : ${script.noriWssUrl}`, `eth b