UNPKG

@moonwall/cli

Version:

Testing framework for the Moon family of projects

695 lines (682 loc) 25 kB
// src/internal/commandParsers.ts import { createLogger as createLogger2 } from "@moonwall/util"; import path2 from "path"; import net from "net"; import { Effect as Effect3 } from "effect"; // src/lib/repoDefinitions/moonbeam.ts var repo = { name: "moonbeam", binaries: [ { name: "moonbeam", defaultArgs: [ "--no-hardware-benchmarks", "--no-telemetry", "--reserved-only", "--rpc-cors=all", "--unsafe-rpc-external", "--unsafe-force-node-key-generation", "--no-grandpa", "--sealing=manual", "--force-authoring", "--database=paritydb", "--no-prometheus", "--alice", "--chain=moonbase-dev", "--tmp" ] }, { name: "moonbase-runtime" }, { name: "moonbeam-runtime" }, { name: "moonriver-runtime" } ], ghAuthor: "moonbeam-foundation", ghRepo: "moonbeam" }; var moonbeam_default = repo; // src/lib/repoDefinitions/polkadot.ts var repo2 = { name: "polkadot", binaries: [ { name: "polkadot" }, { name: "polkadot-prepare-worker" }, { name: "polkadot-execute-worker" } ], ghAuthor: "paritytech", ghRepo: "polkadot-sdk" }; var polkadot_default = repo2; // src/lib/repoDefinitions/tanssi.ts var repo3 = { name: "tanssi", binaries: [ { name: "tanssi-node", defaultArgs: ["--dev", "--sealing=manual", "--no-hardware-benchmarks"] }, { name: "container-chain-template-simple-node" }, { name: "container-chain-template-frontier-node" } ], ghAuthor: "moondance-labs", ghRepo: "tanssi" }; var tanssi_default = repo3; // src/lib/configReader.ts import { readFile, access } from "fs/promises"; import { readFileSync, existsSync, constants } from "fs"; import JSONC from "jsonc-parser"; import path, { extname } from "path"; // src/lib/repoDefinitions/index.ts function standardRepos() { const defaultRepos = [moonbeam_default, polkadot_default, tanssi_default]; return [...defaultRepos]; } // src/lib/shardManager.ts var ShardManager = class _ShardManager { static instance = null; _shardInfo = null; constructor() { } static getInstance() { if (!_ShardManager.instance) { _ShardManager.instance = new _ShardManager(); } return _ShardManager.instance; } /** * Initialize shard configuration from command line argument or environment * @param shardArg Optional shard argument from CLI (format: "current/total") */ initializeSharding(shardArg) { if (shardArg) { this._shardInfo = this.parseShardString(shardArg); process.env.MOONWALL_TEST_SHARD = shardArg; } else if (process.env.MOONWALL_TEST_SHARD) { this._shardInfo = this.parseShardString(process.env.MOONWALL_TEST_SHARD); } else { this._shardInfo = { current: 1, total: 1, isSharded: false }; } } /** * Get current shard information */ getShardInfo() { if (!this._shardInfo) { this.initializeSharding(); } return this._shardInfo; } /** * Get shard index (0-based) for port calculations */ getShardIndex() { return this.getShardInfo().current - 1; } /** * Get total number of shards */ getTotalShards() { return this.getShardInfo().total; } /** * Check if sharding is enabled */ isSharded() { return this.getShardInfo().isSharded; } /** * Reset shard configuration (mainly for testing) */ reset() { this._shardInfo = null; delete process.env.MOONWALL_TEST_SHARD; } parseShardString(shardString) { if (!shardString.includes("/")) { throw new Error( `Invalid shard format: "${shardString}". Expected format: "current/total" (e.g., "1/3")` ); } const [currentStr, totalStr] = shardString.split("/"); const current = parseInt(currentStr, 10); const total = parseInt(totalStr, 10); if (Number.isNaN(current) || Number.isNaN(total) || current < 1 || total < 1) { throw new Error( `Invalid shard numbers in "${shardString}". Both current and total must be positive integers.` ); } if (current > total) { throw new Error( `Invalid shard configuration: current shard ${current} cannot be greater than total ${total}` ); } const isSharded = total > 1; return { current, total, isSharded }; } }; var shardManager = ShardManager.getInstance(); // src/internal/commandParsers.ts import invariant from "tiny-invariant"; // src/internal/effect/StartupCacheService.ts import { Command, FileSystem as FileSystem2, Path } from "@effect/platform"; import { NodeContext } from "@effect/platform-node"; import { createLogger } from "@moonwall/util"; import { Context, Duration as Duration2, Effect as Effect2, Layer, Option, Stream } from "effect"; import * as crypto from "crypto"; // src/internal/effect/errors.ts import { Data } from "effect"; var PortDiscoveryError = class extends Data.TaggedError("PortDiscoveryError") { }; var NodeLaunchError = class extends Data.TaggedError("NodeLaunchError") { }; var NodeReadinessError = class extends Data.TaggedError("NodeReadinessError") { }; var ProcessError = class extends Data.TaggedError("ProcessError") { }; var StartupCacheError = class extends Data.TaggedError("StartupCacheError") { }; var FileLockError = class extends Data.TaggedError("FileLockError") { }; // src/internal/effect/FileLock.ts import { FileSystem } from "@effect/platform"; import { Duration, Effect, Schedule } from "effect"; import * as os from "os"; var LOCK_MAX_AGE = Duration.minutes(2); var LOCK_POLL_INTERVAL = Duration.millis(500); var isProcessAlive = (pid) => Effect.try(() => { process.kill(pid, 0); return true; }).pipe(Effect.orElseSucceed(() => false)); var isLockStale = (info) => Effect.gen(function* () { const isTimedOut = Date.now() - info.timestamp > Duration.toMillis(LOCK_MAX_AGE); if (isTimedOut) return true; const isSameHost = info.hostname === os.hostname(); if (!isSameHost) return false; const alive = yield* isProcessAlive(info.pid); return !alive; }); var cleanupStaleLock = (lockPath) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const infoPath = `${lockPath}/lock.json`; const exists = yield* fs.exists(infoPath).pipe(Effect.orElseSucceed(() => false)); if (!exists) return; const content = yield* fs.readFileString(infoPath).pipe(Effect.orElseSucceed(() => "")); const info = yield* Effect.try(() => JSON.parse(content)).pipe( Effect.orElseSucceed(() => null) ); if (!info) return; const stale = yield* isLockStale(info); if (stale) { yield* fs.remove(lockPath, { recursive: true }).pipe(Effect.ignore); } }); var writeLockInfo = (lockPath) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const info = { pid: process.pid, timestamp: Date.now(), hostname: os.hostname() }; yield* fs.writeFileString(`${lockPath}/lock.json`, JSON.stringify(info)).pipe(Effect.ignore); }); var tryAcquireLock = (lockPath) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; yield* cleanupStaleLock(lockPath); yield* fs.makeDirectory(lockPath).pipe(Effect.mapError(() => new FileLockError({ reason: "acquisition_failed", lockPath }))); yield* writeLockInfo(lockPath); }); var acquireFileLock = (lockPath, timeout = Duration.minutes(2)) => tryAcquireLock(lockPath).pipe( Effect.retry(Schedule.fixed(LOCK_POLL_INTERVAL).pipe(Schedule.upTo(timeout))), Effect.catchAll(() => Effect.fail(new FileLockError({ reason: "timeout", lockPath }))) ); var releaseFileLock = (lockPath) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; yield* fs.remove(lockPath, { recursive: true }).pipe(Effect.ignore); }); var withFileLock = (lockPath, effect, timeout = Duration.minutes(2)) => Effect.acquireUseRelease( acquireFileLock(lockPath, timeout), () => effect, () => releaseFileLock(lockPath) ); // src/internal/effect/StartupCacheService.ts var logger = createLogger({ name: "StartupCacheService" }); var StartupCacheService = class extends Context.Tag("StartupCacheService")() { }; var hashFile = (filePath) => Effect2.gen(function* () { const fs = yield* FileSystem2.FileSystem; const hash = crypto.createHash("sha256"); yield* fs.stream(filePath).pipe(Stream.runForEach((chunk) => Effect2.sync(() => hash.update(chunk)))); return hash.digest("hex"); }).pipe(Effect2.mapError((cause) => new StartupCacheError({ cause, operation: "hash" }))); var findPrecompiledWasm = (dir) => Effect2.gen(function* () { const fs = yield* FileSystem2.FileSystem; const pathService = yield* Path.Path; const exists = yield* fs.exists(dir); if (!exists) return Option.none(); const files = yield* fs.readDirectory(dir).pipe(Effect2.orElseSucceed(() => [])); const wasmFile = files.find( (f) => f.startsWith("precompiled_wasm_") || f.endsWith(".cwasm") || f.endsWith(".wasm") ); return wasmFile ? Option.some(pathService.join(dir, wasmFile)) : Option.none(); }).pipe(Effect2.catchAll(() => Effect2.succeed(Option.none()))); var checkCache = (cacheDir, hashPath, expectedHash) => Effect2.gen(function* () { const fs = yield* FileSystem2.FileSystem; const savedHash = yield* fs.readFileString(hashPath).pipe(Effect2.orElseSucceed(() => "")); if (savedHash.trim() !== expectedHash) return Option.none(); const wasmPath = yield* findPrecompiledWasm(cacheDir); if (Option.isNone(wasmPath)) return Option.none(); const accessible = yield* fs.access(wasmPath.value).pipe( Effect2.as(true), Effect2.orElseSucceed(() => false) ); return accessible ? wasmPath : Option.none(); }); var runPrecompile = (binPath, chainArg, outputDir) => Effect2.gen(function* () { const fs = yield* FileSystem2.FileSystem; const pathService = yield* Path.Path; const args = chainArg ? ["precompile-wasm", chainArg, outputDir] : ["precompile-wasm", outputDir]; logger.debug(`Precompiling: ${binPath} ${args.join(" ")}`); const startTime = Date.now(); const exitCode = yield* Command.exitCode(Command.make(binPath, ...args)).pipe( Effect2.mapError( (e) => new StartupCacheError({ cause: e, operation: "precompile" }) ) ); const files = yield* fs.readDirectory(outputDir).pipe(Effect2.mapError((e) => new StartupCacheError({ cause: e, operation: "precompile" }))); const wasmFile = files.find( (f) => f.startsWith("precompiled_wasm_") || f.endsWith(".cwasm") || f.endsWith(".wasm") ); if (!wasmFile) { return yield* Effect2.fail( new StartupCacheError({ cause: `precompile-wasm failed (code ${exitCode}): no WASM file generated`, operation: "precompile" }) ); } const wasmPath = pathService.join(outputDir, wasmFile); logger.debug(`Precompiled in ${Date.now() - startTime}ms: ${wasmPath}`); return wasmPath; }); var generateRawChainSpec = (binPath, chainName, outputPath) => Effect2.gen(function* () { const fs = yield* FileSystem2.FileSystem; const args = chainName === "dev" || chainName === "default" ? ["build-spec", "--dev", "--raw"] : ["build-spec", `--chain=${chainName}`, "--raw"]; logger.debug(`Generating raw chain spec: ${binPath} ${args.join(" ")}`); const stdout = yield* Command.string(Command.make(binPath, ...args)).pipe( Effect2.mapError( (e) => new StartupCacheError({ cause: e, operation: "chainspec" }) ) ); if (!stdout.length) { return yield* Effect2.fail( new StartupCacheError({ cause: "build-spec produced no output", operation: "chainspec" }) ); } yield* fs.writeFileString(outputPath, stdout).pipe(Effect2.mapError((e) => new StartupCacheError({ cause: e, operation: "chainspec" }))); return outputPath; }); var maybeGetRawChainSpec = (binPath, chainName, cacheSubDir, shouldGenerate) => Effect2.gen(function* () { if (!shouldGenerate) return Option.none(); const fs = yield* FileSystem2.FileSystem; const pathService = yield* Path.Path; const rawSpecPath = pathService.join(cacheSubDir, `${chainName}-raw.json`); const exists = yield* fs.exists(rawSpecPath).pipe(Effect2.orElseSucceed(() => false)); if (exists) return Option.some(rawSpecPath); return yield* generateRawChainSpec(binPath, chainName, rawSpecPath).pipe( Effect2.map(Option.some), Effect2.catchAll(() => Effect2.succeed(Option.none())) ); }); var getCachedArtifactsImpl = (config) => Effect2.gen(function* () { const fs = yield* FileSystem2.FileSystem; const pathService = yield* Path.Path; const binaryHash = yield* hashFile(config.binPath); const shortHash = binaryHash.substring(0, 12); const chainName = config.isDevMode ? "dev" : config.chainArg?.match(/--chain[=\s]?(\S+)/)?.[1] || "default"; const binName = pathService.basename(config.binPath); const cacheSubDir = pathService.join(config.cacheDir, `${binName}-${chainName}-${shortHash}`); const hashPath = pathService.join(cacheSubDir, "binary.hash"); const lockPath = pathService.join(config.cacheDir, `${binName}-${chainName}.lock`); yield* fs.makeDirectory(cacheSubDir, { recursive: true }).pipe(Effect2.mapError((e) => new StartupCacheError({ cause: e, operation: "cache" }))); const cached = yield* checkCache(cacheSubDir, hashPath, binaryHash); if (Option.isSome(cached)) { logger.debug(`Using cached precompiled WASM: ${cached.value}`); const rawChainSpecPath = yield* maybeGetRawChainSpec( config.binPath, chainName, cacheSubDir, config.generateRawChainSpec ?? false ); return { precompiledPath: cached.value, fromCache: true, rawChainSpecPath: Option.getOrUndefined(rawChainSpecPath) }; } return yield* withFileLock( lockPath, Effect2.gen(function* () { const nowCached = yield* checkCache(cacheSubDir, hashPath, binaryHash); if (Option.isSome(nowCached)) { logger.debug( `Using cached precompiled WASM (created by another process): ${nowCached.value}` ); const rawChainSpecPath2 = yield* maybeGetRawChainSpec( config.binPath, chainName, cacheSubDir, config.generateRawChainSpec ?? false ); return { precompiledPath: nowCached.value, fromCache: true, rawChainSpecPath: Option.getOrUndefined(rawChainSpecPath2) }; } logger.debug("Precompiling WASM (this may take a moment)..."); const wasmPath = yield* runPrecompile(config.binPath, config.chainArg, cacheSubDir); yield* fs.writeFileString(hashPath, binaryHash).pipe(Effect2.mapError((e) => new StartupCacheError({ cause: e, operation: "cache" }))); const rawChainSpecPath = yield* maybeGetRawChainSpec( config.binPath, chainName, cacheSubDir, config.generateRawChainSpec ?? false ); return { precompiledPath: wasmPath, fromCache: false, rawChainSpecPath: Option.getOrUndefined(rawChainSpecPath) }; }), Duration2.minutes(2) ); }); var StartupCacheServiceLive = Layer.succeed(StartupCacheService, { getCachedArtifacts: (config) => getCachedArtifactsImpl(config).pipe( Effect2.mapError( (e) => e._tag === "FileLockError" ? new StartupCacheError({ cause: e, operation: "lock" }) : e ), Effect2.provide(NodeContext.layer) ) }); var StartupCacheServiceTestable = Layer.succeed(StartupCacheService, { getCachedArtifacts: (config) => getCachedArtifactsImpl(config).pipe( Effect2.mapError( (e) => e._tag === "FileLockError" ? new StartupCacheError({ cause: e, operation: "lock" }) : e ) ) }); // src/internal/commandParsers.ts var logger2 = createLogger2({ name: "commandParsers" }); function parseZombieCmd(launchSpec) { if (launchSpec) { return { cmd: launchSpec.configPath }; } throw new Error( "No ZombieSpec found in config. Are you sure your moonwall.config.json file has the correct 'configPath' in zombieSpec?" ); } function fetchDefaultArgs(binName, additionalRepos = []) { let defaultArgs; const repos = [...standardRepos(), ...additionalRepos]; for (const repo4 of repos) { const foundBin = repo4.binaries.find((bin) => bin.name === binName); if (foundBin) { defaultArgs = foundBin.defaultArgs; break; } } if (!defaultArgs) { defaultArgs = ["--dev"]; } return defaultArgs; } var LaunchCommandParser = class _LaunchCommandParser { args; cmd; launch; launchSpec; launchOverrides; constructor(options) { const { launchSpec, additionalRepos, launchOverrides } = options; this.launchSpec = launchSpec; this.launchOverrides = launchOverrides; this.launch = !launchSpec.running ? true : launchSpec.running; this.cmd = launchSpec.binPath; this.args = launchSpec.options ? [...launchSpec.options] : fetchDefaultArgs(path2.basename(launchSpec.binPath), additionalRepos); } overrideArg(newArg) { const newArgKey = newArg.split("=")[0]; const existingIndex = this.args.findIndex((arg) => arg.startsWith(`${newArgKey}=`)); if (existingIndex !== -1) { this.args[existingIndex] = newArg; } else { this.args.push(newArg); } } async withPorts() { if (process.env.MOON_RECYCLE === "true") { const existingPort = process.env.MOONWALL_RPC_PORT; if (existingPort) { this.overrideArg(`--rpc-port=${existingPort}`); } return this; } if (this.launchSpec.ports) { const ports = this.launchSpec.ports; if (ports.p2pPort) { this.overrideArg(`--port=${ports.p2pPort}`); } if (ports.wsPort) { this.overrideArg(`--ws-port=${ports.wsPort}`); } if (ports.rpcPort) { this.overrideArg(`--rpc-port=${ports.rpcPort}`); } } else { const freePort = (await getFreePort()).toString(); process.env.MOONWALL_RPC_PORT = freePort; this.overrideArg(`--rpc-port=${freePort}`); } return this; } withDefaultForkConfig() { const forkOptions = this.launchSpec.defaultForkConfig; if (forkOptions) { this.applyForkOptions(forkOptions); } return this; } withLaunchOverrides() { if (this.launchOverrides?.forkConfig) { this.applyForkOptions(this.launchOverrides.forkConfig); } return this; } print() { logger2.debug(`Command to run: ${this.cmd}`); logger2.debug(`Arguments: ${this.args.join(" ")}`); return this; } applyForkOptions(forkOptions) { if (forkOptions.url) { invariant(forkOptions.url.startsWith("http"), "Fork URL must start with http:// or https://"); this.overrideArg(`--fork-chain-from-rpc=${forkOptions.url}`); } if (forkOptions.blockHash) { this.overrideArg(`--block=${forkOptions.blockHash}`); } if (forkOptions.stateOverridePath) { this.overrideArg(`--fork-state-overrides=${forkOptions.stateOverridePath}`); } if (forkOptions.verbose) { this.overrideArg("-llazy-loading=trace"); } } /** * Cache startup artifacts if enabled in launchSpec. * This uses an Effect-based service that caches artifacts by binary hash. * * When cacheStartupArtifacts is enabled, this generates: * 1. Precompiled WASM for the runtime * 2. Raw chain spec to skip genesis WASM compilation * * This reduces startup from ~3s to ~200ms (~10x improvement). */ async withStartupCache() { if (!this.launchSpec.cacheStartupArtifacts) { return this; } if (this.launchSpec.useDocker) { logger2.warn("Startup caching is not supported for Docker images, skipping"); return this; } const chainArg = this.args.find((arg) => arg.startsWith("--chain")); const hasDevFlag = this.args.includes("--dev"); const existingChainName = chainArg?.match(/--chain[=\s]?(\S+)/)?.[1]; const canGenerateRawSpec = hasDevFlag || !!existingChainName; const cacheDir = this.launchSpec.startupCacheDir || path2.join(process.cwd(), "tmp", "startup-cache"); const program = StartupCacheService.pipe( Effect3.flatMap( (service) => service.getCachedArtifacts({ binPath: this.launchSpec.binPath, chainArg, cacheDir, // Generate raw chain spec for faster startup (works for both --dev and --chain=XXX) generateRawChainSpec: canGenerateRawSpec, // Pass dev mode flag for proper chain name detection isDevMode: hasDevFlag }) ), Effect3.provide(StartupCacheServiceLive) ); try { const result = await Effect3.runPromise(program); const precompiledDir = path2.dirname(result.precompiledPath); this.overrideArg(`--wasmtime-precompiled=${precompiledDir}`); if (result.rawChainSpecPath) { if (hasDevFlag) { this.args = this.args.filter((arg) => arg !== "--dev"); this.overrideArg(`--chain=${result.rawChainSpecPath}`); this.overrideArg("--alice"); this.overrideArg("--force-authoring"); this.overrideArg("--rpc-cors=all"); this.overrideArg( "--node-key=0000000000000000000000000000000000000000000000000000000000000001" ); } else if (existingChainName) { this.overrideArg(`--chain=${result.rawChainSpecPath}`); } logger2.debug(`Using raw chain spec for ~10x faster startup: ${result.rawChainSpecPath}`); } process.env.MOONWALL_CACHE_DIR = precompiledDir; logger2.debug( result.fromCache ? `Using cached precompiled WASM: ${result.precompiledPath}` : `Precompiled WASM created: ${result.precompiledPath}` ); } catch (error) { logger2.warn(`WASM precompilation failed, continuing without: ${error}`); } return this; } build() { return { cmd: this.cmd, args: this.args, launch: this.launch }; } static async create(options) { const parser = new _LaunchCommandParser(options); const parsed = await parser.withPorts().then((p) => p.withDefaultForkConfig().withLaunchOverrides()).then((p) => p.withStartupCache()); if (options.verbose) { parsed.print(); } return parsed.build(); } }; function parseChopsticksRunCmd(launchSpecs) { const launch = !launchSpecs[0].running ? true : launchSpecs[0].running; if (launchSpecs.length === 1) { const chopsticksCmd2 = "node"; const chopsticksArgs2 = [ "node_modules/@acala-network/chopsticks/chopsticks.cjs", `--config=${launchSpecs[0].configPath}`, `--host=${launchSpecs[0].address ?? "127.0.0.1"}` ]; const mode = launchSpecs[0].buildBlockMode ? launchSpecs[0].buildBlockMode : "manual"; const num = mode === "batch" ? "Batch" : mode === "instant" ? "Instant" : "Manual"; chopsticksArgs2.push(`--build-block-mode=${num}`); if (launchSpecs[0].wsPort) { chopsticksArgs2.push(`--port=${launchSpecs[0].wsPort}`); } if (launchSpecs[0].wasmOverride) { chopsticksArgs2.push(`--wasm-override=${launchSpecs[0].wasmOverride}`); } if (launchSpecs[0].allowUnresolvedImports) { chopsticksArgs2.push("--allow-unresolved-imports"); } return { cmd: chopsticksCmd2, args: chopsticksArgs2, launch }; } const chopsticksCmd = "node"; const chopsticksArgs = ["node_modules/@acala-network/chopsticks/chopsticks.cjs", "xcm"]; for (const spec of launchSpecs) { const type = spec.type ? spec.type : "parachain"; switch (type) { case "parachain": chopsticksArgs.push(`--parachain=${spec.configPath}`); break; case "relaychain": chopsticksArgs.push(`--relaychain=${spec.configPath}`); } } return { cmd: chopsticksCmd, args: chopsticksArgs, launch }; } var isPortAvailable = async (port) => { return new Promise((resolve) => { const server = net.createServer(); server.listen(port, () => { server.once("close", () => resolve(true)); server.close(); }); server.on("error", () => resolve(false)); }); }; var getNextAvailablePort = async (startPort) => { let port = startPort; while (port <= 65535) { if (await isPortAvailable(port)) { return port; } port++; } throw new Error(`No available ports found starting from ${startPort}`); }; var getFreePort = async () => { const shardIndex = shardManager.getShardIndex(); const totalShards = shardManager.getTotalShards(); const poolId = parseInt(process.env.VITEST_POOL_ID || "0", 10); const basePort = 1e4; const shardOffset = shardIndex * 1e3; const poolOffset = poolId * 100; const processOffset = process.pid % 50; const calculatedPort = basePort + shardOffset + poolOffset + processOffset; const startPort = Math.min(calculatedPort, 6e4 + shardIndex * 100 + poolId); logger2.debug( `Port calculation: shard=${shardIndex + 1}/${totalShards}, pool=${poolId}, final=${startPort}` ); return getNextAvailablePort(startPort); }; export { LaunchCommandParser, getFreePort, parseChopsticksRunCmd, parseZombieCmd };