@bsv/overlay-express/src/OverlayExpress.ts

BSV Blockchain Overlay Express

import express from 'express'
import bodyParser from 'body-parser'
import {
  Engine,
  KnexStorage,
  LookupService,
  TopicManager,
  KnexStorageMigrations,
  Advertiser
} from '@bsv/overlay'
import {
  ARC,
  ChainTracker,
  MerklePath,
  STEAK,
  TaggedBEEF,
  WhatsOnChain,
  Broadcaster,
  OverlayBroadcastFacilitator,
  HTTPSOverlayBroadcastFacilitator,
  DEFAULT_TESTNET_SLAP_TRACKERS,
  DEFAULT_SLAP_TRACKERS,
  Utils,
  Beef,
  Transaction,
  PrivateKey,
  KeyDeriver,
  WalletInterface
} from '@bsv/sdk'
import Knex from 'knex'
import { MongoClient, Db } from 'mongodb'
import makeUserInterface, { type UIConfig } from './makeUserInterface.js'
import * as DiscoveryServices from '@bsv/overlay-discovery-services'
import chalk from 'chalk'
import util from 'node:util'
import { v4 as uuidv4 } from 'uuid'
import { JanitorService, type JanitorReport } from './JanitorService.js'
import { BanService } from './BanService.js'
import { BanAwareLookupWrapper } from './BanAwareLookupWrapper.js'
import { BanAwareTopicManager } from './BanAwareTopicManager.js'
import { BanAwareSHIPStorage, BanAwareSLAPStorage } from './BanAwareDiscoveryStorage.js'
import { ReorgSseAdapter, type ReorgHandlerInput } from './ReorgStream.js'
import { Wallet, WalletSigner, WalletStorageManager, Services } from '@bsv/wallet-toolbox-client'
import { createAuthMiddleware, type AuthRequest } from '@bsv/auth-express-middleware'
import { ArcadeProvider, isTerminalArcStatus, type ArcadeMerkleProof } from './ArcadeProvider.js'
import { ProviderChainBroadcaster, type NamedBroadcaster } from './ProviderChainBroadcaster.js'
import { ChaintracksProvider } from './ChaintracksProvider.js'

/**
 * Knex database migration.
 */
interface Migration {
  name?: string
  up: (knex: Knex.Knex) => Promise<void>
  down?: (knex: Knex.Knex) => Promise<void>
}

/**
 * In-memory migration source for Knex migrations.
 * Allows running migrations defined in code rather than files.
 */
class InMemoryMigrationSource implements Knex.Knex.MigrationSource<Migration> {
  constructor (private readonly migrations: Migration[]) { }

  /**
   * Gets the list of migrations.
   * @param loadExtensions - Array of file extensions to filter by (not used here)
   * @returns Promise resolving to the array of migrations
   */
  async getMigrations (loadExtensions: readonly string[]): Promise<Migration[]> {
    return this.migrations
  }

  /**
   * Gets the name of a migration.
   * @param migration - The migration object
   * @returns The name of the migration
   */
  getMigrationName (migration: Migration): string {
    return typeof migration.name === 'string' ? migration.name : `Migration at index ${this.migrations.indexOf(migration)}`
  }

  /**
   * Gets the migration object.
   * @param migration - The migration object
   * @returns Promise resolving to the migration object
   */
  async getMigration (migration: Migration): Promise<Knex.Knex.Migration> {
    return await Promise.resolve(migration)
  }
}

/**
 * Configuration options that map to Engine constructor parameters.
 */
export interface EngineConfig {
  chainTracker?: ChainTracker | 'scripts only'
  shipTrackers?: string[]
  slapTrackers?: string[]
  broadcaster?: Broadcaster
  advertiser?: Advertiser
  syncConfiguration?: Record<string, string[] | 'SHIP' | false>
  logTime?: boolean
  logPrefix?: string
  throwOnBroadcastFailure?: boolean
  overlayBroadcastFacilitator?: OverlayBroadcastFacilitator
  suppressDefaultSyncAdvertisements?: boolean
  topicAnchorHeaderResolver?: TopicAnchorHeaderResolver
  enableBASMSync?: boolean
  unprovenEvictionBlocks?: number
  reorgStreamUrl?: string
  reorgScanDepth?: number
  unprovenMaintenanceIntervalMs?: number
}

export type HealthStatus = 'ok' | 'degraded' | 'error'

export interface HealthCheckResult {
  name: string
  scope: 'live' | 'ready'
  status: HealthStatus
  critical: boolean
  message?: string
  details?: Record<string, any>
  durationMs: number
}

export type HealthCheckHandler = () => Promise<Omit<HealthCheckResult, 'name' | 'scope' | 'critical' | 'durationMs'> | void> | Omit<HealthCheckResult, 'name' | 'scope' | 'critical' | 'durationMs'> | void

export interface HealthCheckDefinition {
  name: string
  scope?: 'live' | 'ready'
  critical?: boolean
  handler: HealthCheckHandler
}

export interface HealthConfig {
  includeDetails: boolean
  timeoutMs: number
  contextProvider?: () => Promise<Record<string, any> | undefined> | Record<string, any> | undefined
}

export interface HealthReport {
  status: HealthStatus
  live: boolean
  ready: boolean
  service: {
    name: string
    advertisableFQDN: string
    port: number
    network: 'main' | 'test'
    startedAt?: string
    uptimeMs: number
    topicManagerCount: number
    lookupServiceCount: number
  }
  checks: HealthCheckResult[]
  context?: Record<string, any>
}

export type TopicAnchorHeaderResolver = (blockHeight: number) => Promise<{
  blockHeight: number
  blockHash: string
  merkleRoot?: string
} | undefined>

interface BASMCapableEngine extends Engine {
  provideTopicAnchorTip: (topic: string) => Promise<any>
  provideTopicAnchorRange: (topic: string, fromHeight: number, toHeight: number) => Promise<any>
  provideAdmittedList: (topic: string, blockHeight: number, blockHash?: string) => Promise<any>
  provideCompoundMerklePath: (topic: string, blockHeight: number, txids: string[]) => Promise<any>
  provideRawTransactions: (txids: string[]) => Promise<any>
  startBASMSync: () => Promise<any>
  advanceTopicAnchorChains: (toHeight?: number) => Promise<void>
  evictUnprovenTransactions: (options?: { topic?: string, thresholdBlocks?: number }) => Promise<any>
  refreshUnprovenTransactionProofs: (options: {
    topic?: string
    thresholdBlocks?: number
    proofProvider: (txid: string) => Promise<{ merklePath: MerklePath, blockHeight?: number } | undefined>
  }) => Promise<any>
  maintainUnprovenTransactions: (options: {
    topic?: string
    thresholdBlocks?: number
    proofProvider: (txid: string) => Promise<{ merklePath: MerklePath, blockHeight?: number } | undefined>
  }) => Promise<any>
  evictAppliedTransaction: (txid: string, options?: { topic?: string, reason?: string }) => Promise<any>
  handleReorg: (input: ReorgHandlerInput) => Promise<any>
  revalidateRecentAnchors: (depth?: number) => Promise<any>
}

function parseTopicsHeader (header: string): string[] {
  const value = header.trim()
  const parsed: unknown = value.startsWith('[')
    ? JSON.parse(value)
    : value.split(',').map(topic => topic.trim())

  if (!Array.isArray(parsed) || parsed.some(topic => typeof topic !== 'string' || topic.length === 0)) {
    throw new TypeError('Invalid x-topics header: expected a comma-separated list or JSON string array')
  }
  return parsed
}

/**
 * OverlayExpress class provides an Express-based server for hosting Overlay Services.
 * It allows configuration of various components like databases, topic managers, and lookup services.
 * It encapsulates an Express application and provides methods to start the server.
 */
export default class OverlayExpress {
  // Express application
  app: express.Application

  // Server port
  port: number = 3000

  // Logger (defaults to console)
  logger: typeof console = console

  // Knex (SQL) database
  knex?: Knex.Knex

  // Knex migrations to run
  migrationsToRun: Migration[] = []

  // MongoDB database
  mongoDb?: Db

  // MongoDB client retained for health checks
  mongoClient?: MongoClient

  // Network ('main' or 'test')
  network: 'main' | 'test' = 'main'

  // If no custom ChainTracker is configured, default is a WhatsOnChain instance
  // (We keep a property for it, so we can pass it to Engine)
  chainTracker: ChainTracker | 'scripts only' = new WhatsOnChain(this.network)

  // The Overlay Engine
  engine?: Engine

  // Configured Topic Managers
  managers: Record<string, TopicManager> = {}

  // Configured Lookup Services
  services: Record<string, LookupService> = {}

  // Enable GASP Sync
  // (We allow an on/off toggle, but also can do advanced custom sync config below)
  enableGASPSync: boolean = true

  // Enable BRC-136 BASM sync. Off by default; endpoints remain available when
  // storage supports anchors.
  enableBASMSync: boolean = false

  // Opt-in unproven eviction default threshold in blocks.
  unprovenEvictionBlocks: number = 144

  // How often (ms) to poll the chain tip and extend each topic's BASM anchor
  // chain with empty anchors, so the cumulative TAC advances "after each new
  // block" per BRC-136. Set to 0 to disable polling (startup extension still runs).
  basmBlockPollIntervalMs: number = 10 * 60 * 1000

  // Handle for the BASM block-poll timer so it can be stopped.
  private basmBlockPollTimer?: ReturnType<typeof setInterval>

  // How often (ms) to refresh proofs for old unproven rows and then evict rows
  // that still have no proof. Set to 0 to disable background maintenance.
  unprovenMaintenanceIntervalMs: number = 0

  private unprovenMaintenanceTimer?: ReturnType<typeof setInterval>

  // Optional go-chaintracks (Arcade) reorg SSE URL (e.g. `<base>/v2/reorg/stream`).
  // When set, reorgs are reconciled in real time; the block poll also runs a
  // revalidation sweep as a fallback / reconnect catch-up.
  reorgStreamUrl?: string

  // Depth (in blocks from the tip) for the reorg revalidation sweep.
  reorgScanDepth: number = 3

  // Handle for the reorg SSE adapter.
  private reorgAdapter?: ReorgSseAdapter

  // Optional resolver for block hashes and header merkle roots used by BASM.
  topicAnchorHeaderResolver?: TopicAnchorHeaderResolver

  // ARC API Key
  arcApiKey: string | undefined = undefined

  // Optional ARC callback token for /arc-ingest notifications
  arcCallbackToken: string | undefined = undefined

  // Optional Arcade URL/API key used for propagation, proof refresh, and
  // go-chaintracks header/reorg access when available.
  arcadeUrl: string | undefined = undefined
  arcadeApiKey: string | undefined = undefined
  arcadeDeploymentId: string | undefined = undefined
  arcadeChaintracksApiPrefix: string = '/chaintracks/v2'

  private arcadeProvider?: ArcadeProvider

  // Verbose request logging
  verboseRequestLogging: boolean = false

  // Web UI configuration
  webUIConfig: UIConfig = {}

  // Additional advanced engine config (these map to Engine constructor parameters).
  // Default to undefined or default values that are used in the Engine if not specified.
  engineConfig: EngineConfig = {}

  // The administrative Bearer token used for the admin routes.
  // If not passed in, we'll generate a random one.
  private readonly adminToken: string

  // Configuration for the janitor service
  janitorConfig: {
    requestTimeoutMs: number
    hostDownRevokeScore: number
    autoBanOnRemoval: boolean
  } = {
      requestTimeoutMs: 10000, // 10 seconds
      hostDownRevokeScore: 3,
      autoBanOnRemoval: true
    }

  // Ban service for persistent domain/outpoint blocking
  banService?: BanService

  // Admin identity key for wallet-based admin detection on the frontend
  adminIdentityKey?: string

  // Server-side wallet (WalletInterface) used for BSV mutual authentication
  serverWallet?: WalletInterface

  // Server start time for uptime tracking
  private startTime?: Date

  // Health endpoint configuration
  healthConfig: HealthConfig = {
    includeDetails: true,
    timeoutMs: 5000
  }

  // Extra application-specific health checks
  healthChecks: HealthCheckDefinition[] = []

  // Lifecycle marker for readiness/liveness reporting
  isListening: boolean = false

  /**
   * Constructs an instance of OverlayExpress.
   * @param name - The name of the service
   * @param privateKey - Private key used for signing advertisements
   * @param advertisableFQDN - The fully qualified domain name where this service is available. Does not include "https://".
   * @param adminToken - Optional. An administrative Bearer token used to protect admin routes.
   *                     If not provided, a random token will be generated at runtime.
   */
  constructor (
    public name: string,
    public privateKey: string,
    public advertisableFQDN: string,
    adminToken?: string
  ) {
    this.app = express()
    this.logger.log(chalk.green.bold(`${name} constructed`))
    this.adminToken = adminToken ?? uuidv4() // generate random if not provided
  }

  /**
   * Returns the current admin token in case you need to programmatically retrieve or display it.
   */
  getAdminToken (): string {
    return this.adminToken
  }

  /**
   * Configures the port on which the server will listen.
   * @param port - The port number
   */
  configurePort (port: number): void {
    this.port = port
    this.logger.log(chalk.blue(`Server port set to ${port}`))
  }

  /**
   * Configures the web user interface
   * @param config - Web UI configuration options
   */
  configureWebUI (config: UIConfig): void {
    this.webUIConfig = config
    this.logger.log(chalk.blue('Web UI has been configured.'))
  }

  /**
   * Configures the janitor service parameters
   * @param config - Janitor configuration options
   *   - requestTimeoutMs: Timeout for health check requests (default: 10000ms)
   *   - hostDownRevokeScore: Number of consecutive failures before deleting output (default: 3)
   *   - autoBanOnRemoval: Whether to auto-ban domains when removed by janitor (default: true)
   */
  configureJanitor (config: Partial<typeof this.janitorConfig>): void {
    this.janitorConfig = {
      ...this.janitorConfig,
      ...config
    }
    this.logger.log(chalk.blue('Janitor service has been configured.'))
  }

  /**
   * Configures health-report behavior.
   */
  configureHealth (config: Partial<HealthConfig>): void {
    this.healthConfig = {
      ...this.healthConfig,
      ...config
    }
    this.logger.log(chalk.blue('Health reporting has been configured.'))
  }

  /**
   * Registers an application-specific health check.
   */
  registerHealthCheck (definition: HealthCheckDefinition): void {
    this.healthChecks = this.healthChecks.filter(check => check.name !== definition.name)
    this.healthChecks.push({
      scope: 'ready',
      critical: false,
      ...definition
    })
    this.logger.log(chalk.blue(`Registered health check ${definition.name}`))
  }

  /**
   * Configures the admin identity key for wallet-based admin detection.
   * When set, the frontend can compare the user's wallet identity key against this
   * to determine whether to show the admin dashboard.
   *
   * @param identityKey - The hex-encoded public key of the admin
   */
  configureAdminIdentityKey (identityKey: string): void {
    this.adminIdentityKey = identityKey
    this.logger.log(chalk.blue('Admin identity key has been configured.'))
  }

  /**
   * Configures the logger to be used by the server.
   * @param logger - A logger object (e.g., console)
   */
  configureLogger (logger: typeof console): void {
    this.logger = logger
    this.logger.log(chalk.blue('Logger has been configured.'))
  }

  /**
   * Configures the BSV Blockchain network to be used ('main' or 'test').
   * By default, it re-initializes chainTracker as a WhatsOnChain for that network.
   * @param network - The network ('main' or 'test')
   */
  configureNetwork (network: 'main' | 'test'): void {
    this.network = network
    this.chainTracker = new WhatsOnChain(this.network)
    this.logger.log(chalk.blue(`Network set to ${network}`))
  }

  /**
   * Configures the ChainTracker to be used.
   * If 'scripts only' is used, it implies no full SPV chain tracking in the Engine.
   * @param chainTracker - An instance of ChainTracker or 'scripts only'
   */
  configureChainTracker (chainTracker: ChainTracker | 'scripts only' = new WhatsOnChain(this.network)): void {
    this.chainTracker = chainTracker
    this.logger.log(chalk.blue('ChainTracker has been configured.'))
  }

  /**
   * Configures the ARC API key.
   * @param apiKey - The ARC API key
   */
  configureArcApiKey (apiKey: string): void {
    this.arcApiKey = apiKey
    this.logger.log(chalk.blue('ARC API key has been configured.'))
  }

  /**
   * Configures the ARC callback token expected by /arc-ingest.
   * @param token - The token ARC should present when posting callback notifications.
   */
  configureArcCallbackToken (token: string): void {
    this.arcCallbackToken = token
    this.logger.log(chalk.blue('ARC callback token has been configured.'))
  }

  /**
   * Configures Arcade for first-choice transaction propagation and proof lookup.
   */
  configureArcade (url: string, config: {
    apiKey?: string
    deploymentId?: string
    chaintracksApiPrefix?: string
  } = {}): void {
    this.arcadeUrl = url
    this.arcadeApiKey = config.apiKey
    this.arcadeDeploymentId = config.deploymentId
    if (config.chaintracksApiPrefix !== undefined) {
      this.arcadeChaintracksApiPrefix = config.chaintracksApiPrefix
    }
    this.logger.log(chalk.blue('Arcade provider has been configured.'))
  }

  /**
   * Configures a go-chaintracks compatible service for header validation and
   * BASM reorg streaming. Arcade exposes this at `/chaintracks/v2`.
   */
  configureChaintracks (url: string, config: {
    apiPrefix?: string
    reorgStream?: boolean
    scanDepth?: number
  } = {}): void {
    const apiPrefix = config.apiPrefix ?? '/chaintracks/v2'
    const client = new ChaintracksProvider(url, { apiPrefix })
    this.configureChainTracker(client)
    this.configureTopicAnchorHeaderResolver(async blockHeight => {
      const header = await client.findHeaderForHeight(blockHeight)
      if (header === undefined) return undefined
      return {
        blockHeight,
        blockHash: header.hash,
        merkleRoot: header.merkleRoot
      }
    })
    if (config.reorgStream !== false) {
      this.configureReorgStream(client.reorgStreamUrl(), config.scanDepth)
    }
    this.logger.log(chalk.blue('go-chaintracks provider has been configured.'))
  }

  /**
   * Enables or disables GASP synchronization (high-level setting).
   * This is a broad toggle that can be overridden or customized through syncConfiguration.
   * @param enable - true to enable, false to disable
   */
  configureEnableGASPSync (enable: boolean): void {
    this.enableGASPSync = enable
    this.logger.log(chalk.blue(`GASP synchronization ${enable ? 'enabled' : 'disabled'}.`))
  }

  /**
   * Enables or disables BRC-136 BASM synchronization.
   * BASM is opt-in because it requires direct proofs and block hash resolution.
   */
  configureEnableBASMSync (enable: boolean): void {
    this.enableBASMSync = enable
    this.logger.log(chalk.blue(`BASM synchronization ${enable ? 'enabled' : 'disabled'}.`))
  }

  /**
   * Configures the block header resolver used to derive BASM block hashes.
   */
  configureTopicAnchorHeaderResolver (resolver: TopicAnchorHeaderResolver): void {
    this.topicAnchorHeaderResolver = resolver
    this.logger.log(chalk.blue('BASM topic anchor header resolver has been configured.'))
  }

  /**
   * Configures the go-chaintracks (Arcade) reorg SSE stream used to reconcile
   * BASM anchors with blockchain reorganizations in real time.
   * @param url - The reorg stream URL, e.g. `https://arcade.example/v2/reorg/stream`.
   * @param scanDepth - Optional revalidation-sweep depth in blocks (default 3).
   */
  configureReorgStream (url: string, scanDepth?: number): void {
    this.reorgStreamUrl = url
    if (scanDepth !== undefined) {
      this.reorgScanDepth = scanDepth
    }
    this.logger.log(chalk.blue('BASM reorg stream has been configured.'))
  }

  /**
   * Configures the opt-in unproven state eviction threshold.
   */
  configureUnprovenEviction (config: { thresholdBlocks?: number }): void {
    if (config.thresholdBlocks !== undefined) {
      this.unprovenEvictionBlocks = config.thresholdBlocks
    }
    this.logger.log(chalk.blue('Unproven transaction eviction has been configured.'))
  }

  /**
   * Configures periodic unproven maintenance. Each run first tries configured
   * proof providers, then evicts rows that are still unproven past the threshold.
   */
  configureUnprovenMaintenance (config: { intervalMs?: number, thresholdBlocks?: number }): void {
    if (config.intervalMs !== undefined) {
      this.unprovenMaintenanceIntervalMs = config.intervalMs
    }
    if (config.thresholdBlocks !== undefined) {
      this.unprovenEvictionBlocks = config.thresholdBlocks
    }
    this.logger.log(chalk.blue('Unproven transaction maintenance has been configured.'))
  }

  /**
   * Configures how often the BASM anchor chain is extended with empty anchors to
   * follow the chain tip. Set to 0 to disable periodic polling.
   */
  configureBASMBlockPollInterval (intervalMs: number): void {
    this.basmBlockPollIntervalMs = intervalMs
    this.logger.log(chalk.blue(`BASM block poll interval set to ${intervalMs}ms.`))
  }

  /**
   * Enables or disables verbose request logging.
   * @param enable - true to enable, false to disable
   */
  configureVerboseRequestLogging (enable: boolean): void {
    this.verboseRequestLogging = enable
    this.logger.log(chalk.blue(`Verbose request logging ${enable ? 'enabled' : 'disabled'}.`))
  }

  /**
   * Configure Knex (SQL) database connection.
   * @param config - Knex configuration object, or a MySQL connection string loaded from configuration.
   */
  async configureKnex (config: Knex.Knex.Config | string): Promise<void> {
    if (typeof config === 'string') {
      config = {
        client: 'mysql2',
        connection: config
      }
    }
    this.knex = Knex(config)
    this.logger.log(chalk.blue('Knex successfully configured.'))
  }

  /**
   * Configures the MongoDB database connection.
   * Also initializes the BanService for persistent ban tracking.
   * @param connectionString - MongoDB connection string
   */
  async configureMongo (connectionString: string): Promise<void> {
    const mongoClient = new MongoClient(connectionString)
    await mongoClient.connect()
    this.mongoClient = mongoClient
    const db = mongoClient.db(`${this.name}_lookup_services`)
    this.mongoDb = db

    // Initialize the BanService
    this.banService = new BanService(db)
    await this.banService.ensureIndexes()

    this.logger.log(chalk.blue('MongoDB successfully configured and connected.'))
  }

  /**
   * Configures a Topic Manager.
   * @param name - The name of the Topic Manager
   * @param manager - An instance of TopicManager
   */
  configureTopicManager (name: string, manager: TopicManager): void {
    this.managers[name] = manager
    this.logger.log(chalk.blue(`Configured topic manager ${name}`))
  }

  /**
   * Configures a Lookup Service.
   * @param name - The name of the Lookup Service
   * @param service - An instance of LookupService
   */
  configureLookupService (name: string, service: LookupService): void {
    this.services[name] = service
    this.logger.log(chalk.blue(`Configured lookup service ${name}`))
  }

  /**
   * Configures a Lookup Service using Knex (SQL) database.
   * @param name - The name of the Lookup Service
   * @param serviceFactory - A factory function that creates a LookupService instance using Knex
   */
  configureLookupServiceWithKnex (
    name: string,
    serviceFactory: (knex: Knex.Knex) => { service: LookupService, migrations: Migration[] }
  ): void {
    const knex = this.ensureKnex()
    const factoryResult = serviceFactory(knex)
    this.services[name] = factoryResult.service
    this.migrationsToRun.push(...factoryResult.migrations)
    this.logger.log(chalk.blue(`Configured lookup service ${name} with Knex`))
  }

  /**
   * Configures a Lookup Service using MongoDB.
   * @param name - The name of the Lookup Service
   * @param serviceFactory - A factory function that creates a LookupService instance using MongoDB
   */
  configureLookupServiceWithMongo (name: string, serviceFactory: (mongoDb: Db) => LookupService): void {
    const mongoDb = this.ensureMongo()
    this.services[name] = serviceFactory(mongoDb)
    this.logger.log(chalk.blue(`Configured lookup service ${name} with MongoDB`))
  }

  /**
   * Advanced configuration method for setting or overriding any
   * Engine constructor parameters via an EngineConfig object.
   *
   * Example usage:
   *   configureEngineParams({
   *     logTime: true,
   *     throwOnBroadcastFailure: true,
   *     overlayBroadcastFacilitator: new MyCustomFacilitator()
   *   })
   *
   * These fields will be respected when we finally build/configure the Engine
   * in the `configureEngine()` method below.
   */
  configureEngineParams (params: EngineConfig): void {
    this.engineConfig = {
      ...this.engineConfig,
      ...params
    }
    this.logger.log(chalk.blue('Advanced Engine configuration params have been updated.'))
  }

  /**
   * Configures the Overlay Engine itself.
   * By default, auto-configures SHIP and SLAP unless autoConfigureShipSlap = false
   * Then it merges in any advanced engine config from `this.engineConfig`.
   *
   * When a BanService is available (from configureMongo), auto-configured SHIP
   * and SLAP managers, discovery storage, and lookup services are wrapped so
   * banned outputs are not admitted or indexed.
   *
   * @param autoConfigureShipSlap - Whether to auto-configure SHIP and SLAP services (default: true)
   */
  async configureEngine (autoConfigureShipSlap = true): Promise<void> {
    const knex = this.ensureKnex()

    if (autoConfigureShipSlap) {
      const mongoDb = this.ensureMongo()
      const shipStorage = new DiscoveryServices.SHIPStorage(mongoDb)
      const slapStorage = new DiscoveryServices.SLAPStorage(mongoDb)

      // Run the one-time discovery migration before the engine can accept
      // traffic, so a failed unique-index build is a visible startup failure.
      await shipStorage.ensureIndexes()
      await slapStorage.ensureIndexes()

      this.configureTopicManager('tm_ship', new DiscoveryServices.SHIPTopicManager())
      this.configureTopicManager('tm_slap', new DiscoveryServices.SLAPTopicManager())

      const shipStorageForLookup = this.banService === undefined
        ? shipStorage
        : new BanAwareSHIPStorage(shipStorage, this.banService, this.logger)
      const slapStorageForLookup = this.banService === undefined
        ? slapStorage
        : new BanAwareSLAPStorage(slapStorage, this.banService, this.logger)

      this.services.ls_ship = new DiscoveryServices.SHIPLookupService(shipStorageForLookup as any)
      this.services.ls_slap = new DiscoveryServices.SLAPLookupService(slapStorageForLookup as any)
      this.logger.log(chalk.blue('Configured lookup service ls_ship with MongoDB'))
      this.logger.log(chalk.blue('Configured lookup service ls_slap with MongoDB'))
    }

    this.wrapBanAwareServices()

    const syncConfig = this.buildSyncConfig()
    const storage = new KnexStorage(knex)
    this.migrationsToRun = [...KnexStorageMigrations.default, ...this.migrationsToRun]

    const broadcaster = this.buildBroadcaster()
    const advertiser = await this.buildAdvertiser()

    const EngineWithBASM = Engine as unknown as new (...args: any[]) => Engine
    this.engine = new EngineWithBASM(
      this.managers,
      this.services,
      storage,
      this.engineConfig.chainTracker ?? this.chainTracker,
      `https://${this.advertisableFQDN}`,
      this.network === 'test'
        ? (this.engineConfig.shipTrackers ?? DEFAULT_TESTNET_SLAP_TRACKERS)
        : this.engineConfig.shipTrackers,
      this.resolveSlapTrackers(),
      broadcaster ?? this.engineConfig.broadcaster,
      advertiser,
      syncConfig,
      this.engineConfig.logTime ?? false,
      this.engineConfig.logPrefix ?? '[OVERLAY_ENGINE] ',
      this.engineConfig.throwOnBroadcastFailure ?? true,
      this.engineConfig.overlayBroadcastFacilitator ?? new HTTPSOverlayBroadcastFacilitator(),
      this.logger,
      this.engineConfig.suppressDefaultSyncAdvertisements ?? true,
      this.buildTopicAnchorHeaderResolver(),
      this.engineConfig.enableBASMSync ?? this.enableBASMSync,
      this.engineConfig.unprovenEvictionBlocks ?? this.unprovenEvictionBlocks
    )

    this.initServerWallet()
    this.logger.log(chalk.green('Engine has been configured.'))
  }

  /** Wrap SHIP/SLAP managers and services with ban-aware filters if BanService is configured. */
  private wrapBanAwareServices (): void {
    if (this.banService === undefined) return
    for (const key of ['tm_ship', 'tm_slap'] as const) {
      if (this.managers[key] !== undefined) {
        const label = key === 'tm_ship' ? 'SHIP' : 'SLAP'
        this.managers[key] = new BanAwareTopicManager(this.managers[key], this.banService, label, this.logger)
        this.logger.log(chalk.blue(`${label} topic manager wrapped with ban-aware filter.`))
      }
    }
    for (const key of ['ls_ship', 'ls_slap'] as const) {
      if (this.services[key] !== undefined) {
        const label = key === 'ls_ship' ? 'SHIP' : 'SLAP'
        this.services[key] = new BanAwareLookupWrapper(this.services[key], this.banService, label, this.logger)
        this.logger.log(chalk.blue(`${label} lookup service wrapped with ban-aware filter.`))
      }
    }
  }

  /** Build the sync config based on enableGASPSync and engineConfig. */
  private buildSyncConfig (): Record<string, string[] | 'SHIP' | false> {
    if (this.enableGASPSync) {
      return this.engineConfig.syncConfiguration ?? {}
    }
    const syncConfig: Record<string, string[] | 'SHIP' | false> = {}
    for (const name of Object.keys(this.managers)) {
      syncConfig[name] = false
    }
    return syncConfig
  }

  /** Build the configured transaction propagation provider chain. */
  private buildBroadcaster (): Broadcaster | undefined {
    const providers: NamedBroadcaster[] = []
    const callbackUrl = `https://${this.advertisableFQDN}/arc-ingest`

    if (typeof this.arcadeUrl === 'string' && this.arcadeUrl.length > 0) {
      this.arcadeProvider = new ArcadeProvider(this.arcadeUrl, {
        apiKey: this.arcadeApiKey,
        callbackUrl,
        callbackToken: this.arcCallbackToken,
        deploymentId: this.arcadeDeploymentId
      })
      providers.push({
        name: 'Arcade',
        broadcaster: this.arcadeProvider
      })
    } else {
      this.arcadeProvider = undefined
    }

    if (typeof this.arcApiKey === 'string' && this.arcApiKey.length > 0) {
      const arcUrl = this.network === 'test' ? 'https://arc-test.taal.com' : 'https://arc.taal.com'
      providers.push({
        name: 'ARC',
        broadcaster: new ARC(arcUrl, {
          apiKey: this.arcApiKey,
          callbackUrl,
          callbackToken: this.arcCallbackToken
        })
      })
    }

    if (providers.length === 0) return undefined
    if (providers.length === 1) return providers[0].broadcaster
    return new ProviderChainBroadcaster(providers)
  }

  private ensureArcadeProvider (): ArcadeProvider | undefined {
    if (this.arcadeProvider !== undefined) return this.arcadeProvider
    if (typeof this.arcadeUrl !== 'string' || this.arcadeUrl.length === 0) return undefined
    this.arcadeProvider = new ArcadeProvider(this.arcadeUrl, {
      apiKey: this.arcadeApiKey,
      callbackUrl: `https://${this.advertisableFQDN}/arc-ingest`,
      callbackToken: this.arcCallbackToken,
      deploymentId: this.arcadeDeploymentId
    })
    return this.arcadeProvider
  }

  private async fetchArcadeProof (txid: string): Promise<ArcadeMerkleProof | undefined> {
    const provider = this.ensureArcadeProvider()
    if (provider === undefined) return undefined
    const proof = await provider.fetchMerkleProof(txid)
    if (proof === undefined) return undefined
    const chainTracker = this.engineConfig.chainTracker ?? this.chainTracker
    if (chainTracker === 'scripts only') {
      throw new Error('Cannot validate Arcade proof with scripts-only chain tracker')
    }
    const blockHeight = proof.blockHeight ?? proof.merklePath.blockHeight
    if (blockHeight === undefined) {
      throw new Error(`Arcade proof for ${txid} did not include a block height`)
    }
    const valid = await chainTracker.isValidRootForHeight(proof.merkleRoot, blockHeight)
    if (!valid) {
      throw new Error(`Arcade proof for ${txid} did not match the chain tracker at height ${blockHeight}`)
    }
    return {
      ...proof,
      blockHeight
    }
  }

  private async fetchConfiguredMerkleProof (txid: string): Promise<{ merklePath: MerklePath, blockHeight?: number } | undefined> {
    const proof = await this.fetchArcadeProof(txid)
    if (proof === undefined) return undefined
    return {
      merklePath: proof.merklePath,
      blockHeight: proof.blockHeight
    }
  }

  /** Build the BASM block header resolver. */
  private buildTopicAnchorHeaderResolver (): TopicAnchorHeaderResolver | undefined {
    const configured = this.engineConfig.topicAnchorHeaderResolver ?? this.topicAnchorHeaderResolver
    if (configured !== undefined) {
      return configured
    }

    return async (blockHeight: number) => {
      const response = await fetch(`https://api.whatsonchain.com/v1/bsv/${this.network}/block/${blockHeight}/header`, {
        method: 'GET',
        headers: { Accept: 'application/json' }
      })
      if (!response.ok) {
        throw new Error(`WhatsOnChain header lookup failed for height ${blockHeight}: ${response.status}`)
      }
      const header = await response.json() as { hash?: string, merkleroot?: string }
      if (typeof header.hash !== 'string') {
        throw new TypeError(`WhatsOnChain did not return a block hash for height ${blockHeight}`)
      }
      return {
        blockHeight,
        blockHash: header.hash,
        merkleRoot: header.merkleroot
      }
    }
  }

  /** Resolve the SLAP trackers from config or network defaults. */
  private resolveSlapTrackers (): string[] | undefined {
    if (Array.isArray(this.engineConfig.slapTrackers)) return this.engineConfig.slapTrackers
    return this.network === 'test' ? DEFAULT_TESTNET_SLAP_TRACKERS : DEFAULT_SLAP_TRACKERS
  }

  /** Build the WalletAdvertiser (or use user-provided one). */
  private async buildAdvertiser (): Promise<Advertiser | undefined> {
    if (this.engineConfig.advertiser !== undefined) return this.engineConfig.advertiser
    const storageBase = this.network === 'test'
      ? 'https://staging-storage.babbage.systems'
      : 'https://storage.babbage.systems'
    try {
      return new DiscoveryServices.WalletAdvertiser(
        this.network,
        this.privateKey,
        storageBase,
        `https://${this.advertisableFQDN}`
      )
    } catch (e) {
      this.logger.log(`Advertiser not initialized for FQDN ${this.advertisableFQDN} - SHIP and SLAP will be disabled. Reason: ${e}`)
      return undefined
    }
  }

  /** Initialize the server wallet for BSV mutual authentication. */
  private initServerWallet (): void {
    try {
      const keyDeriver = new KeyDeriver(new PrivateKey(this.privateKey, 'hex'))
      const storageManager = new WalletStorageManager(keyDeriver.identityKey)
      const signer = new WalletSigner(this.network, keyDeriver, storageManager)
      const services = new Services(this.network)
      this.serverWallet = new Wallet(signer, services)
      this.adminIdentityKey ??= keyDeriver.identityKey
      this.logger.log(chalk.blue('Server wallet initialized for BSV mutual authentication.'))
    } catch (e) {
      this.logger.log(chalk.yellow(`Server wallet could not be initialized. BSV auth will not be available. Reason: ${e}`))
    }
  }

  /**
   * Ensures that Knex is configured and returns it.
   * @throws Error if Knex is not configured
   */
  private ensureKnex (): Knex.Knex {
    if (this.knex === undefined) {
      throw new TypeError('You must configure your SQL database with the .configureKnex() method first!')
    }
    return this.knex
  }

  /**
   * Ensures that MongoDB is configured and returns it.
   * @throws Error if MongoDB is not configured
   */
  private ensureMongo (): Db {
    if (this.mongoDb === undefined) {
      throw new TypeError('You must configure your MongoDB connection with the .configureMongo() method first!')
    }
    return this.mongoDb
  }

  /**
   * Ensures that the Overlay Engine is configured and returns it.
   * @throws Error if the Engine is not configured
   */
  private ensureEngine (): Engine {
    if (this.engine === undefined) {
      throw new TypeError('You must configure your Overlay Services engine with the .configureEngine() method first!')
    }
    return this.engine
  }

  /**
   * Creates a JanitorService instance with current configuration.
   */
  private createJanitor (): JanitorService {
    const mongoDb = this.ensureMongo()
    return new JanitorService({
      mongoDb,
      logger: this.logger,
      requestTimeoutMs: this.janitorConfig.requestTimeoutMs,
      hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore,
      banService: this.banService,
      autoBanOnRemoval: this.janitorConfig.autoBanOnRemoval
    })
  }

  /** Ban a domain and remove all its SHIP/SLAP records from MongoDB. */
  private async handleBanDomain (res: express.Response, value: string, reason?: string): Promise<express.Response> {
    await this.banService!.banDomain(value, reason)
    const db = this.ensureMongo()
    const [shipDeleted, slapDeleted] = await Promise.all([
      db.collection('shipRecords').deleteMany({ domain: value }),
      db.collection('slapRecords').deleteMany({ domain: value })
    ])
    return res.status(200).json({
      status: 'success',
      message: `Domain "${value}" banned. Removed ${shipDeleted.deletedCount} SHIP and ${slapDeleted.deletedCount} SLAP records.`
    })
  }

  /** Parse outpoint string, ban it, and evict it from all lookup services. */
  private async handleBanOutpoint (res: express.Response, engine: Engine, value: string, reason?: string): Promise<express.Response> {
    const dotIndex = value.lastIndexOf('.')
    if (dotIndex === -1) {
      return res.status(400).json({ status: 'error', message: 'Outpoint format must be "txid.outputIndex"' })
    }
    const txid = value.substring(0, dotIndex)
    const outputIndex = Number.parseInt(value.substring(dotIndex + 1))
    if (Number.isNaN(outputIndex)) {
      return res.status(400).json({ status: 'error', message: 'Invalid outputIndex in outpoint' })
    }
    await this.banService!.banOutpoint(txid, outputIndex, reason)
    await this.evictFromServices(engine, txid, outputIndex)
    return res.status(200).json({
      status: 'success',
      message: `Outpoint "${value}" banned and evicted from lookup services.`
    })
  }

  /** Evict an output from a specific service or all services (silent per-service errors). */
  private async evictFromServices (engine: Engine, txid: string, outputIndex: number, service?: string): Promise<void> {
    if (typeof service === 'string') {
      const svc = engine.lookupServices[service]
      if (svc !== undefined) await svc.outputEvicted(txid, outputIndex)
      return
    }
    for (const svc of Object.values(engine.lookupServices)) {
      try { await svc.outputEvicted(txid, outputIndex) } catch { /* best-effort */ }
    }
  }

  /** Look up the domain of an outpoint from SHIP or SLAP records. */
  private async lookupDomainForOutpoint (txid: string, outputIndex: number): Promise<string | undefined> {
    const db = this.ensureMongo()
    const [shipRecord, slapRecord] = await Promise.all([
      db.collection('shipRecords').findOne({ txid, outputIndex }),
      db.collection('slapRecords').findOne({ txid, outputIndex })
    ])
    return (shipRecord?.domain ?? slapRecord?.domain) as string | undefined
  }

  /** Ban a domain and delete all SHIP/SLAP records for it. */
  private async banDomainAndRemoveRecords (domain: string, reason: string): Promise<void> {
    await this.banService!.banDomain(domain, reason)
    const db = this.ensureMongo()
    await Promise.all([
      db.collection('shipRecords').deleteMany({ domain }),
      db.collection('slapRecords').deleteMany({ domain })
    ])
  }

  private async runHealthCheck (
    definition: Required<Pick<HealthCheckDefinition, 'name' | 'scope' | 'critical'>> & { handler: HealthCheckHandler }
  ): Promise<HealthCheckResult> {
    const startedAt = Date.now()

    try {
      const result = ((await Promise.race([
        Promise.resolve(definition.handler()),
        new Promise<never>((_, reject) => {
          setTimeout(() => reject(new Error(`Timed out after ${this.healthConfig.timeoutMs}ms`)), this.healthConfig.timeoutMs)
        })
      ])) ?? {}) as {
        status?: HealthStatus
        message?: string
        details?: Record<string, any>
      }

      return {
        name: definition.name,
        scope: definition.scope,
        critical: definition.critical,
        status: result.status ?? 'ok',
        message: result.message,
        details: result.details,
        durationMs: Date.now() - startedAt
      }
    } catch (error) {
      return {
        name: definition.name,
        scope: definition.scope,
        critical: definition.critical,
        status: 'error',
        message: error instanceof Error ? error.message : 'Unknown health-check error',
        durationMs: Date.now() - startedAt
      }
    }
  }

  private async collectHealthReport (mode: 'live' | 'ready' | 'full'): Promise<HealthReport> {
    const definitions: Array<Required<Pick<HealthCheckDefinition, 'name' | 'scope' | 'critical'>> & { handler: HealthCheckHandler }> = [
      {
        name: 'process',
        scope: 'live',
        critical: true,
        handler: async () => ({
          status: 'ok',
          details: {
            listening: this.isListening
          }
        })
      },
      {
        name: 'engine',
        scope: 'ready',
        critical: true,
        handler: async () => {
          if (this.engine === undefined) {
            throw new TypeError('Overlay engine is not configured')
          }

          return {
            status: 'ok',
            details: {
              topicManagers: Object.keys(this.managers),
              lookupServices: Object.keys(this.services)
            }
          }
        }
      },
      {
        name: 'knex',
        scope: 'ready',
        critical: true,
        handler: async () => {
          if (this.knex === undefined) {
            throw new TypeError('Knex is not configured')
          }

          await this.knex.raw('select 1 as ok')
          return {
            status: 'ok',
            details: {
              client: this.knex.client?.config?.client ?? 'unknown'
            }
          }
        }
      },
      {
        name: 'mongo',
        scope: 'ready',
        critical: true,
        handler: async () => {
          if (this.mongoDb === undefined) {
            throw new TypeError('MongoDB is not configured')
          }

          await this.mongoDb.command({ ping: 1 })
          return {
            status: 'ok',
            details: {
              database: this.mongoDb.databaseName
            }
          }
        }
      }
    ]

    for (const check of this.healthChecks) {
      definitions.push({
        name: check.name,
        scope: check.scope ?? 'ready',
        critical: check.critical ?? false,
        handler: check.handler
      })
    }

    const filteredDefinitions = definitions.filter((definition) => {
      if (mode === 'full') {
        return true
      }

      return definition.scope === mode
    })

    const checks = await Promise.all(filteredDefinitions.map(async definition => await this.runHealthCheck(definition)))
    const liveChecks = checks.filter(check => check.scope === 'live')
    const readyChecks = checks.filter(check => check.scope === 'ready')
    const live = liveChecks.every(check => !check.critical || check.status === 'ok')
    const ready = readyChecks.every(check => !check.critical || check.status === 'ok')

    let status: HealthStatus = 'ok'
    if (!live || !ready || checks.some(check => check.critical && check.status === 'error')) {
      status = 'error'
    } else if (checks.some(check => check.status !== 'ok')) {
      status = 'degraded'
    }

    const context = typeof this.healthConfig.contextProvider === 'function'
      ? await this.healthConfig.contextProvider()
      : undefined

    const report: HealthReport = {
      status,
      live,
      ready,
      service: {
        name: this.name,
        advertisableFQDN: this.advertisableFQDN,
        port: this.port,
        network: this.network,
        startedAt: this.startTime?.toISOString(),
        uptimeMs: this.startTime === undefined ? 0 : Date.now() - this.startTime.getTime(),
        topicManagerCount: Object.keys(this.managers).length,
        lookupServiceCount: Object.keys(this.services).length
      },
      checks: this.healthConfig.includeDetails
        ? checks
        : checks.map(({ details, ...check }) => check),
      context
    }

    return report
  }

  /**
   * Renders a request or response body for verbose logging, truncating overly long payloads.
   */
  private formatBodyForLog (body: any, tooLongLabel: string, okPrefix: string): string {
    let bodyString: string
    if (typeof body === 'object') {
      bodyString = JSON.stringify(body, null, 2)
    } else if (Buffer.isBuffer(body)) {
      bodyString = body.toString('utf8')
    } else {
      bodyString = String(body)
    }
    return bodyString.length > 280
      ? chalk.yellow(`(${tooLongLabel}, length: ${String(bodyString.length)} characters)`)
      : chalk.green(`${okPrefix}\n${bodyString}`)
  }

  /**
   * Installs middleware that verbosely logs incoming requests and outgoing responses.
   */
  private setupVerboseRequestLogging (): void {
    this.app.use((req, res, next) => {
      const startTime = Date.now()

      // Log incoming request details
      this.logger.log(chalk.magenta.bold(`Incoming Request: ${String(req.method)} ${String(req.originalUrl)}`))
      // Pretty-print headers
      this.logger.log(chalk.cyan('Headers:'))
      this.logger.log(util.inspect(req.headers, { colors: true, depth: null }))

      // Handle request body
      if (req.body != null && Object.keys(req.body).length > 0) {
        this.logger.log(this.formatBodyForLog(req.body, 'Body too long to display', 'Request Body:'))
      }

      // Intercept the res.send method to log responses
      const originalSend = res.send
      let responseBody: any

      res.send = function (body?: any): any {
        responseBody = body
        return originalSend.call(this, body)
      }

      // Log outgoing response details after the response is finished
      res.on('finish', () => {
        const duration = Date.now() - startTime
        this.logger.log(
          chalk.magenta.bold(
            `Outgoing Response: ${String(req.method)} ${String(req.originalUrl)} - Status: ${String(res.statusCode)} - Duration: ${String(duration)}ms`
          )
        )
        this.logger.log(chalk.cyan('Response Headers:'))
        this.logger.log(util.inspect(res.getHeaders(), { colors: true, depth: null }))

        // Handle response body
        if (responseBody != null) {
          this.logger.log(this.formatBodyForLog(responseBody, 'Response body too long to display', 'Response Body:'))
        }
      })

      next()
    })
  }

  /**
   * Starts the Express server.
   * Sets up routes and begins listening on the configured port.
   */
  async start (): Promise<void> {
    const engine = this.ensureEngine()
    const knex = this.ensureKnex()
    this.startTime = new Date()

    this.app.use(bodyParser.json({ limit: '1gb', type: 'application/json' }))
    this.app.use(bodyParser.raw({ limit: '1gb', type: 'application/octet-stream' }))

    if (this.verboseRequestLogging) {
      this.setupVerboseRequestLogging()
    }

    // Enable CORS
    this.app.use((req, res, next) => {
      res.header('Access-Control-Allow-Origin', '*')
      res.header('Access-Control-Allow-Headers', '*')
      res.header('Access-Control-Allow-Methods', '*')
      res.header('Access-Control-Expose-Headers', '*')
      res.header('Access-Control-Allow-Private-Network', 'true')
      if (req.method === 'OPTIONS') {
        res.sendStatus(200)
      } else {
        next()
      }
    })

    // Serve a static documentation site or user interface
    this.app.get('/', (req, res) => {
      res.set('content-type', 'text/html')
      res.send(makeUserInterface({
        ...this.webUIConfig,
        adminIdentityKey: this.adminIdentityKey
      }))
    })

    // Serve health check endpoints
    this.app.get('/health/live', (_, res) => {
      ; (async () => {
        const report = await this.collectHealthReport('live')
        return res.status(report.live ? 200 : 503).json(report)
      })().catch((error) => {
        res.status(500).json({
          status: 'error',
          message: error instanceof Error ? error.message : 'Unexpected error'
        })
      })
    })

    this.app.get('/health/ready', (_, res) => {
      ; (async () => {
        const report = await this.collectHealthReport('ready')
        return res.status(report.ready ? 200 : 503).json(report)
      })().catch((error) => {
        res.status(500).json({
          status: 'error',
          message: error instanceof Error ? error.message : 'Unexpected error'
        })
      })
    })

    this.app.get('/health', (_, res) => {
      ; (async () => {
        const report = await this.collectHealthReport('full')
        return res.status(report.ready ? 200 : 503).json(report)
      })().catch((error) => {
        res.status(500).json({
          status: 'error',
          message: error instanceof Error ? error.message : 'Unexpected error'
        })
      })
    })

    // List hosted topic managers and lookup services
    this.app.get('/listTopicManagers', (_, res) => {
      ; (async () => {
        try {
          const result = await engine.listTopi