@bsv/overlay/src/Engine.ts

BSV Blockchain Overlay Services Engine

import { TopicManager } from './TopicManager.js'
import { LookupService } from './LookupService.js'
import { Storage } from './storage/Storage.js'
import type { Output } from './Output.js'
import {
  Transaction,
  ChainTracker,
  MerklePath,
  Broadcaster,
  isBroadcastFailure,
  TaggedBEEF, STEAK,
  LookupQuestion,
  LookupAnswer,
  AdmittanceInstructions,
  SHIPBroadcaster,
  HTTPSOverlayBroadcastFacilitator,
  LookupResolver,
  LookupResolverConfig,
  OverlayBroadcastFacilitator,
  BroadcastResponse,
  BroadcastFailure
} from '@bsv/sdk'
import { AdvertisementData, Advertiser } from './Advertiser.js'
import { GASP, GASPInitialRequest, GASPInitialResponse, GASPNode } from '@bsv/gasp'
import { SyncConfiguration } from './SyncConfiguration.js'
import { OverlayGASPRemote } from './GASP/OverlayGASPRemote.js'
import { OverlayGASPStorage } from './GASP/OverlayGASPStorage.js'
import {
  BASM_ZERO_HASH,
  type AdmittedListResponse,
  type BASMPeerSyncReport,
  type CompoundMerklePathResponse,
  type RawTransactionResponse,
  type ReorgReport,
  type TopicAnchorHeaderResolver,
  type TopicAnchorRangeResponse,
  type TopicAnchorTip,
  type TopicBlockAnchor,
  computeBasmRoot,
  computeTac,
  extractMerkleProofMetadata
} from './BASM.js'
import { BASMRemote } from './BASMRemote.js'

const DEFAULT_GASP_SYNC_LIMIT = 10000
const DEFAULT_BASM_RANGE_LIMIT = 1024

type UTXOHistoryHydrationContext = {
  outputCache: Map<string, Promise<Output | null>>
}

type HydratedUTXOHistoryNode = {
  output: Output
  transaction: Transaction
}

/**
 * An engine for running BSV Overlay Services (topic managers and lookup services).
 */
export class Engine {
  /**
   * Creates a new Overlay Services Engine
   * @param {[key: string]: TopicManager} managers - manages topic admittance
   * @param {[key: string]: LookupService} lookupServices - manages UTXO lookups
   * @param {Storage} storage - for interacting with internally-managed persistent data
   * @param {ChainTracker | 'scripts only'} chainTracker - Verifies SPV data associated with transactions
   * @param {string} [hostingURL] - The URL this engine is hosted at. Required if going to support peer-discovery with an advertiser.
   * @param {Broadcaster} [Broadcaster] - broadcaster used for broadcasting the incoming transaction
   * @param {Advertiser} [Advertiser] - handles SHIP and SLAP advertisements for peer-discovery
   * @param {string[]} shipTrackers - SHIP domains we know to bootstrap the system
   * @param {string[]} slapTrackers - SLAP domains we know to bootstrap the system
   * @param {SyncConfiguration} syncConfiguration — Configuration object describing historical synchronization of topics.
   * @param {boolean} logTime - Enables / disables the timing logs for various operations in the Overlay submit route.
   * @param {string} logPrefix - Supports overriding the log prefix with a custom string.
   * @param {boolean} throwOnBroadcastFailure - Enables / disables throwing an error when a transaction broadcast failure is detected.
   * @param {OverlayBroadcastFacilitator} overlayBroadcastFacilitator - Facilitator for propagation to other Overlay Services.
   * @param {typeof console} logger - The place where log entries are written.
   * @param {boolean} suppressDefaultSyncAdvertisements - Whether to suppress the default (SHIP/SLAP) sync advertisements.
   * @param {TopicAnchorHeaderResolver} topicAnchorHeaderResolver - Resolves block hashes for BASM anchors.
   * @param {boolean} basmSyncEnabled - Whether BASM sync should run automatically.
   * @param {number} unprovenEvictionBlocks - Default block age for opt-in unproven state eviction.
   */
  constructor(
    public managers: { [key: string]: TopicManager },
    public lookupServices: { [key: string]: LookupService },
    public storage: Storage,
    public chainTracker: ChainTracker | 'scripts only',
    public hostingURL?: string,
    public shipTrackers?: string[],
    public slapTrackers?: string[],
    public broadcaster?: Broadcaster,
    public advertiser?: Advertiser,
    public syncConfiguration?: SyncConfiguration,
    public logTime = false,
    public logPrefix = '[OVERLAY_ENGINE] ',
    public throwOnBroadcastFailure = false,
    public overlayBroadcastFacilitator: OverlayBroadcastFacilitator = new HTTPSOverlayBroadcastFacilitator(),
    public logger: typeof console = console,
    public suppressDefaultSyncAdvertisements = true,
    public topicAnchorHeaderResolver?: TopicAnchorHeaderResolver,
    public basmSyncEnabled = false,
    public unprovenEvictionBlocks = 144
  ) {
    // To encourage synchronization of overlay services, the SHIP sync strategy is used by default for all overlay topics, except for 'tm_ship' and 'tm_slap'.
    // For these two topics, any existing trackers are combined with the provided shipTrackers and slapTrackers omitting any duplicates.
    if (syncConfiguration === undefined) {
      this.syncConfiguration = {}
    } else {
      this.syncConfiguration = syncConfiguration
    }

    for (const managerName of Object.keys(managers)) {
      if (managerName === 'tm_ship' && this.shipTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
        // Combine tm_ship trackers with preexisting entries if any
        const combinedSet = new Set([
          ...(Array.isArray(this.syncConfiguration[managerName]) ? this.syncConfiguration[managerName] : []),
          ...this.shipTrackers
        ])
        this.syncConfiguration[managerName] = Array.from(combinedSet)
      } else if (managerName === 'tm_slap' && this.slapTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
        // Combine tm_slap trackers with preexisting entries if any
        const combinedSet = new Set([
          ...(Array.isArray(this.syncConfiguration[managerName]) ? this.syncConfiguration[managerName] : []),
          ...this.slapTrackers
        ])
        this.syncConfiguration[managerName] = Array.from(combinedSet)
      } else {
        // Set undefined managers to 'SHIP' by default
        this.syncConfiguration[managerName] ??= 'SHIP'
      }
    }
  }

  // Helper functions for logging timings
  private startTime(label: string): void {
    if (this.logTime) {
      this.logger.time(`${this.logPrefix} ${label}`)
    }
  }

  private endTime(label: string): void {
    if (this.logTime) {
      this.logger.timeEnd(`${this.logPrefix} ${label}`)
    }
  }

  private async currentHeightOrUndefined(): Promise<number | undefined> {
    if (this.chainTracker === 'scripts only') {
      return undefined
    }
    try {
      return await this.chainTracker.currentHeight()
    } catch (error) {
      this.logger.warn(`Unable to resolve current chain height for overlay metadata: ${error instanceof Error ? error.message : String(error)}`)
      return undefined
    }
  }

  private async resolveBlockHash(blockHeight: number, merkleRoot?: string): Promise<string | undefined> {
    try {
      const header = await this.topicAnchorHeaderResolver?.(blockHeight)
      if (header === undefined) {
        return undefined
      }
      if (header.merkleRoot !== undefined && merkleRoot !== undefined && header.merkleRoot !== merkleRoot) {
        throw new Error(`Header merkle root ${header.merkleRoot} does not match proof root ${merkleRoot} at height ${blockHeight}`)
      }
      return header.blockHash
    } catch (error) {
      this.logger.warn(`Unable to resolve BASM block hash for height ${blockHeight}: ${error instanceof Error ? error.message : String(error)}`)
      return undefined
    }
  }

  private compactBEEFForStorage(tx: Transaction, originalBEEF: number[]): number[] {
    return tx.merklePath === undefined ? originalBEEF : tx.toAtomicBEEF()
  }

  private async recordTransactionData(tx: Transaction, beef: number[], blockHash?: string): Promise<void> {
    if (typeof this.storage.upsertTransactionRecord !== 'function') {
      return
    }

    const txid = tx.id('hex')
    const metadata = extractMerkleProofMetadata(txid, tx.merklePath)
    await this.storage.upsertTransactionRecord({
      txid,
      beef: this.compactBEEFForStorage(tx, beef),
      rawTx: Array.from(tx.toBinary()),
      merklePath: tx.merklePath?.toBinary(),
      blockHeight: metadata?.blockHeight,
      blockHash,
      blockIndex: metadata?.blockIndex,
      merkleRoot: metadata?.merkleRoot
    })
  }

  private async buildAppliedTransactionRecord(tx: Transaction): Promise<{
    blockHeight?: number
    blockHash?: string
    blockIndex?: number
    merkleRoot?: string
    firstSeenHeight?: number
    proven: boolean
  }> {
    const txid = tx.id('hex')
    const metadata = extractMerkleProofMetadata(txid, tx.merklePath)
    const [firstSeenHeight, blockHash] = await Promise.all([
      this.currentHeightOrUndefined(),
      metadata === undefined ? undefined : this.resolveBlockHash(metadata.blockHeight, metadata.merkleRoot)
    ])

    return {
      blockHeight: metadata?.blockHeight,
      blockHash,
      blockIndex: metadata?.blockIndex,
      merkleRoot: metadata?.merkleRoot,
      firstSeenHeight: firstSeenHeight ?? metadata?.blockHeight,
      proven: metadata !== undefined
    }
  }

  private async recomputeTopicBlockAnchor(topic: string, blockHeight: number, blockHash?: string): Promise<TopicBlockAnchor | undefined> {
    if (
      typeof this.storage.findAdmittedTransactionsForBlock !== 'function' ||
      typeof this.storage.upsertTopicBlockAnchor !== 'function' ||
      typeof this.storage.findTopicBlockAnchor !== 'function'
    ) {
      return undefined
    }

    const anchorBlockHash = blockHash ?? (await this.storage.findTopicBlockAnchor(topic, blockHeight))?.blockHash
    if (anchorBlockHash === undefined) {
      return undefined
    }

    // BRC-136 per-block completeness: establish the chain's genesis at the first
    // admitted height, then keep every height from there to the tip contiguous so
    // the cumulative TAC never resets across blocks with no admitted transactions.
    // We rebuild [fromHeight, toHeight] rather than only the touched height so that
    // an out-of-order proof (older height arriving after a newer one) can never
    // leave a gap that silently breaks the chain.
    const tip = await this.storage.findTopicAnchorTip?.(topic)
    const tipHeight = tip !== undefined && tip.blockHeight >= 0 ? tip.blockHeight : undefined
    const fromHeight = tipHeight === undefined ? blockHeight : Math.min(blockHeight, tipHeight + 1)
    const toHeight = tipHeight === undefined ? blockHeight : Math.max(blockHeight, tipHeight)

    await this.rebuildTopicAnchorChain(topic, fromHeight, toHeight, new Map([[blockHeight, anchorBlockHash]]))
    return await this.storage.findTopicBlockAnchor(topic, blockHeight)
  }

  /**
   * Extends every configured topic's anchor chain forward with empty Topic Block
   * Anchors (basmRoot = zero hash, admittedCount = 0) up to `toHeight`, so the
   * cumulative TAC advances on every block even when a topic admits nothing —
   * this is what lets a peer authoritatively confirm "this block contained no
   * transactions for this topic". Chains with no first admission yet are left
   * unstarted (genesis is the topic's first admitted height).
   */
  async advanceTopicAnchorChains(toHeight?: number): Promise<void> {
    if (
      typeof this.storage.findTopicAnchorTip !== 'function' ||
      typeof this.storage.upsertTopicBlockAnchor !== 'function'
    ) {
      return
    }
    const targetHeight = toHeight ?? await this.currentHeightOrUndefined()
    if (targetHeight === undefined) {
      return
    }
    for (const topic of Object.keys(this.managers)) {
      const tip = await this.storage.findTopicAnchorTip(topic)
      if (tip === undefined || tip.blockHeight < 0 || tip.blockHeight >= targetHeight) {
        continue
      }
      await this.rebuildTopicAnchorChain(topic, tip.blockHeight + 1, targetHeight)
    }
  }

  /**
   * Rebuilds a contiguous slice of a topic's anchor chain over [fromHeight,
   * toHeight]. Each height uses its admitted transactions (empty -> zero basmRoot)
   * and chains the cumulative TAC from the prior height. Missing heights are
   * filled rather than skipped, so the chain stays gap-free. If a block hash
   * cannot be resolved for some height the extension halts there to preserve
   * contiguity instead of leaving a hole.
   */
  private async rebuildTopicAnchorChain(
    topic: string,
    fromHeight: number,
    toHeight: number,
    blockHashHints: Map<number, string> = new Map(),
    forceResolve = false
  ): Promise<void> {
    if (
      typeof this.storage.findAdmittedTransactionsForBlock !== 'function' ||
      typeof this.storage.upsertTopicBlockAnchor !== 'function' ||
      typeof this.storage.findTopicBlockAnchor !== 'function' ||
      toHeight < fromHeight
    ) {
      return
    }

    if (toHeight - fromHeight + 1 > DEFAULT_BASM_RANGE_LIMIT) {
      // Bound the work per pass; the next trigger resumes from the new tip.
      this.logger.warn(`[BASM] capping anchor chain extension for "${topic}" at ${DEFAULT_BASM_RANGE_LIMIT} blocks (requested ${fromHeight}..${toHeight}); will continue on the next pass`)
      toHeight = fromHeight + DEFAULT_BASM_RANGE_LIMIT - 1
    }

    const previousAnchor = fromHeight > 0
      ? await this.storage.findTopicBlockAnchor(topic, fromHeight - 1)
      : undefined
    let prevTac = previousAnchor?.tac ?? BASM_ZERO_HASH

    for (let height = fromHeight; height <= toHeight; height++) {
      const admitted = await this.storage.findAdmittedTransactionsForBlock(topic, height)
      const existing = await this.storage.findTopicBlockAnchor(topic, height)
      // On a reorg rebuild the existing anchor's block hash is stale, so force
      // canonical re-resolution from the header resolver instead of reusing it.
      const blockHash = blockHashHints.get(height) ?? (forceResolve ? undefined : existing?.blockHash) ?? await this.resolveBlockHash(height)
      if (blockHash === undefined) {
        this.logger.warn(`[BASM] unable to resolve block hash for "${topic}" at height ${height}; halting chain extension`)
        return
      }

      const basmRoot = computeBasmRoot(admitted)
      const tac = computeTac(prevTac, blockHash, basmRoot)
      await this.storage.upsertTopicBlockAnchor({
        topic,
        blockHeight: height,
        blockHash,
        basmRoot,
        admittedCount: admitted.length,
        tac
      })
      prevTac = tac
    }
  }

  /**
   * Reconciles BASM anchors with a blockchain reorganization reported by the
   * chain tracker (e.g. go-chaintracks `/v2/reorg/stream`). Proven topic
   * transactions whose block was orphaned are demoted to unproven so they leave
   * the admitted set, then every topic anchor chain intersecting the affected
   * height range is rebuilt over the canonical block hashes. A reorg changes the
   * canonical block hash for the affected heights, so topics with no demoted
   * transaction are rebuilt too. Idempotent: a clean window demotes nothing and
   * reproduces an identical TAC, so this is safe to invoke on every reorg event,
   * SSE reconnect, and poll.
   */
  async handleReorg(input: {
    orphanedBlockHashes: string[]
    rebuildFromHeight: number
    newTipHeight: number
  }): Promise<ReorgReport> {
    const report: ReorgReport = { perTopic: [] }
    if (
      typeof this.storage.findProvenAppliedTransactionsByBlockHash !== 'function' ||
      typeof this.storage.demoteAppliedTransactionToUnproven !== 'function' ||
      typeof this.storage.findTopicBlockAnchors !== 'function' ||
      typeof this.storage.upsertTopicBlockAnchor !== 'function'
    ) {
      return report
    }

    // 1) Demote proven admissions whose block was orphaned. Hashes are
    //    normalized to lower-case display hex to match stored block hashes
    //    (go-sdk chainhash.Hash marshals as reversed display hex).
    const demotedByTopic = new Map<string, string[]>()
    for (const rawHash of input.orphanedBlockHashes) {
      const blockHash = rawHash.toLowerCase()
      const rows = await this.storage.findProvenAppliedTransactionsByBlockHash(blockHash)
      for (const row of rows) {
        await this.storage.demoteAppliedTransactionToUnproven(row.txid, row.topic)
        const list = demotedByTopic.get(row.topic) ?? []
        list.push(row.txid)
        demotedByTopic.set(row.topic, list)
      }
    }

    // 2) Rebuild every topic anchor chain that intersects the reorged range,
    //    forcing canonical block-hash re-resolution so stale hashes are replaced.
    for (const topic of Object.keys(this.managers)) {
      const existing = await this.storage.findTopicBlockAnchors(topic, input.rebuildFromHeight, input.newTipHeight)
      if (existing.length === 0) {
        continue
      }
      const startHeight = Math.min(...existing.map(anchor => anchor.blockHeight))
      await this.rebuildTopicAnchorChain(topic, startHeight, input.newTipHeight, new Map(), true)
      report.perTopic.push({
        topic,
        demotedTxids: demotedByTopic.get(topic) ?? [],
        rebuiltFrom: startHeight,
        rebuiltTo: input.newTipHeight
      })
    }

    return report
  }

  /**
   * Revalidation sweep: the reorg fallback for chain trackers without a reorg
   * event stream, and the catch-up step on every reorg-SSE (re)connect (the
   * go-chaintracks reorg stream carries no event ids, so a reconnect cannot
   * replay events missed while disconnected). Scans proven applied transactions
   * in `[tip - depth + 1, tip]`; any whose proof root no longer validates against
   * the chain tracker, or whose block hash diverges from the canonical header, is
   * treated as orphaned and reconciled via {@link handleReorg}.
   */
  private async isProvenAnchorStale(
    row: { txid: string, blockHeight: number, blockHash?: string, merkleRoot?: string },
    chainTracker: ChainTracker
  ): Promise<boolean | undefined> {
    if (row.merkleRoot !== undefined) {
      try {
        if (!(await chainTracker.isValidRootForHeight(row.merkleRoot, row.blockHeight))) {
          return true
        }
      } catch (error) {
        this.logger.warn(`[BASM] root validation failed for ${row.txid} at height ${row.blockHeight}: ${error instanceof Error ? error.message : String(error)}`)
        return undefined
      }
    }

    const canonical = await this.resolveBlockHash(row.blockHeight)
    return canonical !== undefined && canonical.toLowerCase() !== row.blockHash?.toLowerCase()
  }

  async revalidateRecentAnchors(depth = 3): Promise<ReorgReport | undefined> {
    const chainTracker = this.chainTracker
    if (chainTracker === 'scripts only') {
      this.logger.warn('[BASM] revalidation sweep requires a ChainTracker; skipping')
      return undefined
    }
    if (typeof this.storage.findProvenAppliedTransactionsInRange !== 'function') {
      return undefined
    }
    const tip = await this.currentHeightOrUndefined()
    if (tip === undefined) {
      return undefined
    }

    const fromHeight = Math.max(0, tip - depth + 1)
    const rows = await this.storage.findProvenAppliedTransactionsInRange(fromHeight, tip)
    const orphaned = new Set<string>()
    let minAffected = Number.POSITIVE_INFINITY

    for (const row of rows) {
      if (row.blockHash === undefined) {
        continue
      }
      const stale = await this.isProvenAnchorStale(row, chainTracker)
      if (stale === true) {
        orphaned.add(row.blockHash.toLowerCase())
        minAffected = Math.min(minAffected, row.blockHeight)
      }
    }

    if (orphaned.size === 0) {
      return { perTopic: [] }
    }

    return await this.handleReorg({
      orphanedBlockHashes: Array.from(orphaned),
      rebuildFromHeight: minAffected,
      newTipHeight: tip
    })
  }

  /**
   * Submits a transaction for processing by Overlay Services.
   * @param {TaggedBEEF} taggedBEEF - The transaction to process
   * @param {function(STEAK): void} [onSTEAKReady] - Optional callback function invoked when the STEAK is ready.
   * @param {string} mode — Indicates the submission behavior, whether historical or current. Historical transactions are not broadcast or propagated.
   * @param {number[]} offChainValues — Values necessary to evaluate topical admittance that are not stored on-chain.
   *
   * The optional callback function should be used to get STEAK when ready, and avoid waiting for broadcast and transaction propagation to complete.
   *
   * @returns {Promise<STEAK>} The submitted transaction execution acknowledgement
   */
  async submit(taggedBEEF: TaggedBEEF, onSteakReady?: (steak: STEAK) => void, mode: 'historical-tx' | 'current-tx' | 'historical-tx-no-spv' = 'current-tx', offChainValues?: number[]): Promise<STEAK> {
    for (const t of taggedBEEF.topics) {
      if (this.managers[t] === undefined || this.managers[t] === null) {
        throw new Error(`This server does not support this topic: ${t}`)
      }
    }

    // Validate the transaction SPV information
    const tx = Transaction.fromBEEF(taggedBEEF.beef)
    const txid = tx.id('hex')

    this.startTime(`submit_${txid}`)
    if (mode !== 'historical-tx-no-spv') {
      this.startTime(`chainTracker_${txid.substring(0, 10)}`)
      const txValid = await tx.verify(this.chainTracker)
      if (!txValid) throw new Error('Unable to verify SPV information.')
      this.endTime(`chainTracker_${txid.substring(0, 10)}`)
    }

    const steak: STEAK = {}
    const dupeTopics = new Set<string>()
    const failedTopics = new Set<string>()

    // ===================================================================
    // PHASE 1: VALIDATE (read-only, no mutations)
    // ===================================================================
    type TopicValidation = {
      topic: string
      isDupe: boolean
      previousCoins: number[]
      previousOutputs: Array<Output | null>
      admissibleOutputs: AdmittanceInstructions
    }

    const topicValidations = taggedBEEF.topics.map(async (topic): Promise<TopicValidation> => {
      try {
        if (this.managers[topic] === undefined || this.managers[topic] === null) {
          throw new Error(`This server does not support this topic: ${topic}`)
        }

        // Check for duplicate transactions
        this.startTime(`dupCheck_${txid.substring(0, 10)}`)
        const dupeCheck = await this.storage.doesAppliedTransactionExist({ txid, topic })
        this.endTime(`dupCheck_${txid.substring(0, 10)}`)

        if (dupeCheck) {
          dupeTopics.add(topic)
          return {
            topic,
            isDupe: true,
            previousCoins: [],
            previousOutputs: [],
            admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
          }
        }

        // Identify previous coins admitted to this specific topic
        const previousCoins: number[] = []
        const outputPromises = tx.inputs.map(async (input, i) => {
          const previousTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex')
          if (previousTXID !== undefined) {
            // Check if the previous output was admitted to this specific topic
            const output = await this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic)
            if (output !== undefined && output !== null) {
              previousCoins.push(i)
              return output
            }
          }
          return null
        })

        this.startTime(`previousOutputQuery_${txid.substring(0, 10)}`)
        const previousOutputs = await Promise.all(outputPromises)
        this.endTime(`previousOutputQuery_${txid.substring(0, 10)}`)

        // Determine which outputs are admissible for this topic (validation only)
        this.startTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`)
        const admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(
          taggedBEEF.beef,
          previousCoins,
          offChainValues,
          mode
        )
        this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`)

        return {
          topic,
          isDupe: false,
          previousCoins,
          previousOutputs,
          admissibleOutputs
        }
      } catch (error) {
        this.logger.error('Error validating topic during submit:', error)
        failedTopics.add(topic)
        return {
          topic,
          isDupe: false,
          previousCoins: [],
          previousOutputs: [],
          admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
        }
      }
    })

    const validations = await Promise.all(topicValidations)

    // Build preliminary STEAK from validation results
    for (const validation of validations) {
      steak[validation.topic] = validation.admissibleOutputs
    }

    // ===================================================================
    // PHASE 2: BROADCAST (before any mutations)
    // ===================================================================
    // Only broadcast when at least one topic actually accepted the
    // transaction. For a non-failed topic, acceptance means: previously
    // accepted (dupe / client retry), outputs admitted, coins retained, or
    // previously-admitted coins consumed (e.g. a consume-only deletion such
    // as a KVStore remove, even one that retains nothing). A topic manager
    // REJECTS by throwing from identifyAdmissibleOutputs (tracked in
    // failedTopics). A transaction every topic rejected must never reach the
    // network: submitters treat an empty STEAK as a rejection and
    // abort/release their held inputs, so broadcasting it anyway would
    // desync their wallets from the chain.
    const anyTopicAccepted = validations.some(v =>
      !failedTopics.has(v.topic) && (
        v.isDupe ||
        v.admissibleOutputs.outputsToAdmit.length > 0 ||
        v.admissibleOutputs.coinsToRetain.length > 0 ||
        v.previousCoins.length > 0
      )
    )
    this.startTime(`broadcast_${txid.substring(0, 10)}`)
    if (mode !== 'historical-tx' && this.broadcaster !== undefined && anyTopicAccepted) {
      try {
        let response: BroadcastResponse | BroadcastFailure
        if (tx.merklePath !== undefined) {
          // tx has been verified, thus if there is a merklePath, the transaction is already on-chain...skip broadcast.
          const txid = tx.id('hex')
          const mp = tx.merklePath
          const leaf = mp.path[0].find(leaf => leaf.hash === txid)
          const r: BroadcastResponse = {
            status: 'success',
            txid: tx.id('hex'),
            message: `In block at height ${mp.blockHeight} index ${leaf?.offset}`,
          }
          response = r
        } else {
          response = await this.broadcaster.broadcast(tx)
        }
        if (isBroadcastFailure(response) && this.throwOnBroadcastFailure) {
          const e = new Error(`Failed to broadcast transaction! Error: ${response.description}`)
            ; (e as any).more = response.more
          throw e
        }
      } catch (error) {
        if (this.throwOnBroadcastFailure) {
          throw error
        }
        this.logger.error('Error broadcasting transaction:', error)
      }
    }
    this.endTime(`broadcast_${txid.substring(0, 10)}`)

    // Call the callback function with STEAK if it is provided (before storage mutations)
    if (onSteakReady !== undefined) {
      onSteakReady(steak)
    }

    // ===================================================================
    // PHASE 3: MUTATE STORAGE (only after broadcast succeeded)
    // ===================================================================
    // Mark previous outputs as spent and notify lookup services
    await Promise.all(validations.map(async (validation) => {
      if (validation.isDupe || failedTopics.has(validation.topic)) {
        return
      }

      const topic = validation.topic
      const previousOutputs = validation.previousOutputs

      // Mark all previous outputs as spent
      const markSpentPromises = previousOutputs.map(async (output) => {
        if (output !== undefined && output !== null) {
          try {
            await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic)
            await Promise.all(Object.values(this.lookupServices).map(async l => {
              try {
                if (typeof l.outputSpent === 'function') {
                  if (l.spendNotificationMode === 'txid') {
                    await l.outputSpent({
                      mode: 'txid',
                      spendingTxid: txid,
                      txid: output.txid,
                      outputIndex: output.outputIndex,
                      topic
                    })
                  } else if (l.spendNotificationMode === 'script') {
                    const inputIndex = tx.inputs.findIndex(i => {
                      let realSource = i.sourceTXID
                      if (!realSource) {
                        realSource = i.sourceTransaction?.id('hex')
                      }
                      return realSource === output.txid && i.sourceOutputIndex === output.outputIndex
                    })
                    if (inputIndex === -1) {
                      throw new Error('Could not find input index')
                    }
                    await l.outputSpent({
                      mode: 'script',
                      spendingTxid: txid,
                      inputIndex,
                      sequenceNumber: tx.inputs[inputIndex].sequence ?? 0xffffffff,
                      unlockingScript: tx.inputs[inputIndex].unlockingScript!,
                      txid: output.txid,
                      outputIndex: output.outputIndex,
                      topic,
                      offChainValues
                    })
                  } else if (l.spendNotificationMode === 'whole-tx') {
                    await l.outputSpent({
                      mode: 'whole-tx',
                      spendingAtomicBEEF: tx.toAtomicBEEF(),
                      txid: output.txid,
                      outputIndex: output.outputIndex,
                      topic,
                      offChainValues
                    })
                  } else { // none
                    await l.outputSpent({
                      mode: 'none',
                      txid: output.txid,
                      outputIndex: output.outputIndex,
                      topic
                    })
                  }
                }
              } catch (error) {
                this.logger.error('Error in lookup service for outputSpent:', error)
              }
            }))
          } catch (error) {
            this.logger.error('Error marking UTXO as spent:', error)
          }
        }
      })

      await Promise.all(markSpentPromises)
    }))

    // Continue with storage updates and lookup service notifications
    for (const validation of validations) {
      const topic = validation.topic
      if (dupeTopics.has(topic)) {
        continue
      }
      if (failedTopics.has(topic)) {
        continue
      }
      try {
        const admissibleOutputs = steak[topic]
        const outputsToAdmit: number[] = admissibleOutputs.outputsToAdmit
        const outputsConsumed: Array<{
          txid: string
          outputIndex: number
        }> = []

        const outputsToMarkStale: Array<{
          txid: string
          previousOutputIndex: number
          inputIndex: number
        }> = []

        // Use previousCoins from validation
        const previousCoins = validation.previousCoins

        // For each of the previous UTXOs for this topic, if the UTXO was not included in the list of UTXOs identified for retention, then it will be marked as stale.
        for (const inputIndex of previousCoins) {
          const previousTXID = tx.inputs[inputIndex].sourceTXID ?? tx.inputs[inputIndex].sourceTransaction?.id('hex')
          if (typeof previousTXID !== 'string') continue
          const previousOutputIndex = tx.inputs[inputIndex].sourceOutputIndex
          if (admissibleOutputs.coinsToRetain.includes(inputIndex)) {
            outputsConsumed.push({
              txid: previousTXID,
              outputIndex: previousOutputIndex
            })
          } else {
            outputsToMarkStale.push({
              txid: previousTXID,
              previousOutputIndex,
              inputIndex
            })
          }
        }

        // Remove stale outputs recursively
        this.startTime(`lookForStaleOutputs_${txid.substring(0, 10)}`)
        await Promise.all(outputsToMarkStale.map(async coin => {
          const output = await this.storage.findOutput(coin.txid, coin.previousOutputIndex, topic)
          if (output !== undefined && output !== null) {
            await this.deleteUTXODeep(output)
          }
        }))
        this.endTime(`lookForStaleOutputs_${txid.substring(0, 10)}`)

        // Update the STEAK to indicate which coins were removed
        steak[topic].coinsRemoved = outputsToMarkStale.map(x => x.inputIndex)

        // Handle admittance and notification of incoming UTXOs
        const newUTXOs: Array<{ txid: string, outputIndex: number }> = []
        await Promise.all(outputsToAdmit.map(async outputIndex => {
          if (typeof tx.outputs[outputIndex].satoshis !== 'number') return
          this.startTime(`insertNewOutput_${txid.substring(0, 10)}`)
          await this.storage.insertOutput({
            txid,
            outputIndex,
            outputScript: tx.outputs[outputIndex].lockingScript.toBinary(),
            satoshis: tx.outputs[outputIndex].satoshis,
            topic,
            spent: false,
            beef: this.compactBEEFForStorage(tx, taggedBEEF.beef),
            consumedBy: [],
            outputsConsumed,
            score: Date.now(),
            blockHeight: extractMerkleProofMetadata(txid, tx.merklePath)?.blockHeight
          })
          this.endTime(`insertNewOutput_${txid.substring(0, 10)}`)
          newUTXOs.push({ txid, outputIndex })

          this.startTime(`notifyLookupService${txid.substring(0, 10)}`)
          await Promise.all(Object.values(this.lookupServices).map(async l => {
            try {
              if (l.admissionMode === 'locking-script') {
                if (
                  typeof tx.outputs[outputIndex].lockingScript !== 'object' ||
                  typeof tx.outputs[outputIndex].satoshis !== 'number'
                ) {
                  return
                }
                await l.outputAdmittedByTopic({
                  mode: 'locking-script',
                  txid,
                  outputIndex,
                  lockingScript: tx.outputs[outputIndex].lockingScript,
                  satoshis: tx.outputs[outputIndex].satoshis,
                  topic,
                  offChainValues
                })
              } else {
                await l.outputAdmittedByTopic({
                  mode: 'whole-tx',
                  atomicBEEF: tx.toAtomicBEEF(),
                  outputIndex,
                  topic,
                  offChainValues
                })
              }
            } catch (error) {
              this.logger.error('Error in lookup service for outputAdmittedByTopic:', error)
            }
          }))
          this.endTime(`notifyLookupService${txid.substring(0, 10)}`)
        }))

        this.startTime(`outputConsumed_${txid.substring(0, 10)}`)
        // Update each output consumed to know who consumed it and insert applied transaction in parallel
        const appliedRecord = await this.buildAppliedTransactionRecord(tx)
        await this.recordTransactionData(tx, taggedBEEF.beef, appliedRecord.blockHash)

        await Promise.all([
          ...outputsConsumed.map(async output => {
            const outputToUpdate = await this.storage.findOutput(output.txid, output.outputIndex, topic)
            if (outputToUpdate !== undefined && outputToUpdate !== null) {
              const newConsumedBy = [...new Set([...newUTXOs, ...outputToUpdate.consumedBy])]
              await this.storage.updateConsumedBy(output.txid, output.outputIndex, topic, newConsumedBy)
            }
          }),
          this.storage.insertAppliedTransaction({
            txid,
            topic,
            ...appliedRecord
          })
        ])

        if (appliedRecord.blockHeight !== undefined && appliedRecord.blockHash !== undefined) {
          await this.recomputeTopicBlockAnchor(topic, appliedRecord.blockHeight, appliedRecord.blockHash)
        }
        this.endTime(`outputConsumed_${txid.substring(0, 10)}`)
      } catch (error) {
        this.logger.error('Error updating storage and notifying lookup services for topic', topic, error)
      }
    }

    // If we don't have an advertiser or we are dealing with historical transactions, just return the steak
    if (this.advertiser === undefined || mode === 'historical-tx' || mode === 'historical-tx-no-spv') {
      return steak
    }

    this.startTime(`transactionPropagation_${txid.substring(0, 10)}`)
    const relevantTopics = taggedBEEF.topics.filter(topic =>
      steak[topic] !== undefined && !dupeTopics.has(topic) && (steak[topic].outputsToAdmit.length !== 0 || steak[topic].coinsRemoved?.length !== 0)
    )

    if (relevantTopics.length === 0) {
      this.endTime(`transactionPropagation_${txid.substring(0, 10)}`)
      return steak
    }

    // Create a SHIPBroadcaster instance
    let customBroadcasterConfig
    if (Array.isArray(this.slapTrackers)) {
      // Custom SLAP trackers warrant a custom broadcaster config
      const resolverConfig: LookupResolverConfig = {
        slapTrackers: this.slapTrackers
      }
      customBroadcasterConfig = {
        resolver: new LookupResolver(resolverConfig)
      }
    }
    const shipBroadcaster = new SHIPBroadcaster(relevantTopics, customBroadcasterConfig)

    try {
      await shipBroadcaster.broadcast(tx)
    } catch (error) {
      this.logger.error('Error during propagation to other nodes:', error)
    }
    this.endTime(`transactionPropagation_${txid.substring(0, 10)}`)

    // Immediately return from the function without waiting for the promises to resolve.
    return steak
  }

  /**
   * Submit a lookup question to the Overlay Services Engine, and receive back a Lookup Answer
   * @param LookupQuestion — The question to ask the Overlay Services Engine
   * @returns The answer to the question
   */
  async lookup(lookupQuestion: LookupQuestion): Promise<LookupAnswer> {
    // Validate a lookup service for the provider is found
    const lookupService = this.lookupServices[lookupQuestion.service]
    if (lookupService === undefined || lookupService === null) throw new Error(`Lookup service not found for provider: ${lookupQuestion.service} `)

    const lookupResult = await lookupService.lookup(lookupQuestion)
    const hydrationContext = this.createUTXOHistoryHydrationContext()
    await this.preloadOutputsWithBEEF(
      lookupResult.map(({ txid, outputIndex }) => ({ txid, outputIndex })),
      hydrationContext
    )
    const hydratedOutputs = (await Promise.all(
      lookupResult.map(async ({ txid, outputIndex, history, context }) => {
        const UTXO = await this.loadOutputWithBEEF(txid, outputIndex, hydrationContext)
        if (UTXO === null) {
          return null
        }

        // Get the history for this utxo and construct a BEEF
        const output = await this.getUTXOHistory(UTXO, history, 0, hydrationContext)
        if (output?.beef === undefined) {
          return null
        }

        return {
          beef: output.beef,
          outputIndex: output.outputIndex,
          context
        }
      })
    ))
      .filter((output): output is { beef: number[], outputIndex: number, context: number[] | undefined } => output !== null)
      .map(({ beef, outputIndex, context }) => (
        context === undefined
          ? { beef, outputIndex }
          : { beef, outputIndex, context }
      ))
    return {
      type: 'output-list',
      outputs: hydratedOutputs
    }
  }

  private createUTXOHistoryHydrationContext(): UTXOHistoryHydrationContext {
    return {
      outputCache: new Map<string, Promise<Output | null>>()
    }
  }

  private toOutputCacheKey(txid: string, outputIndex: number): string {
    return `${txid}:${outputIndex}`
  }

  private async preloadOutputsWithBEEF(
    outpoints: Array<{ txid: string, outputIndex: number }>,
    context: UTXOHistoryHydrationContext
  ): Promise<void> {
    if (outpoints.length === 0) {
      return
    }

    const deduped: Array<{ txid: string, outputIndex: number }> = []
    const seen = new Set<string>()

    for (const outpoint of outpoints) {
      const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
      if (seen.has(cacheKey)) {
        continue
      }
      seen.add(cacheKey)
      if (!context.outputCache.has(cacheKey)) {
        deduped.push(outpoint)
      }
    }

    if (deduped.length === 0) {
      return
    }

    const findOutputsByOutpoints = this.storage.findOutputsByOutpoints
    if (typeof findOutputsByOutpoints === 'function') {
      const outputs = await findOutputsByOutpoints.call(this.storage, deduped, true)
      const outputsByKey = new Map<string, Output>()
      for (const output of outputs) {
        outputsByKey.set(this.toOutputCacheKey(output.txid, output.outputIndex), output)
      }

      for (const outpoint of deduped) {
        const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
        context.outputCache.set(cacheKey, Promise.resolve(outputsByKey.get(cacheKey) ?? null))
      }
      return
    }

    for (const outpoint of deduped) {
      const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex)
      context.outputCache.set(
        cacheKey,
        this.storage.findOutput(outpoint.txid, outpoint.outputIndex, undefined, undefined, true)
      )
    }
  }

  private async loadOutputWithBEEF(
    txid: string,
    outputIndex: number,
    context: UTXOHistoryHydrationContext
  ): Promise<Output | null> {
    const cacheKey = this.toOutputCacheKey(txid, outputIndex)
    let cached = context.outputCache.get(cacheKey)
    if (cached === undefined) {
      cached = this.storage.findOutput(txid, outputIndex, undefined, undefined, true)
      context.outputCache.set(cacheKey, cached)
    }
    const output = await cached
    return output ?? null
  }

  private async hydrateUTXOHistoryNode(
    output: Output,
    historySelector: ((beef: number[], outputIndex: number, currentDepth: number) => Promise<boolean>) | number,
    currentDepth: number,
    context: UTXOHistoryHydrationContext
  ): Promise<HydratedUTXOHistoryNode | undefined> {
    if (output.beef === undefined) {
      throw new Error('Output must have associated transaction BEEF!')
    }

    let shouldTraverseHistory: boolean
    if (typeof historySelector === 'number') {
      shouldTraverseHistory = currentDepth <= historySelector
    } else {
      shouldTraverseHistory = await historySelector(output.beef, output.outputIndex, currentDepth)
    }

    if (shouldTraverseHistory === false) {
      return undefined
    }

    await this.preloadOutputsWithBEEF(output.outputsConsumed, context)

    const childNodes = (await Promise.all(
      output.outputsConsumed.map(async (outputIdentifier) => {
        const childOutput = await this.loadOutputWithBEEF(outputIdentifier.txid, outputIdentifier.outputIndex, context)
        if (childOutput === null) {
          return undefined
        }

        return await this.hydrateUTXOHistoryNode(childOutput, historySelector, currentDepth + 1, context)
      })
    )).filter((node): node is HydratedUTXOHistoryNode => node !== undefined)

    const tx = Transaction.fromBEEF(output.beef)
    const inputIndexBySource = new Map<string, number>()
    tx.inputs.forEach((candidateInput, index) => {
      const sourceTXID = candidateInput.sourceTXID !== undefined && candidateInput.sourceTXID !== ''
        ? candidateInput.sourceTXID
        : candidateInput.sourceTransaction?.id('hex')

      if (sourceTXID === undefined) {
        return
      }

      inputIndexBySource.set(`${sourceTXID}:${candidateInput.sourceOutputIndex}`, index)
    })

    for (const child of childNodes) {
      const inputIndex = inputIndexBySource.get(`${child.output.txid}:${child.output.outputIndex}`)

      if (inputIndex === -1 || inputIndex == null) {
        continue
      }

      const targetInput = tx.inputs[inputIndex]
      if (!targetInput) {
        this.logger.error(`Input at index ${inputIndex} is undefined, but findIndex found it. Possible sparse array from BEEF parsing.`)
        continue
      }

      targetInput.sourceTransaction = child.transaction
    }

    return { output, transaction: tx }
  }

  /**
   * Ensures alignment between the current SHIP/SLAP advertisements and the
   * configured Topic Managers and Lookup Services in the engine.
   *
   * This method performs the following actions:
   * 1. Retrieves the current configuration of topics and services.
   * 2. Fetches the existing SHIP advertisements for each configured topic.
   * 3. Fetches the existing SLAP advertisements for each configured service.
   * 4. Compares the current configuration with the fetched advertisements to determine which advertisements
   *    need to be created or revoked.
   * 5. Creates new SHIP/SLAP advertisements if they do not exist for the configured topics/services.
   * 6. Revokes existing SHIP/SLAP advertisements if they are no longer required based on the current configuration.
   *
   * The function uses the `Advertiser` methods to create or revoke advertisements and ensures the updates are
   * submitted to the SHIP/SLAP overlay networks using the engine's `submit()` method.
   *
   * @throws Will throw an error if there are issues during the advertisement synchronization process.
   * @returns {Promise<void>} A promise that resolves when the synchronization process is complete.
   */
  async syncAdvertisements(): Promise<void> {
    if (
      this.advertiser === undefined ||
      typeof this.hostingURL !== 'string' ||
      this.hostingURL.length < 1 ||
      !this.isValidUrl(this.hostingURL)
    ) {
      return
    }
    const advertiser = this.advertiser

    // Step 1: Retrieve Current Configuration
    let configuredTopics = Object.keys(this.managers)
    let configuredServices = Object.keys(this.lookupServices)

    // Filter out default SHIP/SLAP topics/services if suppressDefaultSyncAdvertisements is true
    if (this.suppressDefaultSyncAdvertisements === true) {
      configuredTopics = configuredTopics.filter(topic => topic !== 'tm_ship' && topic !== 'tm_slap')
      configuredServices = configuredServices.filter(service => service !== 'ls_ship' && service !== 'ls_slap')
    }

    // Step 2: Fetch Existing Advertisements
    const currentSHIPAdvertisements = await advertiser.findAllAdvertisements('SHIP')
    const currentSLAPAdvertisements = await advertiser.findAllAdvertisements('SLAP')

    // Step 3: Compare and Determine Actions
    const requiredSHIPAdvertisements = new Set(configuredTopics)
    const requiredSLAPAdvertisements = new Set(configuredServices)

    const shipsToCreate = Array.from(requiredSHIPAdvertisements).filter(topicOrService => !currentSHIPAdvertisements.some(x => x.topicOrService === topicOrService && x.domain === this.hostingURL))
    const slapsToCreate = Array.from(requiredSLAPAdvertisements).filter(topicOrService => !currentSLAPAdvertisements.some(x => x.topicOrService === topicOrService && x.domain === this.hostingURL))
    const shipsToRevoke = currentSHIPAdvertisements.filter(ad => !requiredSHIPAdvertisements.has(ad.topicOrService))
    const slapsToRevoke = currentSLAPAdvertisements.filter(ad => !requiredSLAPAdvertisements.has(ad.topicOrService))

    // Create needed SHIP/SLAP advertisements
    try {
      if (shipsToCreate.length > 0 || slapsToCreate.length > 0) {
        const advertisementData: AdvertisementData[] = [
          ...shipsToCreate.map(topic => ({
            protocol: 'SHIP' as const,
            topicOrServiceName: topic
          })),
          ...slapsToCreate.map(service => ({
            protocol: 'SLAP' as const,
            topicOrServiceName: service
          }))
        ]
        const taggedBEEF = await advertiser.createAdvertisements(advertisementData)
        await this.submit(taggedBEEF)
      }
    } catch (error) {
      this.logger.error('Failed to create SHIP advertisement:', error)
    }

    // Revoke all advertisements to revoke
    try {
      if (shipsToRevoke.length > 0 || slapsToRevoke.length > 0) {
        const taggedBEEF = await advertiser.revokeAdvertisements([...shipsToRevoke, ...slapsToRevoke])
        await this.submit(taggedBEEF)
      }
    } catch (error) {
      this.logger.error('Failed to revoke SHIP/SLAP advertisements:', error)
    }
  }

  /**
   * This method goes through each topic that we support syncing and attempts to sync with each endpoint
   * associated with that topic. If the sync configuration is 'SHIP', it will sync to all peers that support
   * the topic.
   *
   * @throws Error if the overlay service engine is not configured for topical synchronization.
   */
  async startGASPSync(): Promise<void> {
    if (this.syncConfiguration === undefined) {
      throw new Error('Overlay Service Engine not configured for topical synchronization!')
    }

    for (const topic of Object.keys(this.syncConfiguration)) {
      // Make sure syncEndpoints is an array or SHIP
      let syncEndpoints: string[] | string | false = this.syncConfiguration[topic]

      // Check if this topic has been configured NOT to sync
      if (syncEndpoints === false) {
        continue
      }

      if (syncEndpoints === 'SHIP') {
        // Perform lookup and find ship advertisements to set syncEndpoints for topic
        const resolve