@bsv/overlay
Version:
BSV Blockchain Overlay Services Engine
913 lines (912 loc) • 95.8 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Engine = void 0;
const sdk_1 = require("@bsv/sdk");
const gasp_1 = require("@bsv/gasp");
const OverlayGASPRemote_js_1 = require("./GASP/OverlayGASPRemote.js");
const OverlayGASPStorage_js_1 = require("./GASP/OverlayGASPStorage.js");
const BASM_js_1 = require("./BASM.js");
const BASMRemote_js_1 = require("./BASMRemote.js");
const DEFAULT_GASP_SYNC_LIMIT = 10000;
const DEFAULT_BASM_RANGE_LIMIT = 1024;
/**
* An engine for running BSV Overlay Services (topic managers and lookup services).
*/
class Engine {
/**
* Creates a new Overlay Services Engine
* @param {[key: string]: TopicManager} managers - manages topic admittance
* @param {[key: string]: LookupService} lookupServices - manages UTXO lookups
* @param {Storage} storage - for interacting with internally-managed persistent data
* @param {ChainTracker | 'scripts only'} chainTracker - Verifies SPV data associated with transactions
* @param {string} [hostingURL] - The URL this engine is hosted at. Required if going to support peer-discovery with an advertiser.
* @param {Broadcaster} [Broadcaster] - broadcaster used for broadcasting the incoming transaction
* @param {Advertiser} [Advertiser] - handles SHIP and SLAP advertisements for peer-discovery
* @param {string[]} shipTrackers - SHIP domains we know to bootstrap the system
* @param {string[]} slapTrackers - SLAP domains we know to bootstrap the system
* @param {SyncConfiguration} syncConfiguration — Configuration object describing historical synchronization of topics.
* @param {boolean} logTime - Enables / disables the timing logs for various operations in the Overlay submit route.
* @param {string} logPrefix - Supports overriding the log prefix with a custom string.
* @param {boolean} throwOnBroadcastFailure - Enables / disables throwing an error when a transaction broadcast failure is detected.
* @param {OverlayBroadcastFacilitator} overlayBroadcastFacilitator - Facilitator for propagation to other Overlay Services.
* @param {typeof console} logger - The place where log entries are written.
* @param {boolean} suppressDefaultSyncAdvertisements - Whether to suppress the default (SHIP/SLAP) sync advertisements.
* @param {TopicAnchorHeaderResolver} topicAnchorHeaderResolver - Resolves block hashes for BASM anchors.
* @param {boolean} basmSyncEnabled - Whether BASM sync should run automatically.
* @param {number} unprovenEvictionBlocks - Default block age for opt-in unproven state eviction.
*/
constructor(managers, lookupServices, storage, chainTracker, hostingURL, shipTrackers, slapTrackers, broadcaster, advertiser, syncConfiguration, logTime = false, logPrefix = '[OVERLAY_ENGINE] ', throwOnBroadcastFailure = false, overlayBroadcastFacilitator = new sdk_1.HTTPSOverlayBroadcastFacilitator(), logger = console, suppressDefaultSyncAdvertisements = true, topicAnchorHeaderResolver, basmSyncEnabled = false, unprovenEvictionBlocks = 144) {
var _a;
var _b;
this.managers = managers;
this.lookupServices = lookupServices;
this.storage = storage;
this.chainTracker = chainTracker;
this.hostingURL = hostingURL;
this.shipTrackers = shipTrackers;
this.slapTrackers = slapTrackers;
this.broadcaster = broadcaster;
this.advertiser = advertiser;
this.syncConfiguration = syncConfiguration;
this.logTime = logTime;
this.logPrefix = logPrefix;
this.throwOnBroadcastFailure = throwOnBroadcastFailure;
this.overlayBroadcastFacilitator = overlayBroadcastFacilitator;
this.logger = logger;
this.suppressDefaultSyncAdvertisements = suppressDefaultSyncAdvertisements;
this.topicAnchorHeaderResolver = topicAnchorHeaderResolver;
this.basmSyncEnabled = basmSyncEnabled;
this.unprovenEvictionBlocks = unprovenEvictionBlocks;
// To encourage synchronization of overlay services, the SHIP sync strategy is used by default for all overlay topics, except for 'tm_ship' and 'tm_slap'.
// For these two topics, any existing trackers are combined with the provided shipTrackers and slapTrackers omitting any duplicates.
if (syncConfiguration === undefined) {
this.syncConfiguration = {};
}
else {
this.syncConfiguration = syncConfiguration;
}
for (const managerName of Object.keys(managers)) {
if (managerName === 'tm_ship' && this.shipTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
// Combine tm_ship trackers with preexisting entries if any
const combinedSet = new Set([
...(Array.isArray(this.syncConfiguration[managerName]) ? this.syncConfiguration[managerName] : []),
...this.shipTrackers
]);
this.syncConfiguration[managerName] = Array.from(combinedSet);
}
else if (managerName === 'tm_slap' && this.slapTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
// Combine tm_slap trackers with preexisting entries if any
const combinedSet = new Set([
...(Array.isArray(this.syncConfiguration[managerName]) ? this.syncConfiguration[managerName] : []),
...this.slapTrackers
]);
this.syncConfiguration[managerName] = Array.from(combinedSet);
}
else {
// Set undefined managers to 'SHIP' by default
(_a = (_b = this.syncConfiguration)[managerName]) !== null && _a !== void 0 ? _a : (_b[managerName] = 'SHIP');
}
}
}
// Helper functions for logging timings
startTime(label) {
if (this.logTime) {
this.logger.time(`${this.logPrefix} ${label}`);
}
}
endTime(label) {
if (this.logTime) {
this.logger.timeEnd(`${this.logPrefix} ${label}`);
}
}
async currentHeightOrUndefined() {
if (this.chainTracker === 'scripts only') {
return undefined;
}
try {
return await this.chainTracker.currentHeight();
}
catch (error) {
this.logger.warn(`Unable to resolve current chain height for overlay metadata: ${error instanceof Error ? error.message : String(error)}`);
return undefined;
}
}
async resolveBlockHash(blockHeight, merkleRoot) {
var _a;
try {
const header = await ((_a = this.topicAnchorHeaderResolver) === null || _a === void 0 ? void 0 : _a.call(this, blockHeight));
if (header === undefined) {
return undefined;
}
if (header.merkleRoot !== undefined && merkleRoot !== undefined && header.merkleRoot !== merkleRoot) {
throw new Error(`Header merkle root ${header.merkleRoot} does not match proof root ${merkleRoot} at height ${blockHeight}`);
}
return header.blockHash;
}
catch (error) {
this.logger.warn(`Unable to resolve BASM block hash for height ${blockHeight}: ${error instanceof Error ? error.message : String(error)}`);
return undefined;
}
}
compactBEEFForStorage(tx, originalBEEF) {
return tx.merklePath === undefined ? originalBEEF : tx.toAtomicBEEF();
}
async recordTransactionData(tx, beef, blockHash) {
var _a;
if (typeof this.storage.upsertTransactionRecord !== 'function') {
return;
}
const txid = tx.id('hex');
const metadata = (0, BASM_js_1.extractMerkleProofMetadata)(txid, tx.merklePath);
await this.storage.upsertTransactionRecord({
txid,
beef: this.compactBEEFForStorage(tx, beef),
rawTx: Array.from(tx.toBinary()),
merklePath: (_a = tx.merklePath) === null || _a === void 0 ? void 0 : _a.toBinary(),
blockHeight: metadata === null || metadata === void 0 ? void 0 : metadata.blockHeight,
blockHash,
blockIndex: metadata === null || metadata === void 0 ? void 0 : metadata.blockIndex,
merkleRoot: metadata === null || metadata === void 0 ? void 0 : metadata.merkleRoot
});
}
async buildAppliedTransactionRecord(tx) {
const txid = tx.id('hex');
const metadata = (0, BASM_js_1.extractMerkleProofMetadata)(txid, tx.merklePath);
const [firstSeenHeight, blockHash] = await Promise.all([
this.currentHeightOrUndefined(),
metadata === undefined ? undefined : this.resolveBlockHash(metadata.blockHeight, metadata.merkleRoot)
]);
return {
blockHeight: metadata === null || metadata === void 0 ? void 0 : metadata.blockHeight,
blockHash,
blockIndex: metadata === null || metadata === void 0 ? void 0 : metadata.blockIndex,
merkleRoot: metadata === null || metadata === void 0 ? void 0 : metadata.merkleRoot,
firstSeenHeight: firstSeenHeight !== null && firstSeenHeight !== void 0 ? firstSeenHeight : metadata === null || metadata === void 0 ? void 0 : metadata.blockHeight,
proven: metadata !== undefined
};
}
async recomputeTopicBlockAnchor(topic, blockHeight, blockHash) {
var _a, _b, _c;
if (typeof this.storage.findAdmittedTransactionsForBlock !== 'function' ||
typeof this.storage.upsertTopicBlockAnchor !== 'function' ||
typeof this.storage.findTopicBlockAnchor !== 'function') {
return undefined;
}
const anchorBlockHash = blockHash !== null && blockHash !== void 0 ? blockHash : (_a = (await this.storage.findTopicBlockAnchor(topic, blockHeight))) === null || _a === void 0 ? void 0 : _a.blockHash;
if (anchorBlockHash === undefined) {
return undefined;
}
// BRC-136 per-block completeness: establish the chain's genesis at the first
// admitted height, then keep every height from there to the tip contiguous so
// the cumulative TAC never resets across blocks with no admitted transactions.
// We rebuild [fromHeight, toHeight] rather than only the touched height so that
// an out-of-order proof (older height arriving after a newer one) can never
// leave a gap that silently breaks the chain.
const tip = await ((_c = (_b = this.storage).findTopicAnchorTip) === null || _c === void 0 ? void 0 : _c.call(_b, topic));
const tipHeight = tip !== undefined && tip.blockHeight >= 0 ? tip.blockHeight : undefined;
const fromHeight = tipHeight === undefined ? blockHeight : Math.min(blockHeight, tipHeight + 1);
const toHeight = tipHeight === undefined ? blockHeight : Math.max(blockHeight, tipHeight);
await this.rebuildTopicAnchorChain(topic, fromHeight, toHeight, new Map([[blockHeight, anchorBlockHash]]));
return await this.storage.findTopicBlockAnchor(topic, blockHeight);
}
/**
* Extends every configured topic's anchor chain forward with empty Topic Block
* Anchors (basmRoot = zero hash, admittedCount = 0) up to `toHeight`, so the
* cumulative TAC advances on every block even when a topic admits nothing —
* this is what lets a peer authoritatively confirm "this block contained no
* transactions for this topic". Chains with no first admission yet are left
* unstarted (genesis is the topic's first admitted height).
*/
async advanceTopicAnchorChains(toHeight) {
if (typeof this.storage.findTopicAnchorTip !== 'function' ||
typeof this.storage.upsertTopicBlockAnchor !== 'function') {
return;
}
const targetHeight = toHeight !== null && toHeight !== void 0 ? toHeight : await this.currentHeightOrUndefined();
if (targetHeight === undefined) {
return;
}
for (const topic of Object.keys(this.managers)) {
const tip = await this.storage.findTopicAnchorTip(topic);
if (tip === undefined || tip.blockHeight < 0 || tip.blockHeight >= targetHeight) {
continue;
}
await this.rebuildTopicAnchorChain(topic, tip.blockHeight + 1, targetHeight);
}
}
/**
* Rebuilds a contiguous slice of a topic's anchor chain over [fromHeight,
* toHeight]. Each height uses its admitted transactions (empty -> zero basmRoot)
* and chains the cumulative TAC from the prior height. Missing heights are
* filled rather than skipped, so the chain stays gap-free. If a block hash
* cannot be resolved for some height the extension halts there to preserve
* contiguity instead of leaving a hole.
*/
async rebuildTopicAnchorChain(topic, fromHeight, toHeight, blockHashHints = new Map(), forceResolve = false) {
var _a, _b, _c;
if (typeof this.storage.findAdmittedTransactionsForBlock !== 'function' ||
typeof this.storage.upsertTopicBlockAnchor !== 'function' ||
typeof this.storage.findTopicBlockAnchor !== 'function' ||
toHeight < fromHeight) {
return;
}
if (toHeight - fromHeight + 1 > DEFAULT_BASM_RANGE_LIMIT) {
// Bound the work per pass; the next trigger resumes from the new tip.
this.logger.warn(`[BASM] capping anchor chain extension for "${topic}" at ${DEFAULT_BASM_RANGE_LIMIT} blocks (requested ${fromHeight}..${toHeight}); will continue on the next pass`);
toHeight = fromHeight + DEFAULT_BASM_RANGE_LIMIT - 1;
}
const previousAnchor = fromHeight > 0
? await this.storage.findTopicBlockAnchor(topic, fromHeight - 1)
: undefined;
let prevTac = (_a = previousAnchor === null || previousAnchor === void 0 ? void 0 : previousAnchor.tac) !== null && _a !== void 0 ? _a : BASM_js_1.BASM_ZERO_HASH;
for (let height = fromHeight; height <= toHeight; height++) {
const admitted = await this.storage.findAdmittedTransactionsForBlock(topic, height);
const existing = await this.storage.findTopicBlockAnchor(topic, height);
// On a reorg rebuild the existing anchor's block hash is stale, so force
// canonical re-resolution from the header resolver instead of reusing it.
const blockHash = (_c = (_b = blockHashHints.get(height)) !== null && _b !== void 0 ? _b : (forceResolve ? undefined : existing === null || existing === void 0 ? void 0 : existing.blockHash)) !== null && _c !== void 0 ? _c : await this.resolveBlockHash(height);
if (blockHash === undefined) {
this.logger.warn(`[BASM] unable to resolve block hash for "${topic}" at height ${height}; halting chain extension`);
return;
}
const basmRoot = (0, BASM_js_1.computeBasmRoot)(admitted);
const tac = (0, BASM_js_1.computeTac)(prevTac, blockHash, basmRoot);
await this.storage.upsertTopicBlockAnchor({
topic,
blockHeight: height,
blockHash,
basmRoot,
admittedCount: admitted.length,
tac
});
prevTac = tac;
}
}
/**
* Reconciles BASM anchors with a blockchain reorganization reported by the
* chain tracker (e.g. go-chaintracks `/v2/reorg/stream`). Proven topic
* transactions whose block was orphaned are demoted to unproven so they leave
* the admitted set, then every topic anchor chain intersecting the affected
* height range is rebuilt over the canonical block hashes. A reorg changes the
* canonical block hash for the affected heights, so topics with no demoted
* transaction are rebuilt too. Idempotent: a clean window demotes nothing and
* reproduces an identical TAC, so this is safe to invoke on every reorg event,
* SSE reconnect, and poll.
*/
async handleReorg(input) {
var _a, _b;
const report = { perTopic: [] };
if (typeof this.storage.findProvenAppliedTransactionsByBlockHash !== 'function' ||
typeof this.storage.demoteAppliedTransactionToUnproven !== 'function' ||
typeof this.storage.findTopicBlockAnchors !== 'function' ||
typeof this.storage.upsertTopicBlockAnchor !== 'function') {
return report;
}
// 1) Demote proven admissions whose block was orphaned. Hashes are
// normalized to lower-case display hex to match stored block hashes
// (go-sdk chainhash.Hash marshals as reversed display hex).
const demotedByTopic = new Map();
for (const rawHash of input.orphanedBlockHashes) {
const blockHash = rawHash.toLowerCase();
const rows = await this.storage.findProvenAppliedTransactionsByBlockHash(blockHash);
for (const row of rows) {
await this.storage.demoteAppliedTransactionToUnproven(row.txid, row.topic);
const list = (_a = demotedByTopic.get(row.topic)) !== null && _a !== void 0 ? _a : [];
list.push(row.txid);
demotedByTopic.set(row.topic, list);
}
}
// 2) Rebuild every topic anchor chain that intersects the reorged range,
// forcing canonical block-hash re-resolution so stale hashes are replaced.
for (const topic of Object.keys(this.managers)) {
const existing = await this.storage.findTopicBlockAnchors(topic, input.rebuildFromHeight, input.newTipHeight);
if (existing.length === 0) {
continue;
}
const startHeight = Math.min(...existing.map(anchor => anchor.blockHeight));
await this.rebuildTopicAnchorChain(topic, startHeight, input.newTipHeight, new Map(), true);
report.perTopic.push({
topic,
demotedTxids: (_b = demotedByTopic.get(topic)) !== null && _b !== void 0 ? _b : [],
rebuiltFrom: startHeight,
rebuiltTo: input.newTipHeight
});
}
return report;
}
/**
* Revalidation sweep: the reorg fallback for chain trackers without a reorg
* event stream, and the catch-up step on every reorg-SSE (re)connect (the
* go-chaintracks reorg stream carries no event ids, so a reconnect cannot
* replay events missed while disconnected). Scans proven applied transactions
* in `[tip - depth + 1, tip]`; any whose proof root no longer validates against
* the chain tracker, or whose block hash diverges from the canonical header, is
* treated as orphaned and reconciled via {@link handleReorg}.
*/
async isProvenAnchorStale(row, chainTracker) {
var _a;
if (row.merkleRoot !== undefined) {
try {
if (!(await chainTracker.isValidRootForHeight(row.merkleRoot, row.blockHeight))) {
return true;
}
}
catch (error) {
this.logger.warn(`[BASM] root validation failed for ${row.txid} at height ${row.blockHeight}: ${error instanceof Error ? error.message : String(error)}`);
return undefined;
}
}
const canonical = await this.resolveBlockHash(row.blockHeight);
return canonical !== undefined && canonical.toLowerCase() !== ((_a = row.blockHash) === null || _a === void 0 ? void 0 : _a.toLowerCase());
}
async revalidateRecentAnchors(depth = 3) {
const chainTracker = this.chainTracker;
if (chainTracker === 'scripts only') {
this.logger.warn('[BASM] revalidation sweep requires a ChainTracker; skipping');
return undefined;
}
if (typeof this.storage.findProvenAppliedTransactionsInRange !== 'function') {
return undefined;
}
const tip = await this.currentHeightOrUndefined();
if (tip === undefined) {
return undefined;
}
const fromHeight = Math.max(0, tip - depth + 1);
const rows = await this.storage.findProvenAppliedTransactionsInRange(fromHeight, tip);
const orphaned = new Set();
let minAffected = Number.POSITIVE_INFINITY;
for (const row of rows) {
if (row.blockHash === undefined) {
continue;
}
const stale = await this.isProvenAnchorStale(row, chainTracker);
if (stale === true) {
orphaned.add(row.blockHash.toLowerCase());
minAffected = Math.min(minAffected, row.blockHeight);
}
}
if (orphaned.size === 0) {
return { perTopic: [] };
}
return await this.handleReorg({
orphanedBlockHashes: Array.from(orphaned),
rebuildFromHeight: minAffected,
newTipHeight: tip
});
}
/**
* Submits a transaction for processing by Overlay Services.
* @param {TaggedBEEF} taggedBEEF - The transaction to process
* @param {function(STEAK): void} [onSTEAKReady] - Optional callback function invoked when the STEAK is ready.
* @param {string} mode — Indicates the submission behavior, whether historical or current. Historical transactions are not broadcast or propagated.
* @param {number[]} offChainValues — Values necessary to evaluate topical admittance that are not stored on-chain.
*
* The optional callback function should be used to get STEAK when ready, and avoid waiting for broadcast and transaction propagation to complete.
*
* @returns {Promise<STEAK>} The submitted transaction execution acknowledgement
*/
async submit(taggedBEEF, onSteakReady, mode = 'current-tx', offChainValues) {
var _a, _b;
for (const t of taggedBEEF.topics) {
if (this.managers[t] === undefined || this.managers[t] === null) {
throw new Error(`This server does not support this topic: ${t}`);
}
}
// Validate the transaction SPV information
const tx = sdk_1.Transaction.fromBEEF(taggedBEEF.beef);
const txid = tx.id('hex');
this.startTime(`submit_${txid}`);
if (mode !== 'historical-tx-no-spv') {
this.startTime(`chainTracker_${txid.substring(0, 10)}`);
const txValid = await tx.verify(this.chainTracker);
if (!txValid)
throw new Error('Unable to verify SPV information.');
this.endTime(`chainTracker_${txid.substring(0, 10)}`);
}
const steak = {};
const dupeTopics = new Set();
const failedTopics = new Set();
const topicValidations = taggedBEEF.topics.map(async (topic) => {
try {
if (this.managers[topic] === undefined || this.managers[topic] === null) {
throw new Error(`This server does not support this topic: ${topic}`);
}
// Check for duplicate transactions
this.startTime(`dupCheck_${txid.substring(0, 10)}`);
const dupeCheck = await this.storage.doesAppliedTransactionExist({ txid, topic });
this.endTime(`dupCheck_${txid.substring(0, 10)}`);
if (dupeCheck) {
dupeTopics.add(topic);
return {
topic,
isDupe: true,
previousCoins: [],
previousOutputs: [],
admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
};
}
// Identify previous coins admitted to this specific topic
const previousCoins = [];
const outputPromises = tx.inputs.map(async (input, i) => {
var _a, _b;
const previousTXID = (_a = input.sourceTXID) !== null && _a !== void 0 ? _a : (_b = input.sourceTransaction) === null || _b === void 0 ? void 0 : _b.id('hex');
if (previousTXID !== undefined) {
// Check if the previous output was admitted to this specific topic
const output = await this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic);
if (output !== undefined && output !== null) {
previousCoins.push(i);
return output;
}
}
return null;
});
this.startTime(`previousOutputQuery_${txid.substring(0, 10)}`);
const previousOutputs = await Promise.all(outputPromises);
this.endTime(`previousOutputQuery_${txid.substring(0, 10)}`);
// Determine which outputs are admissible for this topic (validation only)
this.startTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
const admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(taggedBEEF.beef, previousCoins, offChainValues, mode);
this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
return {
topic,
isDupe: false,
previousCoins,
previousOutputs,
admissibleOutputs
};
}
catch (error) {
this.logger.error('Error validating topic during submit:', error);
failedTopics.add(topic);
return {
topic,
isDupe: false,
previousCoins: [],
previousOutputs: [],
admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
};
}
});
const validations = await Promise.all(topicValidations);
// Build preliminary STEAK from validation results
for (const validation of validations) {
steak[validation.topic] = validation.admissibleOutputs;
}
// ===================================================================
// PHASE 2: BROADCAST (before any mutations)
// ===================================================================
// Only broadcast when at least one topic actually accepted the
// transaction. For a non-failed topic, acceptance means: previously
// accepted (dupe / client retry), outputs admitted, coins retained, or
// previously-admitted coins consumed (e.g. a consume-only deletion such
// as a KVStore remove, even one that retains nothing). A topic manager
// REJECTS by throwing from identifyAdmissibleOutputs (tracked in
// failedTopics). A transaction every topic rejected must never reach the
// network: submitters treat an empty STEAK as a rejection and
// abort/release their held inputs, so broadcasting it anyway would
// desync their wallets from the chain.
const anyTopicAccepted = validations.some(v => !failedTopics.has(v.topic) && (v.isDupe ||
v.admissibleOutputs.outputsToAdmit.length > 0 ||
v.admissibleOutputs.coinsToRetain.length > 0 ||
v.previousCoins.length > 0));
this.startTime(`broadcast_${txid.substring(0, 10)}`);
if (mode !== 'historical-tx' && this.broadcaster !== undefined && anyTopicAccepted) {
try {
let response;
if (tx.merklePath !== undefined) {
// tx has been verified, thus if there is a merklePath, the transaction is already on-chain...skip broadcast.
const txid = tx.id('hex');
const mp = tx.merklePath;
const leaf = mp.path[0].find(leaf => leaf.hash === txid);
const r = {
status: 'success',
txid: tx.id('hex'),
message: `In block at height ${mp.blockHeight} index ${leaf === null || leaf === void 0 ? void 0 : leaf.offset}`,
};
response = r;
}
else {
response = await this.broadcaster.broadcast(tx);
}
if ((0, sdk_1.isBroadcastFailure)(response) && this.throwOnBroadcastFailure) {
const e = new Error(`Failed to broadcast transaction! Error: ${response.description}`);
e.more = response.more;
throw e;
}
}
catch (error) {
if (this.throwOnBroadcastFailure) {
throw error;
}
this.logger.error('Error broadcasting transaction:', error);
}
}
this.endTime(`broadcast_${txid.substring(0, 10)}`);
// Call the callback function with STEAK if it is provided (before storage mutations)
if (onSteakReady !== undefined) {
onSteakReady(steak);
}
// ===================================================================
// PHASE 3: MUTATE STORAGE (only after broadcast succeeded)
// ===================================================================
// Mark previous outputs as spent and notify lookup services
await Promise.all(validations.map(async (validation) => {
if (validation.isDupe || failedTopics.has(validation.topic)) {
return;
}
const topic = validation.topic;
const previousOutputs = validation.previousOutputs;
// Mark all previous outputs as spent
const markSpentPromises = previousOutputs.map(async (output) => {
if (output !== undefined && output !== null) {
try {
await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic);
await Promise.all(Object.values(this.lookupServices).map(async (l) => {
var _a;
try {
if (typeof l.outputSpent === 'function') {
if (l.spendNotificationMode === 'txid') {
await l.outputSpent({
mode: 'txid',
spendingTxid: txid,
txid: output.txid,
outputIndex: output.outputIndex,
topic
});
}
else if (l.spendNotificationMode === 'script') {
const inputIndex = tx.inputs.findIndex(i => {
var _a;
let realSource = i.sourceTXID;
if (!realSource) {
realSource = (_a = i.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex');
}
return realSource === output.txid && i.sourceOutputIndex === output.outputIndex;
});
if (inputIndex === -1) {
throw new Error('Could not find input index');
}
await l.outputSpent({
mode: 'script',
spendingTxid: txid,
inputIndex,
sequenceNumber: (_a = tx.inputs[inputIndex].sequence) !== null && _a !== void 0 ? _a : 0xffffffff,
unlockingScript: tx.inputs[inputIndex].unlockingScript,
txid: output.txid,
outputIndex: output.outputIndex,
topic,
offChainValues
});
}
else if (l.spendNotificationMode === 'whole-tx') {
await l.outputSpent({
mode: 'whole-tx',
spendingAtomicBEEF: tx.toAtomicBEEF(),
txid: output.txid,
outputIndex: output.outputIndex,
topic,
offChainValues
});
}
else { // none
await l.outputSpent({
mode: 'none',
txid: output.txid,
outputIndex: output.outputIndex,
topic
});
}
}
}
catch (error) {
this.logger.error('Error in lookup service for outputSpent:', error);
}
}));
}
catch (error) {
this.logger.error('Error marking UTXO as spent:', error);
}
}
});
await Promise.all(markSpentPromises);
}));
// Continue with storage updates and lookup service notifications
for (const validation of validations) {
const topic = validation.topic;
if (dupeTopics.has(topic)) {
continue;
}
if (failedTopics.has(topic)) {
continue;
}
try {
const admissibleOutputs = steak[topic];
const outputsToAdmit = admissibleOutputs.outputsToAdmit;
const outputsConsumed = [];
const outputsToMarkStale = [];
// Use previousCoins from validation
const previousCoins = validation.previousCoins;
// For each of the previous UTXOs for this topic, if the UTXO was not included in the list of UTXOs identified for retention, then it will be marked as stale.
for (const inputIndex of previousCoins) {
const previousTXID = (_a = tx.inputs[inputIndex].sourceTXID) !== null && _a !== void 0 ? _a : (_b = tx.inputs[inputIndex].sourceTransaction) === null || _b === void 0 ? void 0 : _b.id('hex');
if (typeof previousTXID !== 'string')
continue;
const previousOutputIndex = tx.inputs[inputIndex].sourceOutputIndex;
if (admissibleOutputs.coinsToRetain.includes(inputIndex)) {
outputsConsumed.push({
txid: previousTXID,
outputIndex: previousOutputIndex
});
}
else {
outputsToMarkStale.push({
txid: previousTXID,
previousOutputIndex,
inputIndex
});
}
}
// Remove stale outputs recursively
this.startTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
await Promise.all(outputsToMarkStale.map(async (coin) => {
const output = await this.storage.findOutput(coin.txid, coin.previousOutputIndex, topic);
if (output !== undefined && output !== null) {
await this.deleteUTXODeep(output);
}
}));
this.endTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
// Update the STEAK to indicate which coins were removed
steak[topic].coinsRemoved = outputsToMarkStale.map(x => x.inputIndex);
// Handle admittance and notification of incoming UTXOs
const newUTXOs = [];
await Promise.all(outputsToAdmit.map(async (outputIndex) => {
var _a;
if (typeof tx.outputs[outputIndex].satoshis !== 'number')
return;
this.startTime(`insertNewOutput_${txid.substring(0, 10)}`);
await this.storage.insertOutput({
txid,
outputIndex,
outputScript: tx.outputs[outputIndex].lockingScript.toBinary(),
satoshis: tx.outputs[outputIndex].satoshis,
topic,
spent: false,
beef: this.compactBEEFForStorage(tx, taggedBEEF.beef),
consumedBy: [],
outputsConsumed,
score: Date.now(),
blockHeight: (_a = (0, BASM_js_1.extractMerkleProofMetadata)(txid, tx.merklePath)) === null || _a === void 0 ? void 0 : _a.blockHeight
});
this.endTime(`insertNewOutput_${txid.substring(0, 10)}`);
newUTXOs.push({ txid, outputIndex });
this.startTime(`notifyLookupService${txid.substring(0, 10)}`);
await Promise.all(Object.values(this.lookupServices).map(async (l) => {
try {
if (l.admissionMode === 'locking-script') {
if (typeof tx.outputs[outputIndex].lockingScript !== 'object' ||
typeof tx.outputs[outputIndex].satoshis !== 'number') {
return;
}
await l.outputAdmittedByTopic({
mode: 'locking-script',
txid,
outputIndex,
lockingScript: tx.outputs[outputIndex].lockingScript,
satoshis: tx.outputs[outputIndex].satoshis,
topic,
offChainValues
});
}
else {
await l.outputAdmittedByTopic({
mode: 'whole-tx',
atomicBEEF: tx.toAtomicBEEF(),
outputIndex,
topic,
offChainValues
});
}
}
catch (error) {
this.logger.error('Error in lookup service for outputAdmittedByTopic:', error);
}
}));
this.endTime(`notifyLookupService${txid.substring(0, 10)}`);
}));
this.startTime(`outputConsumed_${txid.substring(0, 10)}`);
// Update each output consumed to know who consumed it and insert applied transaction in parallel
const appliedRecord = await this.buildAppliedTransactionRecord(tx);
await this.recordTransactionData(tx, taggedBEEF.beef, appliedRecord.blockHash);
await Promise.all([
...outputsConsumed.map(async (output) => {
const outputToUpdate = await this.storage.findOutput(output.txid, output.outputIndex, topic);
if (outputToUpdate !== undefined && outputToUpdate !== null) {
const newConsumedBy = [...new Set([...newUTXOs, ...outputToUpdate.consumedBy])];
await this.storage.updateConsumedBy(output.txid, output.outputIndex, topic, newConsumedBy);
}
}),
this.storage.insertAppliedTransaction({
txid,
topic,
...appliedRecord
})
]);
if (appliedRecord.blockHeight !== undefined && appliedRecord.blockHash !== undefined) {
await this.recomputeTopicBlockAnchor(topic, appliedRecord.blockHeight, appliedRecord.blockHash);
}
this.endTime(`outputConsumed_${txid.substring(0, 10)}`);
}
catch (error) {
this.logger.error('Error updating storage and notifying lookup services for topic', topic, error);
}
}
// If we don't have an advertiser or we are dealing with historical transactions, just return the steak
if (this.advertiser === undefined || mode === 'historical-tx' || mode === 'historical-tx-no-spv') {
return steak;
}
this.startTime(`transactionPropagation_${txid.substring(0, 10)}`);
const relevantTopics = taggedBEEF.topics.filter(topic => { var _a; return steak[topic] !== undefined && !dupeTopics.has(topic) && (steak[topic].outputsToAdmit.length !== 0 || ((_a = steak[topic].coinsRemoved) === null || _a === void 0 ? void 0 : _a.length) !== 0); });
if (relevantTopics.length === 0) {
this.endTime(`transactionPropagation_${txid.substring(0, 10)}`);
return steak;
}
// Create a SHIPBroadcaster instance
let customBroadcasterConfig;
if (Array.isArray(this.slapTrackers)) {
// Custom SLAP trackers warrant a custom broadcaster config
const resolverConfig = {
slapTrackers: this.slapTrackers
};
customBroadcasterConfig = {
resolver: new sdk_1.LookupResolver(resolverConfig)
};
}
const shipBroadcaster = new sdk_1.SHIPBroadcaster(relevantTopics, customBroadcasterConfig);
try {
await shipBroadcaster.broadcast(tx);
}
catch (error) {
this.logger.error('Error during propagation to other nodes:', error);
}
this.endTime(`transactionPropagation_${txid.substring(0, 10)}`);
// Immediately return from the function without waiting for the promises to resolve.
return steak;
}
/**
* Submit a lookup question to the Overlay Services Engine, and receive back a Lookup Answer
* @param LookupQuestion — The question to ask the Overlay Services Engine
* @returns The answer to the question
*/
async lookup(lookupQuestion) {
// Validate a lookup service for the provider is found
const lookupService = this.lookupServices[lookupQuestion.service];
if (lookupService === undefined || lookupService === null)
throw new Error(`Lookup service not found for provider: ${lookupQuestion.service} `);
const lookupResult = await lookupService.lookup(lookupQuestion);
const hydrationContext = this.createUTXOHistoryHydrationContext();
await this.preloadOutputsWithBEEF(lookupResult.map(({ txid, outputIndex }) => ({ txid, outputIndex })), hydrationContext);
const hydratedOutputs = (await Promise.all(lookupResult.map(async ({ txid, outputIndex, history, context }) => {
const UTXO = await this.loadOutputWithBEEF(txid, outputIndex, hydrationContext);
if (UTXO === null) {
return null;
}
// Get the history for this utxo and construct a BEEF
const output = await this.getUTXOHistory(UTXO, history, 0, hydrationContext);
if ((output === null || output === void 0 ? void 0 : output.beef) === undefined) {
return null;
}
return {
beef: output.beef,
outputIndex: output.outputIndex,
context
};
})))
.filter((output) => output !== null)
.map(({ beef, outputIndex, context }) => (context === undefined
? { beef, outputIndex }
: { beef, outputIndex, context }));
return {
type: 'output-list',
outputs: hydratedOutputs
};
}
createUTXOHistoryHydrationContext() {
return {
outputCache: new Map()
};
}
toOutputCacheKey(txid, outputIndex) {
return `${txid}:${outputIndex}`;
}
async preloadOutputsWithBEEF(outpoints, context) {
var _a;
if (outpoints.length === 0) {
return;
}
const deduped = [];
const seen = new Set();
for (const outpoint of outpoints) {
const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex);
if (seen.has(cacheKey)) {
continue;
}
seen.add(cacheKey);
if (!context.outputCache.has(cacheKey)) {
deduped.push(outpoint);
}
}
if (deduped.length === 0) {
return;
}
const findOutputsByOutpoints = this.storage.findOutputsByOutpoints;
if (typeof findOutputsByOutpoints === 'function') {
const outputs = await findOutputsByOutpoints.call(this.storage, deduped, true);
const outputsByKey = new Map();
for (const output of outputs) {
outputsByKey.set(this.toOutputCacheKey(output.txid, output.outputIndex), output);
}
for (const outpoint of deduped) {
const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex);
context.outputCache.set(cacheKey, Promise.resolve((_a = outputsByKey.get(cacheKey)) !== null && _a !== void 0 ? _a : null));
}
return;
}
for (const outpoint of deduped) {
const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex);
context.outputCache.set(cacheKey, this.storage.findOutput(outpoint.txid, outpoint.outputIndex, undefined, undefined, true));
}
}
async loadOutputWithBEEF(txid, outputIndex, context) {
const cacheKey = this.toOutputCacheKey(txid, outputIndex);
let cached = context.outputCache.get(cacheKey);
if (cached === undefined) {
cached = this.storage.findOutput(txid, outputIndex, undefined, undefined, true);
context.outputCache.set(cacheKey, cached);
}
const output = await cached;
return output !== null && output !== void 0 ? output : null;
}
async hydrateUTXOHistoryNode(output, historySelector, currentDepth, context) {
if (output.beef === undefined) {
throw new Error('Output must have associated transaction BEEF!');
}
let shouldTraverseHistory;
if (typeof historySelector === 'number') {
shouldTraverseHistory = currentDepth <= historySelector;
}
else {
shouldTraverseHistory = await historySelector(output.beef, output.outputIndex, currentDepth);
}
if (shouldTraverseHistory === false) {
return undefined;
}
await this.preloadOutputsWithBEEF(output.outputsConsumed, context);
const childNodes = (await Promise.all(output.outputsConsumed.map(async (outputIdentifier) => {
const childOutput = await this.loadOutputWithBEEF(outputIdentifier.txid, outputIdentifier.outputIndex, context);
if (childOutput === null) {
return undefined;
}
return await this.hydrateUTXOHistoryNode(childOutput, historySelector, currentDepth + 1, context);
}))).filter((node) => node !== undefined);
const tx = sdk_1.Transaction.fromBEEF(output.beef);
const inputIndexBySource = new Map();
tx.inputs.forEach((candidateInput, index) => {
var _a;
const sourceTXID = candidateInput.sourceTXID !== undefined && candidateInput.sourceTXID !== ''
? candidateInput.sourceTXID
: (_a = candidateInput.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex');
if (sourceTXID === undefined) {
return;
}
inputIndexBySource.set(`${sourceTXID}:${candidateInput.sourceOutputIndex}`, index);
});
for (const child of childNodes) {
const inputIndex = inputIndexBySource.get(`${child.output.txid}:${child.output.outputIndex}`);
if (inputIndex === -1 || inputIndex == null) {
continue;
}
const targetInput = tx.inputs[inputIndex];
if (!targetInput) {
this.logger.error(`Input at index ${inputIndex} is undefined, but findIndex found it. Possible sparse array from BEEF parsing.`);
continue;