@bsv/overlay
Version:
BSV Blockchain Overlay Services Engine
969 lines • 93.1 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Engine = void 0;
const sdk_1 = require("@bsv/sdk");
const gasp_1 = require("@bsv/gasp");
const OverlayGASPRemote_js_1 = require("./GASP/OverlayGASPRemote.js");
const OverlayGASPStorage_js_1 = require("./GASP/OverlayGASPStorage.js");
const BASM_js_1 = require("./BASM.js");
const BASMRemote_js_1 = require("./BASMRemote.js");
const SafeLog_js_1 = require("./SafeLog.js");
const DEFAULT_GASP_SYNC_LIMIT = 10000;
const DEFAULT_BASM_RANGE_LIMIT = 1024;
function findSpendingInputIndex(tx, output) {
return tx.inputs.findIndex(input => {
var _a;
const realSource = input.sourceTXID || ((_a = input.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex'));
return realSource === output.txid && input.sourceOutputIndex === output.outputIndex;
});
}
/**
* An engine for running BSV Overlay Services (topic managers and lookup services).
*/
class Engine {
/**
* Creates a new Overlay Services Engine
* @param {[key: string]: TopicManager} managers - manages topic admittance
* @param {[key: string]: LookupService} lookupServices - manages UTXO lookups
* @param {Storage} storage - for interacting with internally-managed persistent data
* @param {ChainTracker | 'scripts only'} chainTracker - Verifies SPV data associated with transactions
* @param {string} [hostingURL] - The URL this engine is hosted at. Required if going to support peer-discovery with an advertiser.
* @param {Broadcaster} [Broadcaster] - broadcaster used for broadcasting the incoming transaction
* @param {Advertiser} [Advertiser] - handles SHIP and SLAP advertisements for peer-discovery
* @param {string[]} shipTrackers - SHIP domains we know to bootstrap the system
* @param {string[]} slapTrackers - SLAP domains we know to bootstrap the system
* @param {SyncConfiguration} syncConfiguration — Configuration object describing historical synchronization of topics.
* @param {boolean} logTime - Enables / disables the timing logs for various operations in the Overlay submit route.
* @param {string} logPrefix - Supports overriding the log prefix with a custom string.
* @param {boolean} throwOnBroadcastFailure - Enables / disables throwing an error when a transaction broadcast failure is detected.
* @param {OverlayBroadcastFacilitator} overlayBroadcastFacilitator - Facilitator for propagation to other Overlay Services.
* @param {typeof console} logger - The place where log entries are written.
* @param {boolean} suppressDefaultSyncAdvertisements - Whether to suppress the default (SHIP/SLAP) sync advertisements.
* @param {TopicAnchorHeaderResolver} topicAnchorHeaderResolver - Resolves block hashes for BASM anchors.
* @param {boolean} basmSyncEnabled - Whether BASM sync should run automatically.
* @param {number} unprovenEvictionBlocks - Default block age for opt-in unproven state eviction.
* @param {number} maxLookupResults - Maximum lookup formulas hydrated per request. Use -1 to opt out.
*/
constructor(managers, lookupServices, storage, chainTracker, hostingURL, shipTrackers, slapTrackers, broadcaster, advertiser, syncConfiguration, logTime = false, logPrefix = '[OVERLAY_ENGINE] ', throwOnBroadcastFailure = false, overlayBroadcastFacilitator = new sdk_1.HTTPSOverlayBroadcastFacilitator(), logger = console, suppressDefaultSyncAdvertisements = true, topicAnchorHeaderResolver, basmSyncEnabled = false, unprovenEvictionBlocks = 144, maxLookupResults = 1000) {
var _a, _b;
var _c;
this.managers = managers;
this.lookupServices = lookupServices;
this.storage = storage;
this.chainTracker = chainTracker;
this.hostingURL = hostingURL;
this.shipTrackers = shipTrackers;
this.slapTrackers = slapTrackers;
this.broadcaster = broadcaster;
this.advertiser = advertiser;
this.syncConfiguration = syncConfiguration;
this.logTime = logTime;
this.logPrefix = logPrefix;
this.throwOnBroadcastFailure = throwOnBroadcastFailure;
this.overlayBroadcastFacilitator = overlayBroadcastFacilitator;
this.logger = logger;
this.suppressDefaultSyncAdvertisements = suppressDefaultSyncAdvertisements;
this.topicAnchorHeaderResolver = topicAnchorHeaderResolver;
this.basmSyncEnabled = basmSyncEnabled;
this.unprovenEvictionBlocks = unprovenEvictionBlocks;
this.maxLookupResults = maxLookupResults;
if (maxLookupResults !== -1 && (!Number.isSafeInteger(maxLookupResults) || maxLookupResults < 1)) {
throw new TypeError('maxLookupResults must be -1 or a positive safe integer');
}
// To encourage synchronization of overlay services, the SHIP sync strategy is used by default for all overlay topics, except for 'tm_ship' and 'tm_slap'.
// For these two topics, any existing trackers are combined with the provided shipTrackers and slapTrackers omitting any duplicates.
(_a = this.syncConfiguration) !== null && _a !== void 0 ? _a : (this.syncConfiguration = {});
for (const managerName of Object.keys(managers)) {
if (managerName === 'tm_ship' && this.shipTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
// Combine tm_ship trackers with preexisting entries if any
const combinedSet = new Set([
...(Array.isArray(this.syncConfiguration[managerName]) ? this.syncConfiguration[managerName] : []),
...this.shipTrackers
]);
this.syncConfiguration[managerName] = Array.from(combinedSet);
}
else if (managerName === 'tm_slap' && this.slapTrackers !== undefined && this.syncConfiguration[managerName] !== false) {
// Combine tm_slap trackers with preexisting entries if any
const combinedSet = new Set([
...(Array.isArray(this.syncConfiguration[managerName]) ? this.syncConfiguration[managerName] : []),
...this.slapTrackers
]);
this.syncConfiguration[managerName] = Array.from(combinedSet);
}
else {
// Set undefined managers to 'SHIP' by default
(_b = (_c = this.syncConfiguration)[managerName]) !== null && _b !== void 0 ? _b : (_c[managerName] = 'SHIP');
}
}
}
// Helper functions for logging timings
startTime(label) {
if (this.logTime) {
this.logger.time(`${this.logPrefix} ${label}`);
}
}
endTime(label) {
if (this.logTime) {
this.logger.timeEnd(`${this.logPrefix} ${label}`);
}
}
async currentHeightOrUndefined() {
if (this.chainTracker === 'scripts only') {
return undefined;
}
try {
return await this.chainTracker.currentHeight();
}
catch (error) {
this.logger.warn(`Unable to resolve current chain height for overlay metadata: ${error instanceof Error ? error.message : String(error)}`);
return undefined;
}
}
async resolveBlockHash(blockHeight, merkleRoot) {
var _a;
try {
const header = await ((_a = this.topicAnchorHeaderResolver) === null || _a === void 0 ? void 0 : _a.call(this, blockHeight));
if (header === undefined) {
return undefined;
}
if (header.merkleRoot !== undefined && merkleRoot !== undefined && header.merkleRoot !== merkleRoot) {
throw new Error(`Header merkle root ${header.merkleRoot} does not match proof root ${merkleRoot} at height ${blockHeight}`);
}
return header.blockHash;
}
catch (error) {
this.logger.warn(`Unable to resolve BASM block hash: height=${(0, SafeLog_js_1.serializeLogValue)(blockHeight)} error=${(0, SafeLog_js_1.serializeErrorForLog)(error)}`);
return undefined;
}
}
compactBEEFForStorage(tx, originalBEEF) {
return tx.merklePath === undefined ? originalBEEF : tx.toAtomicBEEF();
}
async recordTransactionData(tx, beef, blockHash) {
var _a;
if (typeof this.storage.upsertTransactionRecord !== 'function') {
return;
}
const txid = tx.id('hex');
const metadata = (0, BASM_js_1.extractMerkleProofMetadata)(txid, tx.merklePath);
await this.storage.upsertTransactionRecord({
txid,
beef: this.compactBEEFForStorage(tx, beef),
rawTx: Array.from(tx.toBinary()),
merklePath: (_a = tx.merklePath) === null || _a === void 0 ? void 0 : _a.toBinary(),
blockHeight: metadata === null || metadata === void 0 ? void 0 : metadata.blockHeight,
blockHash,
blockIndex: metadata === null || metadata === void 0 ? void 0 : metadata.blockIndex,
merkleRoot: metadata === null || metadata === void 0 ? void 0 : metadata.merkleRoot
});
}
async buildAppliedTransactionRecord(tx) {
const txid = tx.id('hex');
const metadata = (0, BASM_js_1.extractMerkleProofMetadata)(txid, tx.merklePath);
const [firstSeenHeight, blockHash] = await Promise.all([
this.currentHeightOrUndefined(),
metadata === undefined ? undefined : this.resolveBlockHash(metadata.blockHeight, metadata.merkleRoot)
]);
return {
blockHeight: metadata === null || metadata === void 0 ? void 0 : metadata.blockHeight,
blockHash,
blockIndex: metadata === null || metadata === void 0 ? void 0 : metadata.blockIndex,
merkleRoot: metadata === null || metadata === void 0 ? void 0 : metadata.merkleRoot,
firstSeenHeight: firstSeenHeight !== null && firstSeenHeight !== void 0 ? firstSeenHeight : metadata === null || metadata === void 0 ? void 0 : metadata.blockHeight,
proven: metadata !== undefined
};
}
async recomputeTopicBlockAnchor(topic, blockHeight, blockHash) {
var _a, _b, _c;
if (typeof this.storage.findAdmittedTransactionsForBlock !== 'function' ||
typeof this.storage.upsertTopicBlockAnchor !== 'function' ||
typeof this.storage.findTopicBlockAnchor !== 'function') {
return undefined;
}
const anchorBlockHash = blockHash !== null && blockHash !== void 0 ? blockHash : (_a = (await this.storage.findTopicBlockAnchor(topic, blockHeight))) === null || _a === void 0 ? void 0 : _a.blockHash;
if (anchorBlockHash === undefined) {
return undefined;
}
// BRC-136 per-block completeness: establish the chain's genesis at the first
// admitted height, then keep every height from there to the tip contiguous so
// the cumulative TAC never resets across blocks with no admitted transactions.
// We rebuild [fromHeight, toHeight] rather than only the touched height so that
// an out-of-order proof (older height arriving after a newer one) can never
// leave a gap that silently breaks the chain.
const tip = await ((_c = (_b = this.storage).findTopicAnchorTip) === null || _c === void 0 ? void 0 : _c.call(_b, topic));
const tipHeight = tip !== undefined && tip.blockHeight >= 0 ? tip.blockHeight : undefined;
const fromHeight = tipHeight === undefined ? blockHeight : Math.min(blockHeight, tipHeight + 1);
const toHeight = tipHeight === undefined ? blockHeight : Math.max(blockHeight, tipHeight);
await this.rebuildTopicAnchorChain(topic, fromHeight, toHeight, new Map([[blockHeight, anchorBlockHash]]));
return await this.storage.findTopicBlockAnchor(topic, blockHeight);
}
/**
* Extends every configured topic's anchor chain forward with empty Topic Block
* Anchors (basmRoot = zero hash, admittedCount = 0) up to `toHeight`, so the
* cumulative TAC advances on every block even when a topic admits nothing —
* this is what lets a peer authoritatively confirm "this block contained no
* transactions for this topic". Chains with no first admission yet are left
* unstarted (genesis is the topic's first admitted height).
*/
async advanceTopicAnchorChains(toHeight) {
if (typeof this.storage.findTopicAnchorTip !== 'function' ||
typeof this.storage.upsertTopicBlockAnchor !== 'function') {
return;
}
const targetHeight = toHeight !== null && toHeight !== void 0 ? toHeight : await this.currentHeightOrUndefined();
if (targetHeight === undefined) {
return;
}
for (const topic of Object.keys(this.managers)) {
const tip = await this.storage.findTopicAnchorTip(topic);
if (tip === undefined || tip.blockHeight < 0 || tip.blockHeight >= targetHeight) {
continue;
}
await this.rebuildTopicAnchorChain(topic, tip.blockHeight + 1, targetHeight);
}
}
/**
* Rebuilds a contiguous slice of a topic's anchor chain over [fromHeight,
* toHeight]. Each height uses its admitted transactions (empty -> zero basmRoot)
* and chains the cumulative TAC from the prior height. Missing heights are
* filled rather than skipped, so the chain stays gap-free. If a block hash
* cannot be resolved for some height the extension halts there to preserve
* contiguity instead of leaving a hole.
*/
async rebuildTopicAnchorChain(topic, fromHeight, toHeight, blockHashHints = new Map(), forceResolve = false) {
var _a, _b, _c;
if (typeof this.storage.findAdmittedTransactionsForBlock !== 'function' ||
typeof this.storage.upsertTopicBlockAnchor !== 'function' ||
typeof this.storage.findTopicBlockAnchor !== 'function' ||
toHeight < fromHeight) {
return;
}
if (toHeight - fromHeight + 1 > DEFAULT_BASM_RANGE_LIMIT) {
// Bound the work per pass; the next trigger resumes from the new tip.
this.logger.warn(`[BASM] capping anchor chain extension: topic=${(0, SafeLog_js_1.serializeLogValue)(topic)} limit=${(0, SafeLog_js_1.serializeLogValue)(DEFAULT_BASM_RANGE_LIMIT)} requestedFrom=${(0, SafeLog_js_1.serializeLogValue)(fromHeight)} requestedTo=${(0, SafeLog_js_1.serializeLogValue)(toHeight)}; will continue on the next pass`);
toHeight = fromHeight + DEFAULT_BASM_RANGE_LIMIT - 1;
}
const previousAnchor = fromHeight > 0
? await this.storage.findTopicBlockAnchor(topic, fromHeight - 1)
: undefined;
let prevTac = (_a = previousAnchor === null || previousAnchor === void 0 ? void 0 : previousAnchor.tac) !== null && _a !== void 0 ? _a : BASM_js_1.BASM_ZERO_HASH;
for (let height = fromHeight; height <= toHeight; height++) {
const admitted = await this.storage.findAdmittedTransactionsForBlock(topic, height);
const existing = await this.storage.findTopicBlockAnchor(topic, height);
// On a reorg rebuild the existing anchor's block hash is stale, so force
// canonical re-resolution from the header resolver instead of reusing it.
const blockHash = (_c = (_b = blockHashHints.get(height)) !== null && _b !== void 0 ? _b : (forceResolve ? undefined : existing === null || existing === void 0 ? void 0 : existing.blockHash)) !== null && _c !== void 0 ? _c : await this.resolveBlockHash(height);
if (blockHash === undefined) {
this.logger.warn(`[BASM] unable to resolve block hash: topic=${(0, SafeLog_js_1.serializeLogValue)(topic)} height=${(0, SafeLog_js_1.serializeLogValue)(height)}; halting chain extension`);
return;
}
const basmRoot = (0, BASM_js_1.computeBasmRoot)(admitted);
const tac = (0, BASM_js_1.computeTac)(prevTac, blockHash, basmRoot);
await this.storage.upsertTopicBlockAnchor({
topic,
blockHeight: height,
blockHash,
basmRoot,
admittedCount: admitted.length,
tac
});
prevTac = tac;
}
}
/**
* Reconciles BASM anchors with a blockchain reorganization reported by the
* chain tracker (e.g. go-chaintracks `/v2/reorg/stream`). Proven topic
* transactions whose block was orphaned are demoted to unproven so they leave
* the admitted set, then every topic anchor chain intersecting the affected
* height range is rebuilt over the canonical block hashes. A reorg changes the
* canonical block hash for the affected heights, so topics with no demoted
* transaction are rebuilt too. Idempotent: a clean window demotes nothing and
* reproduces an identical TAC, so this is safe to invoke on every reorg event,
* SSE reconnect, and poll.
*/
async handleReorg(input) {
var _a, _b;
const report = { perTopic: [] };
if (typeof this.storage.findProvenAppliedTransactionsByBlockHash !== 'function' ||
typeof this.storage.demoteAppliedTransactionToUnproven !== 'function' ||
typeof this.storage.findTopicBlockAnchors !== 'function' ||
typeof this.storage.upsertTopicBlockAnchor !== 'function') {
return report;
}
// 1) Demote proven admissions whose block was orphaned. Hashes are
// normalized to lower-case display hex to match stored block hashes
// (go-sdk chainhash.Hash marshals as reversed display hex).
const demotedByTopic = new Map();
for (const rawHash of input.orphanedBlockHashes) {
const blockHash = rawHash.toLowerCase();
const rows = await this.storage.findProvenAppliedTransactionsByBlockHash(blockHash);
for (const row of rows) {
await this.storage.demoteAppliedTransactionToUnproven(row.txid, row.topic);
const list = (_a = demotedByTopic.get(row.topic)) !== null && _a !== void 0 ? _a : [];
list.push(row.txid);
demotedByTopic.set(row.topic, list);
}
}
// 2) Rebuild every topic anchor chain that intersects the reorged range,
// forcing canonical block-hash re-resolution so stale hashes are replaced.
for (const topic of Object.keys(this.managers)) {
const existing = await this.storage.findTopicBlockAnchors(topic, input.rebuildFromHeight, input.newTipHeight);
if (existing.length === 0) {
continue;
}
const startHeight = Math.min(...existing.map(anchor => anchor.blockHeight));
await this.rebuildTopicAnchorChain(topic, startHeight, input.newTipHeight, new Map(), true);
report.perTopic.push({
topic,
demotedTxids: (_b = demotedByTopic.get(topic)) !== null && _b !== void 0 ? _b : [],
rebuiltFrom: startHeight,
rebuiltTo: input.newTipHeight
});
}
return report;
}
/**
* Revalidation sweep: the reorg fallback for chain trackers without a reorg
* event stream, and the catch-up step on every reorg-SSE (re)connect (the
* go-chaintracks reorg stream carries no event ids, so a reconnect cannot
* replay events missed while disconnected). Scans proven applied transactions
* in `[tip - depth + 1, tip]`; any whose proof root no longer validates against
* the chain tracker, or whose block hash diverges from the canonical header, is
* treated as orphaned and reconciled via {@link handleReorg}.
*/
async isProvenAnchorStale(row, chainTracker) {
var _a;
if (row.merkleRoot !== undefined) {
try {
if (!(await chainTracker.isValidRootForHeight(row.merkleRoot, row.blockHeight))) {
return true;
}
}
catch (error) {
this.logger.warn(`[BASM] root validation failed for ${row.txid} at height ${row.blockHeight}: ${error instanceof Error ? error.message : String(error)}`);
return undefined;
}
}
const canonical = await this.resolveBlockHash(row.blockHeight);
return canonical !== undefined && canonical.toLowerCase() !== ((_a = row.blockHash) === null || _a === void 0 ? void 0 : _a.toLowerCase());
}
async revalidateRecentAnchors(depth = 3) {
const chainTracker = this.chainTracker;
if (chainTracker === 'scripts only') {
this.logger.warn('[BASM] revalidation sweep requires a ChainTracker; skipping');
return undefined;
}
if (typeof this.storage.findProvenAppliedTransactionsInRange !== 'function') {
return undefined;
}
const tip = await this.currentHeightOrUndefined();
if (tip === undefined) {
return undefined;
}
const fromHeight = Math.max(0, tip - depth + 1);
const rows = await this.storage.findProvenAppliedTransactionsInRange(fromHeight, tip);
const orphaned = new Set();
let minAffected = Number.POSITIVE_INFINITY;
for (const row of rows) {
if (row.blockHash === undefined) {
continue;
}
const stale = await this.isProvenAnchorStale(row, chainTracker);
if (stale === true) {
orphaned.add(row.blockHash.toLowerCase());
minAffected = Math.min(minAffected, row.blockHeight);
}
}
if (orphaned.size === 0) {
return { perTopic: [] };
}
return await this.handleReorg({
orphanedBlockHashes: Array.from(orphaned),
rebuildFromHeight: minAffected,
newTipHeight: tip
});
}
async validateTopicSubmission(topic, context) {
const { tx, txid, beef, offChainValues, mode, dupeTopics, failedTopics } = context;
try {
if (this.managers[topic] === undefined || this.managers[topic] === null) {
throw new Error(`This server does not support this topic: ${topic}`);
}
this.startTime(`dupCheck_${txid.substring(0, 10)}`);
const isDupe = await this.storage.doesAppliedTransactionExist({ txid, topic });
this.endTime(`dupCheck_${txid.substring(0, 10)}`);
if (isDupe) {
dupeTopics.add(topic);
return {
topic,
isDupe: true,
previousCoins: [],
previousOutputs: [],
admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
};
}
const previousCoins = [];
const outputPromises = tx.inputs.map(async (input, inputIndex) => {
var _a;
var _b;
const previousTXID = (_b = input.sourceTXID) !== null && _b !== void 0 ? _b : (_a = input.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex');
if (previousTXID === undefined)
return null;
const output = await this.storage.findOutput(previousTXID, input.sourceOutputIndex, topic);
if (output !== undefined && output !== null)
previousCoins.push(inputIndex);
return output !== null && output !== void 0 ? output : null;
});
this.startTime(`previousOutputQuery_${txid.substring(0, 10)}`);
const previousOutputs = await Promise.all(outputPromises);
this.endTime(`previousOutputQuery_${txid.substring(0, 10)}`);
this.startTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
const admissibleOutputs = await this.managers[topic].identifyAdmissibleOutputs(beef, previousCoins, offChainValues, mode);
this.endTime(`identifyAdmissibleOutputs_${txid.substring(0, 10)}`);
return {
topic,
isDupe: false,
previousCoins,
previousOutputs,
admissibleOutputs
};
}
catch (error) {
this.logger.error(`Error validating topic during submit: topic=${(0, SafeLog_js_1.serializeLogValue)(topic)} error=${(0, SafeLog_js_1.serializeErrorForLog)(error)}`);
failedTopics.add(topic);
return {
topic,
isDupe: false,
previousCoins: [],
previousOutputs: [],
admissibleOutputs: { outputsToAdmit: [], coinsToRetain: [] }
};
}
}
isTopicSubmissionAccepted(validation, failedTopics) {
return !failedTopics.has(validation.topic) && (validation.isDupe ||
validation.admissibleOutputs.outputsToAdmit.length > 0 ||
validation.admissibleOutputs.coinsToRetain.length > 0 ||
validation.previousCoins.length > 0);
}
async broadcastAcceptedSubmission(tx, txid, mode, anyTopicAccepted) {
this.startTime(`broadcast_${txid.substring(0, 10)}`);
if (mode !== 'historical-tx' && this.broadcaster !== undefined && anyTopicAccepted) {
try {
let response;
if (tx.merklePath !== undefined) {
const mp = tx.merklePath;
const leaf = mp.path[0].find(leaf => leaf.hash === txid);
response = {
status: 'success',
txid,
message: `In block at height ${mp.blockHeight} index ${leaf === null || leaf === void 0 ? void 0 : leaf.offset}`
};
}
else {
response = await this.broadcaster.broadcast(tx);
}
if ((0, sdk_1.isBroadcastFailure)(response) && this.throwOnBroadcastFailure) {
const error = new Error(`Failed to broadcast transaction! Error: ${response.description}`);
error.more = response.more;
throw error;
}
}
catch (error) {
if (this.throwOnBroadcastFailure)
throw error;
this.logger.error('Error broadcasting transaction:', error);
}
}
this.endTime(`broadcast_${txid.substring(0, 10)}`);
}
async notifyOutputSpent(lookupService, tx, txid, output, topic, offChainValues) {
var _a;
if (typeof lookupService.outputSpent !== 'function')
return;
if (lookupService.spendNotificationMode === 'txid') {
await lookupService.outputSpent({
mode: 'txid',
spendingTxid: txid,
txid: output.txid,
outputIndex: output.outputIndex,
topic
});
return;
}
if (lookupService.spendNotificationMode === 'script') {
const inputIndex = findSpendingInputIndex(tx, output);
if (inputIndex === -1)
throw new Error('Could not find input index');
await lookupService.outputSpent({
mode: 'script',
spendingTxid: txid,
inputIndex,
sequenceNumber: (_a = tx.inputs[inputIndex].sequence) !== null && _a !== void 0 ? _a : 0xffffffff,
unlockingScript: tx.inputs[inputIndex].unlockingScript,
txid: output.txid,
outputIndex: output.outputIndex,
topic,
offChainValues
});
return;
}
if (lookupService.spendNotificationMode === 'whole-tx') {
await lookupService.outputSpent({
mode: 'whole-tx',
spendingAtomicBEEF: tx.toAtomicBEEF(),
txid: output.txid,
outputIndex: output.outputIndex,
topic,
offChainValues
});
return;
}
await lookupService.outputSpent({
mode: 'none',
txid: output.txid,
outputIndex: output.outputIndex,
topic
});
}
async markPreviousOutputSpent(output, topic, tx, txid, offChainValues) {
if (output === null)
return;
try {
await this.storage.markUTXOAsSpent(output.txid, output.outputIndex, topic);
await Promise.all(Object.values(this.lookupServices).map(async (lookupService) => {
try {
await this.notifyOutputSpent(lookupService, tx, txid, output, topic, offChainValues);
}
catch (error) {
this.logger.error('Error in lookup service for outputSpent:', error);
}
}));
}
catch (error) {
this.logger.error('Error marking UTXO as spent:', error);
}
}
async markPreviousOutputsSpent(validations, failedTopics, tx, txid, offChainValues) {
await Promise.all(validations.map(async (validation) => {
if (validation.isDupe || failedTopics.has(validation.topic))
return;
await Promise.all(validation.previousOutputs.map(async (output) => {
await this.markPreviousOutputSpent(output, validation.topic, tx, txid, offChainValues);
}));
}));
}
classifyPreviousCoins(tx, validation) {
var _a;
var _b;
const outputsConsumed = [];
const outputsToMarkStale = [];
for (const inputIndex of validation.previousCoins) {
const input = tx.inputs[inputIndex];
const previousTXID = (_b = input.sourceTXID) !== null && _b !== void 0 ? _b : (_a = input.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex');
if (typeof previousTXID !== 'string')
continue;
if (validation.admissibleOutputs.coinsToRetain.includes(inputIndex)) {
outputsConsumed.push({
txid: previousTXID,
outputIndex: input.sourceOutputIndex
});
}
else {
outputsToMarkStale.push({
txid: previousTXID,
previousOutputIndex: input.sourceOutputIndex,
inputIndex
});
}
}
return { outputsConsumed, outputsToMarkStale };
}
async removeStaleOutputs(outputs, topic, txid) {
this.startTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
await Promise.all(outputs.map(async (coin) => {
const output = await this.storage.findOutput(coin.txid, coin.previousOutputIndex, topic);
if (output !== undefined && output !== null)
await this.deleteUTXODeep(output);
}));
this.endTime(`lookForStaleOutputs_${txid.substring(0, 10)}`);
}
async notifyOutputAdmitted(lookupService, tx, txid, outputIndex, topic, offChainValues) {
if (lookupService.admissionMode === 'locking-script') {
if (typeof tx.outputs[outputIndex].lockingScript !== 'object' ||
typeof tx.outputs[outputIndex].satoshis !== 'number')
return;
await lookupService.outputAdmittedByTopic({
mode: 'locking-script',
txid,
outputIndex,
lockingScript: tx.outputs[outputIndex].lockingScript,
satoshis: tx.outputs[outputIndex].satoshis,
topic,
offChainValues
});
return;
}
await lookupService.outputAdmittedByTopic({
mode: 'whole-tx',
atomicBEEF: tx.toAtomicBEEF(),
outputIndex,
topic,
offChainValues
});
}
async admitOutput(outputIndex, context) {
var _a;
const { tx, txid, beef, topic, outputsConsumed, newUTXOs, offChainValues } = context;
if (typeof tx.outputs[outputIndex].satoshis !== 'number')
return;
this.startTime(`insertNewOutput_${txid.substring(0, 10)}`);
await this.storage.insertOutput({
txid,
outputIndex,
outputScript: tx.outputs[outputIndex].lockingScript.toBinary(),
satoshis: tx.outputs[outputIndex].satoshis,
topic,
spent: false,
beef: this.compactBEEFForStorage(tx, beef),
consumedBy: [],
outputsConsumed,
score: Date.now(),
blockHeight: (_a = (0, BASM_js_1.extractMerkleProofMetadata)(txid, tx.merklePath)) === null || _a === void 0 ? void 0 : _a.blockHeight
});
this.endTime(`insertNewOutput_${txid.substring(0, 10)}`);
newUTXOs.push({ txid, outputIndex });
this.startTime(`notifyLookupService${txid.substring(0, 10)}`);
await Promise.all(Object.values(this.lookupServices).map(async (lookupService) => {
try {
await this.notifyOutputAdmitted(lookupService, tx, txid, outputIndex, topic, offChainValues);
}
catch (error) {
this.logger.error('Error in lookup service for outputAdmittedByTopic:', error);
}
}));
this.endTime(`notifyLookupService${txid.substring(0, 10)}`);
}
async updateConsumedOutput(output, newUTXOs, topic) {
const storedOutput = await this.storage.findOutput(output.txid, output.outputIndex, topic);
if (storedOutput === undefined || storedOutput === null)
return;
const consumedBy = [...new Set([...newUTXOs, ...storedOutput.consumedBy])];
await this.storage.updateConsumedBy(output.txid, output.outputIndex, topic, consumedBy);
}
async applyTopicStorageMutation(validation, steak, tx, txid, beef, offChainValues) {
const topic = validation.topic;
const { outputsConsumed, outputsToMarkStale } = this.classifyPreviousCoins(tx, validation);
await this.removeStaleOutputs(outputsToMarkStale, topic, txid);
steak[topic].coinsRemoved = outputsToMarkStale.map(output => output.inputIndex);
const newUTXOs = [];
await Promise.all(validation.admissibleOutputs.outputsToAdmit.map(async (outputIndex) => {
await this.admitOutput(outputIndex, {
tx,
txid,
beef,
topic,
outputsConsumed,
newUTXOs,
offChainValues
});
}));
this.startTime(`outputConsumed_${txid.substring(0, 10)}`);
const appliedRecord = await this.buildAppliedTransactionRecord(tx);
await this.recordTransactionData(tx, beef, appliedRecord.blockHash);
await Promise.all([
...outputsConsumed.map(async (output) => {
await this.updateConsumedOutput(output, newUTXOs, topic);
}),
this.storage.insertAppliedTransaction({ txid, topic, ...appliedRecord })
]);
if (appliedRecord.blockHeight !== undefined && appliedRecord.blockHash !== undefined) {
await this.recomputeTopicBlockAnchor(topic, appliedRecord.blockHeight, appliedRecord.blockHash);
}
this.endTime(`outputConsumed_${txid.substring(0, 10)}`);
}
async applyStorageMutations(validations, context) {
const { dupeTopics, failedTopics, steak, tx, txid, beef, offChainValues } = context;
for (const validation of validations) {
const topic = validation.topic;
if (dupeTopics.has(topic) || failedTopics.has(topic))
continue;
try {
await this.applyTopicStorageMutation(validation, steak, tx, txid, beef, offChainValues);
}
catch (error) {
this.logger.error('Error updating storage and notifying lookup services for topic', topic, error);
}
}
}
async propagateSubmission(taggedBEEF, steak, dupeTopics, tx, txid) {
this.startTime(`transactionPropagation_${txid.substring(0, 10)}`);
const relevantTopics = taggedBEEF.topics.filter(topic => { var _a; return steak[topic] !== undefined &&
!dupeTopics.has(topic) &&
(steak[topic].outputsToAdmit.length !== 0 ||
((_a = steak[topic].coinsRemoved) === null || _a === void 0 ? void 0 : _a.length) !== 0); });
if (relevantTopics.length === 0) {
this.endTime(`transactionPropagation_${txid.substring(0, 10)}`);
return;
}
let customBroadcasterConfig;
if (Array.isArray(this.slapTrackers)) {
const resolverConfig = {
slapTrackers: this.slapTrackers
};
customBroadcasterConfig = {
resolver: new sdk_1.LookupResolver(resolverConfig)
};
}
const shipBroadcaster = new sdk_1.SHIPBroadcaster(relevantTopics, customBroadcasterConfig);
try {
await shipBroadcaster.broadcast(tx);
}
catch (error) {
this.logger.error('Error during propagation to other nodes:', error);
}
this.endTime(`transactionPropagation_${txid.substring(0, 10)}`);
}
/**
* Submits a transaction for processing by Overlay Services.
* @param {TaggedBEEF} taggedBEEF - The transaction to process
* @param {function(STEAK): void} [onSTEAKReady] - Optional callback function invoked when the STEAK is ready.
* @param {string} mode — Indicates the submission behavior, whether historical or current. Historical transactions are not broadcast or propagated.
* @param {number[]} offChainValues — Values necessary to evaluate topical admittance that are not stored on-chain.
*
* The optional callback function should be used to get STEAK when ready, and avoid waiting for broadcast and transaction propagation to complete.
*
* @returns {Promise<STEAK>} The submitted transaction execution acknowledgement
*/
async submit(taggedBEEF, onSteakReady, mode = 'current-tx', offChainValues) {
for (const t of taggedBEEF.topics) {
if (this.managers[t] === undefined || this.managers[t] === null) {
throw new Error(`This server does not support this topic: ${t}`);
}
}
// Validate the transaction SPV information
const tx = sdk_1.Transaction.fromBEEF(taggedBEEF.beef);
const txid = tx.id('hex');
this.startTime(`submit_${txid}`);
if (mode !== 'historical-tx-no-spv') {
this.startTime(`chainTracker_${txid.substring(0, 10)}`);
const txValid = await tx.verify(this.chainTracker);
if (!txValid)
throw new Error('Unable to verify SPV information.');
this.endTime(`chainTracker_${txid.substring(0, 10)}`);
}
const steak = {};
const dupeTopics = new Set();
const failedTopics = new Set();
// ===================================================================
// PHASE 1: VALIDATE (read-only, no mutations)
// ===================================================================
const topicValidations = taggedBEEF.topics.map(async (topic) => await this.validateTopicSubmission(topic, {
tx,
txid,
beef: taggedBEEF.beef,
offChainValues,
mode,
dupeTopics,
failedTopics
}));
const validations = await Promise.all(topicValidations);
// Build preliminary STEAK from validation results
for (const validation of validations) {
steak[validation.topic] = validation.admissibleOutputs;
}
// ===================================================================
// PHASE 2: BROADCAST (before any mutations)
// ===================================================================
// Only broadcast when at least one topic actually accepted the
// transaction. For a non-failed topic, acceptance means: previously
// accepted (dupe / client retry), outputs admitted, coins retained, or
// previously-admitted coins consumed (e.g. a consume-only deletion such
// as a KVStore remove, even one that retains nothing). A topic manager
// REJECTS by throwing from identifyAdmissibleOutputs (tracked in
// failedTopics). A transaction every topic rejected must never reach the
// network: submitters treat an empty STEAK as a rejection and
// abort/release their held inputs, so broadcasting it anyway would
// desync their wallets from the chain.
const anyTopicAccepted = validations.some(validation => this.isTopicSubmissionAccepted(validation, failedTopics));
await this.broadcastAcceptedSubmission(tx, txid, mode, anyTopicAccepted);
// Call the callback function with STEAK if it is provided (before storage mutations)
if (onSteakReady !== undefined) {
onSteakReady(steak);
}
// ===================================================================
// PHASE 3: MUTATE STORAGE (only after broadcast succeeded)
// ===================================================================
// Mark previous outputs as spent and notify lookup services
await this.markPreviousOutputsSpent(validations, failedTopics, tx, txid, offChainValues);
await this.applyStorageMutations(validations, {
dupeTopics,
failedTopics,
steak,
tx,
txid,
beef: taggedBEEF.beef,
offChainValues
});
// If we don't have an advertiser or we are dealing with historical transactions, just return the steak
if (this.advertiser === undefined || mode === 'historical-tx' || mode === 'historical-tx-no-spv') {
return steak;
}
await this.propagateSubmission(taggedBEEF, steak, dupeTopics, tx, txid);
// Immediately return from the function without waiting for the promises to resolve.
return steak;
}
/**
* Submit a lookup question to the Overlay Services Engine, and receive back a Lookup Answer
* @param LookupQuestion — The question to ask the Overlay Services Engine
* @returns The answer to the question
*/
async lookup(lookupQuestion) {
// Validate a lookup service for the provider is found
const lookupService = this.lookupServices[lookupQuestion.service];
if (lookupService === undefined || lookupService === null)
throw new Error(`Lookup service not found for provider: ${lookupQuestion.service}`);
const lookupResult = await lookupService.lookup(lookupQuestion);
if (this.maxLookupResults !== -1 && lookupResult.length > this.maxLookupResults) {
throw new RangeError(`Lookup returned ${lookupResult.length} results; maximum is ${this.maxLookupResults}`);
}
const hydrationContext = this.createUTXOHistoryHydrationContext();
await this.preloadOutputsWithBEEF(lookupResult.map(({ txid, outputIndex }) => ({ txid, outputIndex })), hydrationContext);
const hydratedOutputs = (await Promise.all(lookupResult.map(async ({ txid, outputIndex, history, context }) => {
const UTXO = await this.loadOutputWithBEEF(txid, outputIndex, hydrationContext);
if (UTXO === null) {
return null;
}
// Get the history for this utxo and construct a BEEF
const output = await this.getUTXOHistory(UTXO, history, 0, hydrationContext);
if ((output === null || output === void 0 ? void 0 : output.beef) === undefined) {
return null;
}
return {
beef: output.beef,
outputIndex: output.outputIndex,
context
};
})))
.filter((output) => output !== null)
.map(({ beef, outputIndex, context }) => (context === undefined
? { beef, outputIndex }
: { beef, outputIndex, context }));
return {
type: 'output-list',
outputs: hydratedOutputs
};
}
createUTXOHistoryHydrationContext() {
return {
outputCache: new Map()
};
}
toOutputCacheKey(txid, outputIndex) {
return `${txid}:${outputIndex}`;
}
async preloadOutputsWithBEEF(outpoints, context) {
var _a;
if (outpoints.length === 0) {
return;
}
const deduped = [];
const seen = new Set();
for (const outpoint of outpoints) {
const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex);
if (seen.has(cacheKey)) {
continue;
}
seen.add(cacheKey);
if (!context.outputCache.has(cacheKey)) {
deduped.push(outpoint);
}
}
if (deduped.length === 0) {
return;
}
const findOutputsByOutpoints = this.storage.findOutputsByOutpoints;
if (typeof findOutputsByOutpoints === 'function') {
const outputs = await findOutputsByOutpoints.call(this.storage, deduped, true);
const outputsByKey = new Map();
for (const output of outputs) {
outputsByKey.set(this.toOutputCacheKey(output.txid, output.outputIndex), output);
}
for (const outpoint of deduped) {
const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex);
context.outputCache.set(cacheKey, Promise.resolve((_a = outputsByKey.get(cacheKey)) !== null && _a !== void 0 ? _a : null));
}
return;
}
for (const outpoint of deduped) {
const cacheKey = this.toOutputCacheKey(outpoint.txid, outpoint.outputIndex);
context.outputCache.set(cacheKey, this.storage.findOutput(outpoint.txid, outpoint.outputIndex, undefined, undefined, true));
}
}
async loadOutputWithBEEF(txid, outputIndex, context) {
const cacheKey = this.toOutputCacheKey(txid, outputIndex);
let cached = context.outputCache.get(cacheKey);
if (cached === undefined) {
cached = this.storage.findOutput(txid, outputIndex, undefined, undefined, true);
context.outputCache.set(cacheKey, cached);
}
const output = await cached;
return output !== null && output !== void 0 ? output : null;
}
async hydrateUTXOHistoryNode(output, historySelector, currentDepth, context) {
if (output.beef === undefined) {
throw new Error('Output must have associated transaction BEEF!');
}
let shouldTraverseHistory;
if (typeof historySelector === 'number') {
shouldTraverseHistory = currentDepth <= historySelector;
}
else {
shouldTraverseHistory = await historySelector(output.beef, output.outputIndex, currentDepth);
}
if (shouldTraverseHistory === false) {
return undefined;
}
await this.preloadOutputsWithBEEF(output.outputsConsumed, context);
const childNodes = (await Promise.all(output.outputsConsumed.map(async (outputIdentifier) => {
const childOutput = await this.loadOutputWithBEEF(outputIdentifier.txid, outputIdentifier.outputIndex, context);
if (childOutput === null) {
return undefined;
}
return await this.hydrateUTXOHistoryNode(childOutput, historySelector, currentDepth + 1, context);
}))).filter((node) => node !== undefined);
const tx = sdk_1.Transaction.fromBEEF(output.beef);
const inputIndexBySource = new Map();
tx.inputs.forEach((candidateInput, index) => {
var _a;
const sourceTXID = candidateInput.sourceTXID !== undefined && candidateInput.sourceTXID !== ''
? candidateInput.sourceTXID
: (_a = candidateInput.sourceTransaction) === null || _a === void 0 ? void 0 : _a.id('hex');
if (sourceTXID === undefined) {
return;
}
inputIndexBySource.set(`${sourceTXID}:${candidateInput.sourceOutputIndex}`, index);
});
for (const child of childNodes) {
const inputIndex = inputIndexBySource.get(`${child.output.txid}:${child.output.outputIndex}`);
if (inputIndex === -1 || inputIndex == null) {
continue;
}
const targetInput = tx.inputs[inputIndex];
if (!targetInput) {
this.logger.error(`Input at index ${inputIndex} is undefined, but findIndex found it. Possible sparse array from BEEF parsing.`);
continue;
}
targetInput.sourceTransaction = child.transaction;
}
return { output, transaction: tx };
}
/**
* Ensures alignment between the current SHIP/SLAP advertisements and the
* configured Topic Managers and Lookup Services in the engine.
*
* This method performs the following actions:
* 1. Retrieves the current configuration of topics and services.
* 2. Fetches the existing SHIP advertisements for each configured topic.
* 3. Fetches the existing SLAP advertisements for each configured service.
* 4. Compares t