@bsv/overlay-express
Version:
BSV Blockchain Overlay Express
1,170 lines • 97.4 kB
JavaScript
import express from 'express';
import bodyParser from 'body-parser';
import { Engine, KnexStorage, KnexStorageMigrations } from '@bsv/overlay';
import { ARC, MerklePath, WhatsOnChain, HTTPSOverlayBroadcastFacilitator, DEFAULT_TESTNET_SLAP_TRACKERS, DEFAULT_SLAP_TRACKERS, Utils, Beef, Transaction, PrivateKey, KeyDeriver } from '@bsv/sdk';
import Knex from 'knex';
import { MongoClient } from 'mongodb';
import makeUserInterface from './makeUserInterface.js';
import * as DiscoveryServices from '@bsv/overlay-discovery-services';
import chalk from 'chalk';
import util from 'node:util';
import { v4 as uuidv4 } from 'uuid';
import { JanitorService } from './JanitorService.js';
import { BanService } from './BanService.js';
import { BanAwareLookupWrapper } from './BanAwareLookupWrapper.js';
import { BanAwareTopicManager } from './BanAwareTopicManager.js';
import { BanAwareSHIPStorage, BanAwareSLAPStorage } from './BanAwareDiscoveryStorage.js';
import { ReorgSseAdapter } from './ReorgStream.js';
import { Wallet, WalletSigner, WalletStorageManager, Services } from '@bsv/wallet-toolbox-client';
import { createAuthMiddleware } from '@bsv/auth-express-middleware';
import { ArcadeProvider, isTerminalArcStatus } from './ArcadeProvider.js';
import { ProviderChainBroadcaster } from './ProviderChainBroadcaster.js';
import { ChaintracksProvider } from './ChaintracksProvider.js';
/**
* In-memory migration source for Knex migrations.
* Allows running migrations defined in code rather than files.
*/
class InMemoryMigrationSource {
migrations;
constructor(migrations) {
this.migrations = migrations;
}
/**
* Gets the list of migrations.
* @param loadExtensions - Array of file extensions to filter by (not used here)
* @returns Promise resolving to the array of migrations
*/
async getMigrations(loadExtensions) {
return this.migrations;
}
/**
* Gets the name of a migration.
* @param migration - The migration object
* @returns The name of the migration
*/
getMigrationName(migration) {
return typeof migration.name === 'string' ? migration.name : `Migration at index ${this.migrations.indexOf(migration)}`;
}
/**
* Gets the migration object.
* @param migration - The migration object
* @returns Promise resolving to the migration object
*/
async getMigration(migration) {
return await Promise.resolve(migration);
}
}
function parseTopicsHeader(header) {
const value = header.trim();
const parsed = value.startsWith('[')
? JSON.parse(value)
: value.split(',').map(topic => topic.trim());
if (!Array.isArray(parsed) || parsed.some(topic => typeof topic !== 'string' || topic.length === 0)) {
throw new TypeError('Invalid x-topics header: expected a comma-separated list or JSON string array');
}
return parsed;
}
/**
* OverlayExpress class provides an Express-based server for hosting Overlay Services.
* It allows configuration of various components like databases, topic managers, and lookup services.
* It encapsulates an Express application and provides methods to start the server.
*/
export default class OverlayExpress {
name;
privateKey;
advertisableFQDN;
// Express application
app;
// Server port
port = 3000;
// Logger (defaults to console)
logger = console;
// Knex (SQL) database
knex;
// Knex migrations to run
migrationsToRun = [];
// MongoDB database
mongoDb;
// MongoDB client retained for health checks
mongoClient;
// Network ('main' or 'test')
network = 'main';
// If no custom ChainTracker is configured, default is a WhatsOnChain instance
// (We keep a property for it, so we can pass it to Engine)
chainTracker = new WhatsOnChain(this.network);
// The Overlay Engine
engine;
// Configured Topic Managers
managers = {};
// Configured Lookup Services
services = {};
// Enable GASP Sync
// (We allow an on/off toggle, but also can do advanced custom sync config below)
enableGASPSync = true;
// Enable BRC-136 BASM sync. Off by default; endpoints remain available when
// storage supports anchors.
enableBASMSync = false;
// Opt-in unproven eviction default threshold in blocks.
unprovenEvictionBlocks = 144;
// How often (ms) to poll the chain tip and extend each topic's BASM anchor
// chain with empty anchors, so the cumulative TAC advances "after each new
// block" per BRC-136. Set to 0 to disable polling (startup extension still runs).
basmBlockPollIntervalMs = 10 * 60 * 1000;
// Handle for the BASM block-poll timer so it can be stopped.
basmBlockPollTimer;
// How often (ms) to refresh proofs for old unproven rows and then evict rows
// that still have no proof. Set to 0 to disable background maintenance.
unprovenMaintenanceIntervalMs = 0;
unprovenMaintenanceTimer;
// Optional go-chaintracks (Arcade) reorg SSE URL (e.g. `<base>/v2/reorg/stream`).
// When set, reorgs are reconciled in real time; the block poll also runs a
// revalidation sweep as a fallback / reconnect catch-up.
reorgStreamUrl;
// Depth (in blocks from the tip) for the reorg revalidation sweep.
reorgScanDepth = 3;
// Handle for the reorg SSE adapter.
reorgAdapter;
// Optional resolver for block hashes and header merkle roots used by BASM.
topicAnchorHeaderResolver;
// ARC API Key
arcApiKey = undefined;
// Optional ARC callback token for /arc-ingest notifications
arcCallbackToken = undefined;
// Optional Arcade URL/API key used for propagation, proof refresh, and
// go-chaintracks header/reorg access when available.
arcadeUrl = undefined;
arcadeApiKey = undefined;
arcadeDeploymentId = undefined;
arcadeChaintracksApiPrefix = '/chaintracks/v2';
arcadeProvider;
// Verbose request logging
verboseRequestLogging = false;
// Web UI configuration
webUIConfig = {};
// Additional advanced engine config (these map to Engine constructor parameters).
// Default to undefined or default values that are used in the Engine if not specified.
engineConfig = {};
// The administrative Bearer token used for the admin routes.
// If not passed in, we'll generate a random one.
adminToken;
// Configuration for the janitor service
janitorConfig = {
requestTimeoutMs: 10000, // 10 seconds
hostDownRevokeScore: 3,
autoBanOnRemoval: true
};
// Ban service for persistent domain/outpoint blocking
banService;
// Admin identity key for wallet-based admin detection on the frontend
adminIdentityKey;
// Server-side wallet (WalletInterface) used for BSV mutual authentication
serverWallet;
// Server start time for uptime tracking
startTime;
// Health endpoint configuration
healthConfig = {
includeDetails: true,
timeoutMs: 5000
};
// Extra application-specific health checks
healthChecks = [];
// Lifecycle marker for readiness/liveness reporting
isListening = false;
/**
* Constructs an instance of OverlayExpress.
* @param name - The name of the service
* @param privateKey - Private key used for signing advertisements
* @param advertisableFQDN - The fully qualified domain name where this service is available. Does not include "https://".
* @param adminToken - Optional. An administrative Bearer token used to protect admin routes.
* If not provided, a random token will be generated at runtime.
*/
constructor(name, privateKey, advertisableFQDN, adminToken) {
this.name = name;
this.privateKey = privateKey;
this.advertisableFQDN = advertisableFQDN;
this.app = express();
this.logger.log(chalk.green.bold(`${name} constructed`));
this.adminToken = adminToken ?? uuidv4(); // generate random if not provided
}
/**
* Returns the current admin token in case you need to programmatically retrieve or display it.
*/
getAdminToken() {
return this.adminToken;
}
/**
* Configures the port on which the server will listen.
* @param port - The port number
*/
configurePort(port) {
this.port = port;
this.logger.log(chalk.blue(`Server port set to ${port}`));
}
/**
* Configures the web user interface
* @param config - Web UI configuration options
*/
configureWebUI(config) {
this.webUIConfig = config;
this.logger.log(chalk.blue('Web UI has been configured.'));
}
/**
* Configures the janitor service parameters
* @param config - Janitor configuration options
* - requestTimeoutMs: Timeout for health check requests (default: 10000ms)
* - hostDownRevokeScore: Number of consecutive failures before deleting output (default: 3)
* - autoBanOnRemoval: Whether to auto-ban domains when removed by janitor (default: true)
*/
configureJanitor(config) {
this.janitorConfig = {
...this.janitorConfig,
...config
};
this.logger.log(chalk.blue('Janitor service has been configured.'));
}
/**
* Configures health-report behavior.
*/
configureHealth(config) {
this.healthConfig = {
...this.healthConfig,
...config
};
this.logger.log(chalk.blue('Health reporting has been configured.'));
}
/**
* Registers an application-specific health check.
*/
registerHealthCheck(definition) {
this.healthChecks = this.healthChecks.filter(check => check.name !== definition.name);
this.healthChecks.push({
scope: 'ready',
critical: false,
...definition
});
this.logger.log(chalk.blue(`Registered health check ${definition.name}`));
}
/**
* Configures the admin identity key for wallet-based admin detection.
* When set, the frontend can compare the user's wallet identity key against this
* to determine whether to show the admin dashboard.
*
* @param identityKey - The hex-encoded public key of the admin
*/
configureAdminIdentityKey(identityKey) {
this.adminIdentityKey = identityKey;
this.logger.log(chalk.blue('Admin identity key has been configured.'));
}
/**
* Configures the logger to be used by the server.
* @param logger - A logger object (e.g., console)
*/
configureLogger(logger) {
this.logger = logger;
this.logger.log(chalk.blue('Logger has been configured.'));
}
/**
* Configures the BSV Blockchain network to be used ('main' or 'test').
* By default, it re-initializes chainTracker as a WhatsOnChain for that network.
* @param network - The network ('main' or 'test')
*/
configureNetwork(network) {
this.network = network;
this.chainTracker = new WhatsOnChain(this.network);
this.logger.log(chalk.blue(`Network set to ${network}`));
}
/**
* Configures the ChainTracker to be used.
* If 'scripts only' is used, it implies no full SPV chain tracking in the Engine.
* @param chainTracker - An instance of ChainTracker or 'scripts only'
*/
configureChainTracker(chainTracker = new WhatsOnChain(this.network)) {
this.chainTracker = chainTracker;
this.logger.log(chalk.blue('ChainTracker has been configured.'));
}
/**
* Configures the ARC API key.
* @param apiKey - The ARC API key
*/
configureArcApiKey(apiKey) {
this.arcApiKey = apiKey;
this.logger.log(chalk.blue('ARC API key has been configured.'));
}
/**
* Configures the ARC callback token expected by /arc-ingest.
* @param token - The token ARC should present when posting callback notifications.
*/
configureArcCallbackToken(token) {
this.arcCallbackToken = token;
this.logger.log(chalk.blue('ARC callback token has been configured.'));
}
/**
* Configures Arcade for first-choice transaction propagation and proof lookup.
*/
configureArcade(url, config = {}) {
this.arcadeUrl = url;
this.arcadeApiKey = config.apiKey;
this.arcadeDeploymentId = config.deploymentId;
if (config.chaintracksApiPrefix !== undefined) {
this.arcadeChaintracksApiPrefix = config.chaintracksApiPrefix;
}
this.logger.log(chalk.blue('Arcade provider has been configured.'));
}
/**
* Configures a go-chaintracks compatible service for header validation and
* BASM reorg streaming. Arcade exposes this at `/chaintracks/v2`.
*/
configureChaintracks(url, config = {}) {
const apiPrefix = config.apiPrefix ?? '/chaintracks/v2';
const client = new ChaintracksProvider(url, { apiPrefix });
this.configureChainTracker(client);
this.configureTopicAnchorHeaderResolver(async (blockHeight) => {
const header = await client.findHeaderForHeight(blockHeight);
if (header === undefined)
return undefined;
return {
blockHeight,
blockHash: header.hash,
merkleRoot: header.merkleRoot
};
});
if (config.reorgStream !== false) {
this.configureReorgStream(client.reorgStreamUrl(), config.scanDepth);
}
this.logger.log(chalk.blue('go-chaintracks provider has been configured.'));
}
/**
* Enables or disables GASP synchronization (high-level setting).
* This is a broad toggle that can be overridden or customized through syncConfiguration.
* @param enable - true to enable, false to disable
*/
configureEnableGASPSync(enable) {
this.enableGASPSync = enable;
this.logger.log(chalk.blue(`GASP synchronization ${enable ? 'enabled' : 'disabled'}.`));
}
/**
* Enables or disables BRC-136 BASM synchronization.
* BASM is opt-in because it requires direct proofs and block hash resolution.
*/
configureEnableBASMSync(enable) {
this.enableBASMSync = enable;
this.logger.log(chalk.blue(`BASM synchronization ${enable ? 'enabled' : 'disabled'}.`));
}
/**
* Configures the block header resolver used to derive BASM block hashes.
*/
configureTopicAnchorHeaderResolver(resolver) {
this.topicAnchorHeaderResolver = resolver;
this.logger.log(chalk.blue('BASM topic anchor header resolver has been configured.'));
}
/**
* Configures the go-chaintracks (Arcade) reorg SSE stream used to reconcile
* BASM anchors with blockchain reorganizations in real time.
* @param url - The reorg stream URL, e.g. `https://arcade.example/v2/reorg/stream`.
* @param scanDepth - Optional revalidation-sweep depth in blocks (default 3).
*/
configureReorgStream(url, scanDepth) {
this.reorgStreamUrl = url;
if (scanDepth !== undefined) {
this.reorgScanDepth = scanDepth;
}
this.logger.log(chalk.blue('BASM reorg stream has been configured.'));
}
/**
* Configures the opt-in unproven state eviction threshold.
*/
configureUnprovenEviction(config) {
if (config.thresholdBlocks !== undefined) {
this.unprovenEvictionBlocks = config.thresholdBlocks;
}
this.logger.log(chalk.blue('Unproven transaction eviction has been configured.'));
}
/**
* Configures periodic unproven maintenance. Each run first tries configured
* proof providers, then evicts rows that are still unproven past the threshold.
*/
configureUnprovenMaintenance(config) {
if (config.intervalMs !== undefined) {
this.unprovenMaintenanceIntervalMs = config.intervalMs;
}
if (config.thresholdBlocks !== undefined) {
this.unprovenEvictionBlocks = config.thresholdBlocks;
}
this.logger.log(chalk.blue('Unproven transaction maintenance has been configured.'));
}
/**
* Configures how often the BASM anchor chain is extended with empty anchors to
* follow the chain tip. Set to 0 to disable periodic polling.
*/
configureBASMBlockPollInterval(intervalMs) {
this.basmBlockPollIntervalMs = intervalMs;
this.logger.log(chalk.blue(`BASM block poll interval set to ${intervalMs}ms.`));
}
/**
* Enables or disables verbose request logging.
* @param enable - true to enable, false to disable
*/
configureVerboseRequestLogging(enable) {
this.verboseRequestLogging = enable;
this.logger.log(chalk.blue(`Verbose request logging ${enable ? 'enabled' : 'disabled'}.`));
}
/**
* Configure Knex (SQL) database connection.
* @param config - Knex configuration object, or a MySQL connection string loaded from configuration.
*/
async configureKnex(config) {
if (typeof config === 'string') {
config = {
client: 'mysql2',
connection: config
};
}
this.knex = Knex(config);
this.logger.log(chalk.blue('Knex successfully configured.'));
}
/**
* Configures the MongoDB database connection.
* Also initializes the BanService for persistent ban tracking.
* @param connectionString - MongoDB connection string
*/
async configureMongo(connectionString) {
const mongoClient = new MongoClient(connectionString);
await mongoClient.connect();
this.mongoClient = mongoClient;
const db = mongoClient.db(`${this.name}_lookup_services`);
this.mongoDb = db;
// Initialize the BanService
this.banService = new BanService(db);
await this.banService.ensureIndexes();
this.logger.log(chalk.blue('MongoDB successfully configured and connected.'));
}
/**
* Configures a Topic Manager.
* @param name - The name of the Topic Manager
* @param manager - An instance of TopicManager
*/
configureTopicManager(name, manager) {
this.managers[name] = manager;
this.logger.log(chalk.blue(`Configured topic manager ${name}`));
}
/**
* Configures a Lookup Service.
* @param name - The name of the Lookup Service
* @param service - An instance of LookupService
*/
configureLookupService(name, service) {
this.services[name] = service;
this.logger.log(chalk.blue(`Configured lookup service ${name}`));
}
/**
* Configures a Lookup Service using Knex (SQL) database.
* @param name - The name of the Lookup Service
* @param serviceFactory - A factory function that creates a LookupService instance using Knex
*/
configureLookupServiceWithKnex(name, serviceFactory) {
const knex = this.ensureKnex();
const factoryResult = serviceFactory(knex);
this.services[name] = factoryResult.service;
this.migrationsToRun.push(...factoryResult.migrations);
this.logger.log(chalk.blue(`Configured lookup service ${name} with Knex`));
}
/**
* Configures a Lookup Service using MongoDB.
* @param name - The name of the Lookup Service
* @param serviceFactory - A factory function that creates a LookupService instance using MongoDB
*/
configureLookupServiceWithMongo(name, serviceFactory) {
const mongoDb = this.ensureMongo();
this.services[name] = serviceFactory(mongoDb);
this.logger.log(chalk.blue(`Configured lookup service ${name} with MongoDB`));
}
/**
* Advanced configuration method for setting or overriding any
* Engine constructor parameters via an EngineConfig object.
*
* Example usage:
* configureEngineParams({
* logTime: true,
* throwOnBroadcastFailure: true,
* overlayBroadcastFacilitator: new MyCustomFacilitator()
* })
*
* These fields will be respected when we finally build/configure the Engine
* in the `configureEngine()` method below.
*/
configureEngineParams(params) {
this.engineConfig = {
...this.engineConfig,
...params
};
this.logger.log(chalk.blue('Advanced Engine configuration params have been updated.'));
}
/**
* Configures the Overlay Engine itself.
* By default, auto-configures SHIP and SLAP unless autoConfigureShipSlap = false
* Then it merges in any advanced engine config from `this.engineConfig`.
*
* When a BanService is available (from configureMongo), auto-configured SHIP
* and SLAP managers, discovery storage, and lookup services are wrapped so
* banned outputs are not admitted or indexed.
*
* @param autoConfigureShipSlap - Whether to auto-configure SHIP and SLAP services (default: true)
*/
async configureEngine(autoConfigureShipSlap = true) {
const knex = this.ensureKnex();
if (autoConfigureShipSlap) {
const mongoDb = this.ensureMongo();
const shipStorage = new DiscoveryServices.SHIPStorage(mongoDb);
const slapStorage = new DiscoveryServices.SLAPStorage(mongoDb);
// Run the one-time discovery migration before the engine can accept
// traffic, so a failed unique-index build is a visible startup failure.
await shipStorage.ensureIndexes();
await slapStorage.ensureIndexes();
this.configureTopicManager('tm_ship', new DiscoveryServices.SHIPTopicManager());
this.configureTopicManager('tm_slap', new DiscoveryServices.SLAPTopicManager());
const shipStorageForLookup = this.banService === undefined
? shipStorage
: new BanAwareSHIPStorage(shipStorage, this.banService, this.logger);
const slapStorageForLookup = this.banService === undefined
? slapStorage
: new BanAwareSLAPStorage(slapStorage, this.banService, this.logger);
this.services.ls_ship = new DiscoveryServices.SHIPLookupService(shipStorageForLookup);
this.services.ls_slap = new DiscoveryServices.SLAPLookupService(slapStorageForLookup);
this.logger.log(chalk.blue('Configured lookup service ls_ship with MongoDB'));
this.logger.log(chalk.blue('Configured lookup service ls_slap with MongoDB'));
}
this.wrapBanAwareServices();
const syncConfig = this.buildSyncConfig();
const storage = new KnexStorage(knex);
this.migrationsToRun = [...KnexStorageMigrations.default, ...this.migrationsToRun];
const broadcaster = this.buildBroadcaster();
const advertiser = await this.buildAdvertiser();
const EngineWithBASM = Engine;
this.engine = new EngineWithBASM(this.managers, this.services, storage, this.engineConfig.chainTracker ?? this.chainTracker, `https://${this.advertisableFQDN}`, this.network === 'test'
? (this.engineConfig.shipTrackers ?? DEFAULT_TESTNET_SLAP_TRACKERS)
: this.engineConfig.shipTrackers, this.resolveSlapTrackers(), broadcaster ?? this.engineConfig.broadcaster, advertiser, syncConfig, this.engineConfig.logTime ?? false, this.engineConfig.logPrefix ?? '[OVERLAY_ENGINE] ', this.engineConfig.throwOnBroadcastFailure ?? true, this.engineConfig.overlayBroadcastFacilitator ?? new HTTPSOverlayBroadcastFacilitator(), this.logger, this.engineConfig.suppressDefaultSyncAdvertisements ?? true, this.buildTopicAnchorHeaderResolver(), this.engineConfig.enableBASMSync ?? this.enableBASMSync, this.engineConfig.unprovenEvictionBlocks ?? this.unprovenEvictionBlocks);
this.initServerWallet();
this.logger.log(chalk.green('Engine has been configured.'));
}
/** Wrap SHIP/SLAP managers and services with ban-aware filters if BanService is configured. */
wrapBanAwareServices() {
if (this.banService === undefined)
return;
for (const key of ['tm_ship', 'tm_slap']) {
if (this.managers[key] !== undefined) {
const label = key === 'tm_ship' ? 'SHIP' : 'SLAP';
this.managers[key] = new BanAwareTopicManager(this.managers[key], this.banService, label, this.logger);
this.logger.log(chalk.blue(`${label} topic manager wrapped with ban-aware filter.`));
}
}
for (const key of ['ls_ship', 'ls_slap']) {
if (this.services[key] !== undefined) {
const label = key === 'ls_ship' ? 'SHIP' : 'SLAP';
this.services[key] = new BanAwareLookupWrapper(this.services[key], this.banService, label, this.logger);
this.logger.log(chalk.blue(`${label} lookup service wrapped with ban-aware filter.`));
}
}
}
/** Build the sync config based on enableGASPSync and engineConfig. */
buildSyncConfig() {
if (this.enableGASPSync) {
return this.engineConfig.syncConfiguration ?? {};
}
const syncConfig = {};
for (const name of Object.keys(this.managers)) {
syncConfig[name] = false;
}
return syncConfig;
}
/** Build the configured transaction propagation provider chain. */
buildBroadcaster() {
const providers = [];
const callbackUrl = `https://${this.advertisableFQDN}/arc-ingest`;
if (typeof this.arcadeUrl === 'string' && this.arcadeUrl.length > 0) {
this.arcadeProvider = new ArcadeProvider(this.arcadeUrl, {
apiKey: this.arcadeApiKey,
callbackUrl,
callbackToken: this.arcCallbackToken,
deploymentId: this.arcadeDeploymentId
});
providers.push({
name: 'Arcade',
broadcaster: this.arcadeProvider
});
}
else {
this.arcadeProvider = undefined;
}
if (typeof this.arcApiKey === 'string' && this.arcApiKey.length > 0) {
const arcUrl = this.network === 'test' ? 'https://arc-test.taal.com' : 'https://arc.taal.com';
providers.push({
name: 'ARC',
broadcaster: new ARC(arcUrl, {
apiKey: this.arcApiKey,
callbackUrl,
callbackToken: this.arcCallbackToken
})
});
}
if (providers.length === 0)
return undefined;
if (providers.length === 1)
return providers[0].broadcaster;
return new ProviderChainBroadcaster(providers);
}
ensureArcadeProvider() {
if (this.arcadeProvider !== undefined)
return this.arcadeProvider;
if (typeof this.arcadeUrl !== 'string' || this.arcadeUrl.length === 0)
return undefined;
this.arcadeProvider = new ArcadeProvider(this.arcadeUrl, {
apiKey: this.arcadeApiKey,
callbackUrl: `https://${this.advertisableFQDN}/arc-ingest`,
callbackToken: this.arcCallbackToken,
deploymentId: this.arcadeDeploymentId
});
return this.arcadeProvider;
}
async fetchArcadeProof(txid) {
const provider = this.ensureArcadeProvider();
if (provider === undefined)
return undefined;
const proof = await provider.fetchMerkleProof(txid);
if (proof === undefined)
return undefined;
const chainTracker = this.engineConfig.chainTracker ?? this.chainTracker;
if (chainTracker === 'scripts only') {
throw new Error('Cannot validate Arcade proof with scripts-only chain tracker');
}
const blockHeight = proof.blockHeight ?? proof.merklePath.blockHeight;
if (blockHeight === undefined) {
throw new Error(`Arcade proof for ${txid} did not include a block height`);
}
const valid = await chainTracker.isValidRootForHeight(proof.merkleRoot, blockHeight);
if (!valid) {
throw new Error(`Arcade proof for ${txid} did not match the chain tracker at height ${blockHeight}`);
}
return {
...proof,
blockHeight
};
}
async fetchConfiguredMerkleProof(txid) {
const proof = await this.fetchArcadeProof(txid);
if (proof === undefined)
return undefined;
return {
merklePath: proof.merklePath,
blockHeight: proof.blockHeight
};
}
/** Build the BASM block header resolver. */
buildTopicAnchorHeaderResolver() {
const configured = this.engineConfig.topicAnchorHeaderResolver ?? this.topicAnchorHeaderResolver;
if (configured !== undefined) {
return configured;
}
return async (blockHeight) => {
const response = await fetch(`https://api.whatsonchain.com/v1/bsv/${this.network}/block/${blockHeight}/header`, {
method: 'GET',
headers: { Accept: 'application/json' }
});
if (!response.ok) {
throw new Error(`WhatsOnChain header lookup failed for height ${blockHeight}: ${response.status}`);
}
const header = await response.json();
if (typeof header.hash !== 'string') {
throw new TypeError(`WhatsOnChain did not return a block hash for height ${blockHeight}`);
}
return {
blockHeight,
blockHash: header.hash,
merkleRoot: header.merkleroot
};
};
}
/** Resolve the SLAP trackers from config or network defaults. */
resolveSlapTrackers() {
if (Array.isArray(this.engineConfig.slapTrackers))
return this.engineConfig.slapTrackers;
return this.network === 'test' ? DEFAULT_TESTNET_SLAP_TRACKERS : DEFAULT_SLAP_TRACKERS;
}
/** Build the WalletAdvertiser (or use user-provided one). */
async buildAdvertiser() {
if (this.engineConfig.advertiser !== undefined)
return this.engineConfig.advertiser;
const storageBase = this.network === 'test'
? 'https://staging-storage.babbage.systems'
: 'https://storage.babbage.systems';
try {
return new DiscoveryServices.WalletAdvertiser(this.network, this.privateKey, storageBase, `https://${this.advertisableFQDN}`);
}
catch (e) {
this.logger.log(`Advertiser not initialized for FQDN ${this.advertisableFQDN} - SHIP and SLAP will be disabled. Reason: ${e}`);
return undefined;
}
}
/** Initialize the server wallet for BSV mutual authentication. */
initServerWallet() {
try {
const keyDeriver = new KeyDeriver(new PrivateKey(this.privateKey, 'hex'));
const storageManager = new WalletStorageManager(keyDeriver.identityKey);
const signer = new WalletSigner(this.network, keyDeriver, storageManager);
const services = new Services(this.network);
this.serverWallet = new Wallet(signer, services);
this.adminIdentityKey ??= keyDeriver.identityKey;
this.logger.log(chalk.blue('Server wallet initialized for BSV mutual authentication.'));
}
catch (e) {
this.logger.log(chalk.yellow(`Server wallet could not be initialized. BSV auth will not be available. Reason: ${e}`));
}
}
/**
* Ensures that Knex is configured and returns it.
* @throws Error if Knex is not configured
*/
ensureKnex() {
if (this.knex === undefined) {
throw new TypeError('You must configure your SQL database with the .configureKnex() method first!');
}
return this.knex;
}
/**
* Ensures that MongoDB is configured and returns it.
* @throws Error if MongoDB is not configured
*/
ensureMongo() {
if (this.mongoDb === undefined) {
throw new TypeError('You must configure your MongoDB connection with the .configureMongo() method first!');
}
return this.mongoDb;
}
/**
* Ensures that the Overlay Engine is configured and returns it.
* @throws Error if the Engine is not configured
*/
ensureEngine() {
if (this.engine === undefined) {
throw new TypeError('You must configure your Overlay Services engine with the .configureEngine() method first!');
}
return this.engine;
}
/**
* Creates a JanitorService instance with current configuration.
*/
createJanitor() {
const mongoDb = this.ensureMongo();
return new JanitorService({
mongoDb,
logger: this.logger,
requestTimeoutMs: this.janitorConfig.requestTimeoutMs,
hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore,
banService: this.banService,
autoBanOnRemoval: this.janitorConfig.autoBanOnRemoval
});
}
/** Ban a domain and remove all its SHIP/SLAP records from MongoDB. */
async handleBanDomain(res, value, reason) {
await this.banService.banDomain(value, reason);
const db = this.ensureMongo();
const [shipDeleted, slapDeleted] = await Promise.all([
db.collection('shipRecords').deleteMany({ domain: value }),
db.collection('slapRecords').deleteMany({ domain: value })
]);
return res.status(200).json({
status: 'success',
message: `Domain "${value}" banned. Removed ${shipDeleted.deletedCount} SHIP and ${slapDeleted.deletedCount} SLAP records.`
});
}
/** Parse outpoint string, ban it, and evict it from all lookup services. */
async handleBanOutpoint(res, engine, value, reason) {
const dotIndex = value.lastIndexOf('.');
if (dotIndex === -1) {
return res.status(400).json({ status: 'error', message: 'Outpoint format must be "txid.outputIndex"' });
}
const txid = value.substring(0, dotIndex);
const outputIndex = Number.parseInt(value.substring(dotIndex + 1));
if (Number.isNaN(outputIndex)) {
return res.status(400).json({ status: 'error', message: 'Invalid outputIndex in outpoint' });
}
await this.banService.banOutpoint(txid, outputIndex, reason);
await this.evictFromServices(engine, txid, outputIndex);
return res.status(200).json({
status: 'success',
message: `Outpoint "${value}" banned and evicted from lookup services.`
});
}
/** Evict an output from a specific service or all services (silent per-service errors). */
async evictFromServices(engine, txid, outputIndex, service) {
if (typeof service === 'string') {
const svc = engine.lookupServices[service];
if (svc !== undefined)
await svc.outputEvicted(txid, outputIndex);
return;
}
for (const svc of Object.values(engine.lookupServices)) {
try {
await svc.outputEvicted(txid, outputIndex);
}
catch { /* best-effort */ }
}
}
/** Look up the domain of an outpoint from SHIP or SLAP records. */
async lookupDomainForOutpoint(txid, outputIndex) {
const db = this.ensureMongo();
const [shipRecord, slapRecord] = await Promise.all([
db.collection('shipRecords').findOne({ txid, outputIndex }),
db.collection('slapRecords').findOne({ txid, outputIndex })
]);
return (shipRecord?.domain ?? slapRecord?.domain);
}
/** Ban a domain and delete all SHIP/SLAP records for it. */
async banDomainAndRemoveRecords(domain, reason) {
await this.banService.banDomain(domain, reason);
const db = this.ensureMongo();
await Promise.all([
db.collection('shipRecords').deleteMany({ domain }),
db.collection('slapRecords').deleteMany({ domain })
]);
}
async runHealthCheck(definition) {
const startedAt = Date.now();
try {
const result = ((await Promise.race([
Promise.resolve(definition.handler()),
new Promise((_, reject) => {
setTimeout(() => reject(new Error(`Timed out after ${this.healthConfig.timeoutMs}ms`)), this.healthConfig.timeoutMs);
})
])) ?? {});
return {
name: definition.name,
scope: definition.scope,
critical: definition.critical,
status: result.status ?? 'ok',
message: result.message,
details: result.details,
durationMs: Date.now() - startedAt
};
}
catch (error) {
return {
name: definition.name,
scope: definition.scope,
critical: definition.critical,
status: 'error',
message: error instanceof Error ? error.message : 'Unknown health-check error',
durationMs: Date.now() - startedAt
};
}
}
async collectHealthReport(mode) {
const definitions = [
{
name: 'process',
scope: 'live',
critical: true,
handler: async () => ({
status: 'ok',
details: {
listening: this.isListening
}
})
},
{
name: 'engine',
scope: 'ready',
critical: true,
handler: async () => {
if (this.engine === undefined) {
throw new TypeError('Overlay engine is not configured');
}
return {
status: 'ok',
details: {
topicManagers: Object.keys(this.managers),
lookupServices: Object.keys(this.services)
}
};
}
},
{
name: 'knex',
scope: 'ready',
critical: true,
handler: async () => {
if (this.knex === undefined) {
throw new TypeError('Knex is not configured');
}
await this.knex.raw('select 1 as ok');
return {
status: 'ok',
details: {
client: this.knex.client?.config?.client ?? 'unknown'
}
};
}
},
{
name: 'mongo',
scope: 'ready',
critical: true,
handler: async () => {
if (this.mongoDb === undefined) {
throw new TypeError('MongoDB is not configured');
}
await this.mongoDb.command({ ping: 1 });
return {
status: 'ok',
details: {
database: this.mongoDb.databaseName
}
};
}
}
];
for (const check of this.healthChecks) {
definitions.push({
name: check.name,
scope: check.scope ?? 'ready',
critical: check.critical ?? false,
handler: check.handler
});
}
const filteredDefinitions = definitions.filter((definition) => {
if (mode === 'full') {
return true;
}
return definition.scope === mode;
});
const checks = await Promise.all(filteredDefinitions.map(async (definition) => await this.runHealthCheck(definition)));
const liveChecks = checks.filter(check => check.scope === 'live');
const readyChecks = checks.filter(check => check.scope === 'ready');
const live = liveChecks.every(check => !check.critical || check.status === 'ok');
const ready = readyChecks.every(check => !check.critical || check.status === 'ok');
let status = 'ok';
if (!live || !ready || checks.some(check => check.critical && check.status === 'error')) {
status = 'error';
}
else if (checks.some(check => check.status !== 'ok')) {
status = 'degraded';
}
const context = typeof this.healthConfig.contextProvider === 'function'
? await this.healthConfig.contextProvider()
: undefined;
const report = {
status,
live,
ready,
service: {
name: this.name,
advertisableFQDN: this.advertisableFQDN,
port: this.port,
network: this.network,
startedAt: this.startTime?.toISOString(),
uptimeMs: this.startTime === undefined ? 0 : Date.now() - this.startTime.getTime(),
topicManagerCount: Object.keys(this.managers).length,
lookupServiceCount: Object.keys(this.services).length
},
checks: this.healthConfig.includeDetails
? checks
: checks.map(({ details, ...check }) => check),
context
};
return report;
}
/**
* Renders a request or response body for verbose logging, truncating overly long payloads.
*/
formatBodyForLog(body, tooLongLabel, okPrefix) {
let bodyString;
if (typeof body === 'object') {
bodyString = JSON.stringify(body, null, 2);
}
else if (Buffer.isBuffer(body)) {
bodyString = body.toString('utf8');
}
else {
bodyString = String(body);
}
return bodyString.length > 280
? chalk.yellow(`(${tooLongLabel}, length: ${String(bodyString.length)} characters)`)
: chalk.green(`${okPrefix}\n${bodyString}`);
}
/**
* Installs middleware that verbosely logs incoming requests and outgoing responses.
*/
setupVerboseRequestLogging() {
this.app.use((req, res, next) => {
const startTime = Date.now();
// Log incoming request details
this.logger.log(chalk.magenta.bold(`Incoming Request: ${String(req.method)} ${String(req.originalUrl)}`));
// Pretty-print headers
this.logger.log(chalk.cyan('Headers:'));
this.logger.log(util.inspect(req.headers, { colors: true, depth: null }));
// Handle request body
if (req.body != null && Object.keys(req.body).length > 0) {
this.logger.log(this.formatBodyForLog(req.body, 'Body too long to display', 'Request Body:'));
}
// Intercept the res.send method to log responses
const originalSend = res.send;
let responseBody;
res.send = function (body) {
responseBody = body;
return originalSend.call(this, body);
};
// Log outgoing response details after the response is finished
res.on('finish', () => {
const duration = Date.now() - startTime;
this.logger.log(chalk.magenta.bold(`Outgoing Response: ${String(req.method)} ${String(req.originalUrl)} - Status: ${String(res.statusCode)} - Duration: ${String(duration)}ms`));
this.logger.log(chalk.cyan('Response Headers:'));
this.logger.log(util.inspect(res.getHeaders(), { colors: true, depth: null }));
// Handle response body
if (responseBody != null) {
this.logger.log(this.formatBodyForLog(responseBody, 'Response body too long to display', 'Response Body:'));
}
});
next();
});
}
/**
* Starts the Express server.
* Sets up routes and begins listening on the configured port.
*/
async start() {
const engine = this.ensureEngine();
const knex = this.ensureKnex();
this.startTime = new Date();
this.app.use(bodyParser.json({ limit: '1gb', type: 'application/json' }));
this.app.use(bodyParser.raw({ limit: '1gb', type: 'application/octet-stream' }));
if (this.verboseRequestLogging) {
this.setupVerboseRequestLogging();
}
// Enable CORS
this.app.use((req, res, next) => {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Headers', '*');
res.header('Access-Control-Allow-Methods', '*');
res.header('Access-Control-Expose-Headers', '*');
res.header('Access-Control-Allow-Private-Network', 'true');
if (req.method === 'OPTIONS') {
res.sendStatus(200);
}
else {
next();
}
});
// Serve a static documentation site or user interface
this.app.get('/', (req, res) => {
res.set('content-type', 'text/html');
res.send(makeUserInterface({
...this.webUIConfig,
adminIdentityKey: this.adminIdentityKey
}));
});
// Serve health check endpoints
this.app.get('/health/live', (_, res) => {
;
(async () => {
const report = await this.collectHealthReport('live');
return res.status(report.live ? 200 : 503).json(report);
})().catch((error) => {
res.status(500).json({
status: 'error',
message: error instanceof Error ? error.message : 'Unexpected error'
});
});
});
this.app.get('/health/ready', (_, res) => {
;
(async () => {
const report = await this.collectHealthReport('ready');
return res.status(report.ready ? 200 : 503).json(report);
})().catch((error) => {
res.status(500).json({
status: 'error',
message: error instanceof Error ? error.message : 'Unexpected error'
});
});
});
this.app.get('/health', (_, res) => {
;
(async () => {
const report = await this.collectHealthReport('full');
return res.status(report.ready ? 200 : 503).json(report);
})().catch((error) => {
res.status(500).json({
status: 'error',
message: error instanceof Error ? error.message : 'Unexpected error'
});
});
});
// List hosted topic managers and lookup services
this.app.get('/listTopicManagers', (_, res) => {
;
(async () => {
try {
const result = await engine.listTopicManagers();
return res.status(200).json(result);
}
catch (error) {
return res.status(400).json({
status: 'error',
message: error instanceof Error ? error.message : 'An unknown error occurred'
});
}
})().catch(() => {
res.status(500).json({
status: 'error',
message: 'Unexpected error'
});
});
});
this.app.get('/listLookupServiceProviders', (_, res) => {
;
(async () => {
try {
const result = await engine.listLookupServiceProviders();
return res.status(200).json(result);
}
catch (error) {
return res.status(400).json({
status: 'error',
message: error instanceof Error ? error.message : 'An unknown error occurred'
});
}
})().catch(() => {
res.status(500).json({
status: 'error',
message: 'Unexpected error'
});
});
});
// Host documentation for the services
this.app.get('/getDocumentationForTopicManager', (req, res) => {
;
(async () => {
try {
const manager = req.query.manager;
const result = await engine.getDocumentationForTopicManager(manager);
res.setHeader('Content-Type', 'text/markdown');
return res.status(200).send(result);
}
catch (error) {
return res.status(400).json({
status: 'error',
message: error instanceof Error ? error.message : 'An unknown error occurred'
});
}
})().catch(() => {
res.status(500).json({
status: 'error',
message: 'Unexpected error'
});
});
});
this.app.get('/getDocumentationForLookupServiceProvider', (req, res) => {
;
(async () => {
try {
const lookupService = req.query.lookupService;
const result = await engin