bitcore-node
Version:
A blockchain indexing node with extended capabilities using bitcore
331 lines • 16.7 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Pruning = exports.PruningService = void 0;
const stream_1 = require("stream");
const logger_1 = __importDefault(require("../logger"));
const coin_1 = require("../models/coin");
const transaction_1 = require("../models/transaction");
const rpc_1 = require("../rpc");
const parseArgv_1 = __importDefault(require("../utils/parseArgv"));
require("../utils/polyfills");
const config_1 = require("./config");
const { PRUNING_CHAIN, PRUNING_NETWORK, PRUNING_MEMPOOL_AGE, PRUNING_OLD_INTERVAL_HRS, PRUNING_INV_INTERVAL_MINS, PRUNING_DESCENDANT_LIMIT } = process.env;
const args = (0, parseArgv_1.default)([], [
{ arg: 'CHAIN', type: 'string' },
{ arg: 'NETWORK', type: 'string' },
{ arg: 'OLD', type: 'bool' },
{ arg: 'INVALID', type: 'bool' },
{ arg: 'EXIT', type: 'bool' },
{ arg: 'DRY', type: 'bool' },
{ arg: 'MEMPOOL_AGE', type: 'int' },
{ arg: 'OLD_INTERVAL_HRS', type: 'float' },
{ arg: 'INV_INTERVAL_MINS', type: 'float' },
{ arg: 'INV_MATURE_LEN', type: 'int' },
{ arg: 'DESCENDANT_LIMIT', type: 'int' },
{ arg: 'VERBOSE', type: 'bool' }
]);
const ONE_MIN = 1000 * 60;
const ONE_HOUR = 60 * ONE_MIN;
const ONE_DAY = 24 * ONE_HOUR;
const CHAIN = args.CHAIN || PRUNING_CHAIN;
const NETWORK = args.NETWORK || PRUNING_NETWORK;
const OLD_INTERVAL_HRS = args.OLD_INTERVAL_HRS || Number(PRUNING_OLD_INTERVAL_HRS) || 12;
const INV_INTERVAL_MINS = args.INV_INTERVAL_MINS || Number(PRUNING_INV_INTERVAL_MINS) || 10;
const INV_MATURE_LEN = args.INV_MATURE_LEN || 3; // using || means INV_MATURE_LEN needs to be >0
const MEMPOOL_AGE = args.MEMPOOL_AGE || Number(PRUNING_MEMPOOL_AGE) || 7;
const DESCENDANT_LIMIT = args.DESCENDANT_LIMIT || Number(PRUNING_DESCENDANT_LIMIT) || 10;
const VERBOSE = Boolean(args.VERBOSE ?? false);
// If --DRY was given w/o a follow arg (i.e. 'true', '0', etc) assume the user wants to run a dry run (safe)
if (Object.keys(args).includes('DRY') && args.DRY === undefined) {
args.DRY = '1';
}
if (OLD_INTERVAL_HRS > 72) {
throw new Error('OLD_INTERVAL_HRS cannot be over 72 hours. Consider using a cron job.');
}
if (INV_INTERVAL_MINS > 60 * 24 * 3) {
throw new Error('INV_INTERVAL_MINS cannot be over 72 hours. Consider using a cron job.');
}
else if (INV_INTERVAL_MINS < 2) {
throw new Error('INV_INTERVAL_MINS must be at least 2 minutes');
}
class PruningService {
constructor({ transactionModel = transaction_1.TransactionStorage, coinModel = coin_1.CoinStorage } = {}) {
this.stopping = false;
this.rpcs = [];
this.runningOld = false;
this.runningInvalid = false;
this.lastRunTimeOld = 0;
this.lastRunTimeInvalid = 0;
this.transactionModel = transactionModel;
this.coinModel = coinModel;
}
async start() {
logger_1.default.info('Starting Pruning Service');
args.OLD && logger_1.default.info(`Pruning mempool txs older than ${MEMPOOL_AGE} day(s)`);
args.INVALID && logger_1.default.info('Pruning conflicting mempool txs');
args.DRY && logger_1.default.info('Pruning service DRY RUN');
this.registerRpcs();
if (args.EXIT) {
this.detectAndClear().then(() => {
process.emit('SIGINT', 'SIGINT');
});
}
else {
logger_1.default.info('Pruning service OLD interval (hours): ' + OLD_INTERVAL_HRS);
logger_1.default.info('Pruning service INVALID interval (minutes): ' + INV_INTERVAL_MINS);
this.interval = setInterval(this.detectAndClear.bind(this), ONE_MIN);
}
}
async stop() {
logger_1.default.info('Stopping Pruning Service');
this.stopping = true;
clearInterval(this.interval);
}
registerRpcs() {
const chainNetworks = CHAIN ? [{ chain: CHAIN, network: NETWORK }] : config_1.Config.chainNetworks();
for (const chainNetwork of chainNetworks) {
const config = config_1.Config.chainConfig(chainNetwork);
if (!config.rpc) {
continue;
}
this.rpcs[`${chainNetwork.chain}:${chainNetwork.network}`] = new rpc_1.RPC(config.rpc.username, config.rpc.password, config.rpc.host, config.rpc.port);
}
}
async detectAndClear() {
try {
if (CHAIN && NETWORK) {
args.OLD && await this.processOldMempoolTxs(CHAIN, NETWORK, MEMPOOL_AGE);
args.INVALID && await this.processAllInvalidTxs(CHAIN, NETWORK);
}
else {
for (let chainNetwork of config_1.Config.chainNetworks()) {
const { chain, network } = chainNetwork;
if (!chain || !network) {
throw new Error('Config structure should contain both a chain and network');
}
args.OLD && await this.processOldMempoolTxs(chain, network, MEMPOOL_AGE);
args.INVALID && await this.processAllInvalidTxs(chain, network);
}
}
}
catch (err) {
logger_1.default.error('Pruning Error: ' + err.stack || err.message || err);
}
}
async processOldMempoolTxs(chain, network, days) {
if (this.runningOld) {
return;
}
this.runningOld = true;
try {
if (Date.now() - this.lastRunTimeOld < OLD_INTERVAL_HRS * ONE_HOUR) {
return;
}
logger_1.default.info('========== OLD STARTED ===========');
const oldTime = new Date(Date.now() - days * ONE_DAY);
const count = await this.transactionModel.collection.countDocuments({
chain,
network,
blockHeight: -1 /* SpentHeightIndicators.pending */,
blockTimeNormalized: { $lt: oldTime }
});
logger_1.default.info(`Found ${count} outdated ${chain}:${network} mempool txs`);
let rmCount = 0;
await new Promise((resolve, reject) => {
this.transactionModel.collection
.find({ chain, network, blockHeight: -1 /* SpentHeightIndicators.pending */, blockTimeNormalized: { $lt: oldTime } })
.sort(count > 5000 ? { chain: 1, network: 1, blockTimeNormalized: 1 } : {})
.pipe(new stream_1.Transform({
objectMode: true,
transform: async (data, _, cb) => {
if (this.stopping) {
return cb(new Error('Stopping'));
}
const tx = data;
try {
const nodeTx = await this.rpcs[`${chain}:${network}`].getTransaction(tx.txid);
if (nodeTx) {
logger_1.default.warn(`Tx ${tx.txid} is still in the mempool${VERBOSE ? ': %o' : ''}`, nodeTx);
return cb();
}
}
catch (err) {
if (err.code !== -5) { // -5: No such mempool or blockchain transaction. Use gettransaction for wallet transactions.
logger_1.default.error(`Error checking tx ${tx.txid} in the mempool: ${err.message}`);
return cb();
}
}
logger_1.default.info(`Finding ${tx.txid} outputs and dependent outputs`);
const outputGenerator = this.transactionModel.yieldRelatedOutputs(tx.txid);
let spentTxids = new Set();
for await (const coin of outputGenerator) {
if (coin.mintHeight >= 0 || coin.spentHeight >= 0) {
logger_1.default.error(`Cannot prune coin! ${coin.mintTxid}`);
return cb();
}
if (coin.spentTxid) {
spentTxids.add(coin.spentTxid);
if (spentTxids.size > DESCENDANT_LIMIT) {
logger_1.default.warn(`${tx.txid} has too many decendants`);
return cb();
}
}
}
spentTxids.add(tx.txid);
rmCount += spentTxids.size;
const uniqueTxids = Array.from(spentTxids);
await this.removeOldMempool(chain, network, uniqueTxids);
logger_1.default.info(`Removed tx ${tx.txid} and ${spentTxids.size - 1} dependent txs`);
return cb();
}
}))
.on('finish', resolve)
.on('error', reject);
});
logger_1.default.info(`Removed all pending ${chain}:${network} txs older than ${days} days: ${rmCount}`);
this.lastRunTimeOld = Date.now();
logger_1.default.info('========== OLD FINISHED ===========');
}
catch (err) {
logger_1.default.error(`Error processing old mempool txs: ${err.stack || err.message || err}`);
}
finally {
this.runningOld = false;
}
}
async processAllInvalidTxs(chain, network) {
if (this.runningInvalid) {
return;
}
this.runningInvalid = true;
try {
if (Date.now() - this.lastRunTimeInvalid < INV_INTERVAL_MINS * ONE_MIN) {
return;
}
logger_1.default.info('========== INVALID STARTED ===========');
const count = await this.coinModel.collection.countDocuments({ chain, network, mintHeight: -1 /* SpentHeightIndicators.pending */ });
logger_1.default.info(`Found ${count} pending ${chain}:${network} TXOs`);
// Note, realCount <= count since the coinStream returns coins dynamically and related coins are updated.
// The caveat is that new coins could be added to the stream while we are iterating over it.
let realCount = 0;
let invalidCount = 0;
const seen = new Set();
const voutStream = this.coinModel.collection.find({ chain, network, mintHeight: -1 /* SpentHeightIndicators.pending */ });
for await (const vout of voutStream) {
if (this.stopping) {
break;
}
realCount++;
if (seen.has(vout.mintTxid)) {
continue;
}
seen.add(vout.mintTxid);
const tx = await this.transactionModel.collection.findOne({ chain, network, txid: vout.mintTxid });
if (!tx) {
logger_1.default.error(`Coin ${vout.mintTxid} has no corresponding tx`);
continue;
}
if (tx.replacedByTxid) {
if (await this.invalidateTx(chain, network, tx)) {
invalidCount++;
}
}
else {
// Check if the parent tx was replaced since the sync process marks immediate replacements as replaced, but not descendants
const vins = await this.coinModel.collection.find({ chain, network, spentTxid: vout.mintTxid }).toArray();
const vinTxs = await this.transactionModel.collection.find({ chain, network, txid: { $in: vins.map(vin => vin.mintTxid) } }).toArray();
for (const tx of vinTxs) {
if (tx.replacedByTxid) {
if (await this.invalidateTx(chain, network, tx)) {
invalidCount++;
}
;
}
}
}
}
logger_1.default.info(`Invalidated ${invalidCount} (processed ${realCount}) pending TXOs for ${chain}:${network}`);
this.lastRunTimeInvalid = Date.now();
logger_1.default.info('========== INVALID FINISHED ===========');
}
catch (err) {
logger_1.default.error(`Error processing invalid txs: ${err.stack || err.message || err}`);
}
finally {
this.runningInvalid = false;
}
}
/**
* Invalidate a transaction and its descendants
* @param {string} chain
* @param {string} network
* @param {ITransaction} tx Transaction object with replacedByTxid
* @returns
*/
async invalidateTx(chain, network, tx) {
if (tx.blockHeight >= 0) {
// This means that downstream coins are still pending when they should be marked as confirmed.
// This indicates a bug in the sync process.
logger_1.default.warn(`Tx ${tx.txid} is already mined`);
return false;
}
if (!tx.replacedByTxid) {
logger_1.default.warn(`Given tx has no replacedByTxid: ${tx.txid}`);
return false;
}
let rTx = await this.transactionModel.collection.findOne({ chain, network, txid: tx.replacedByTxid });
let txids = [tx.txid];
while (rTx?.replacedByTxid && rTx?.blockHeight < 0 && !txids.includes(rTx?.txid)) {
// replacement tx has also been replaced
// Note: rTx.txid === tx.txid may happen if tx.replacedByTxid => rTx.txid and rTx.replacedByTxid => tx.txid.
// This might happen if tx was rebroadcast _after_ being marked as replaced by rTx, thus marking rTx as replaced by tx.
// Without this check, we could end up in an infinite loop where the two txs keep finding each other as unconfirmed replacements.
txids.push(rTx.txid);
rTx = await this.transactionModel.collection.findOne({ chain, network, txid: rTx.replacedByTxid });
}
// Re-org protection
const tipHeight = await this.rpcs[`${chain}:${network}`].getBlockHeight();
const isMature = rTx?.blockHeight > -1 /* SpentHeightIndicators.pending */ && tipHeight - rTx?.blockHeight > INV_MATURE_LEN;
const isExpired = rTx?.blockHeight === -5 /* SpentHeightIndicators.expired */; // Set by --OLD
if (isMature || isExpired) {
try {
const nConfs = tipHeight - rTx?.blockHeight;
logger_1.default.info(`${args.DRY ? 'DRY RUN - ' : ''}Invalidating ${tx.txid} with replacement => ${tx.replacedByTxid} (${isExpired ? 'expired' : nConfs})`);
if (args.DRY) {
return true;
}
await this.transactionModel._invalidateTx({
chain,
network,
invalidTxid: tx.txid,
replacedByTxid: tx.replacedByTxid
});
return true;
}
catch (err) {
logger_1.default.error(`Error invalidating tx ${tx.txid}: ${err.stack || err.message || err}`);
}
}
else {
logger_1.default.info(`Skipping invalidation of ${tx.txid} with immature replacement => ${tx.replacedByTxid}`);
}
return false;
}
async removeOldMempool(chain, network, txids) {
logger_1.default.info(`${args.DRY ? 'DRY RUN - ' : ''}Removing ${txids.length} txids`);
if (args.DRY) {
return;
}
return Promise.all([
this.transactionModel.collection.updateMany({ chain, network, txid: { $in: txids }, blockHeight: -1 /* SpentHeightIndicators.pending */ }, { $set: { blockHeight: -5 /* SpentHeightIndicators.expired */ } }),
this.coinModel.collection.updateMany({ chain, network, mintTxid: { $in: txids }, mintHeight: -1 /* SpentHeightIndicators.pending */ }, { $set: { mintHeight: -5 /* SpentHeightIndicators.expired */ } }),
this.coinModel.collection.updateMany({ chain, network, spentTxid: { $in: txids }, spentHeight: -1 /* SpentHeightIndicators.pending */ }, { $set: { spentTxid: null, spentHeight: -2 /* SpentHeightIndicators.unspent */ } })
]);
}
}
exports.PruningService = PruningService;
exports.Pruning = new PruningService();
//# sourceMappingURL=pruning.js.map