bitcore-node
Version:
A blockchain indexing node with extended capabilities using bitcore
278 lines • 12.3 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MultiThreadSync = void 0;
const crypto_rpc_1 = require("crypto-rpc");
const events_1 = require("events");
const os = __importStar(require("os"));
const worker_threads_1 = require("worker_threads");
const __1 = require("../../");
const config_1 = __importDefault(require("../../../../config"));
const logger_1 = __importStar(require("../../../../logger"));
const state_1 = require("../../../../models/state");
const utils_1 = require("../../../../utils");
const block_1 = require("../models/block");
class MultiThreadSync extends events_1.EventEmitter {
constructor({ chain, network }) {
super();
this.threads = [];
this.syncingThreads = 0;
this.bestBlock = 0;
this.syncHeight = 0;
this.stopping = false;
this.syncQueue = [];
this.syncing = false;
this.resolvingGaps = false;
this.gapsLength = 0;
this.currentHeight = 0;
this.chain = chain || 'ETH';
this.network = network || 'mainnet';
this.config = config_1.default.chains[chain][network];
this.mtSyncTipPad = this.config.mtSyncTipPad || 100;
}
async addBlockToQueue(blockNum) {
this.syncQueue.push(blockNum);
}
getRpc() {
const providerIdx = worker_threads_1.threadId % (this.config.providers || []).length;
const providerConfig = this.config.provider || this.config.providers[providerIdx];
const rpcConfig = { ...providerConfig, chain: this.chain, currencyConfig: {} };
const rpc = new crypto_rpc_1.CryptoRpc(rpcConfig, {}).get(this.chain);
return rpc;
}
async sync() {
if (this.syncing) {
return false;
}
const { chain, network } = this;
const { parentChain, forkHeight = 0 } = this.config;
this.syncing = true;
try {
let tip = await __1.ChainStateProvider.getLocalTip({ chain, network });
if (parentChain && (!tip || tip.height < forkHeight)) {
let parentTip = await __1.ChainStateProvider.getLocalTip({ chain: parentChain, network });
while (!parentTip || parentTip.height < forkHeight) {
logger_1.default.info(`Waiting until ${parentChain} syncs before ${chain} ${network}`);
await new Promise(resolve => {
setTimeout(resolve, 5000);
});
parentTip = await __1.ChainStateProvider.getLocalTip({ chain: parentChain, network });
}
}
let startHeight = tip ? tip.height : this.config.syncStartHeight || 0;
const rpc = this.getRpc();
this.bestBlock = await rpc.web3.eth.getBlockNumber();
this.currentHeight = tip ? tip.height : this.config.syncStartHeight || 0;
this.syncHeight = this.currentHeight;
startHeight = this.currentHeight;
logger_1.default.info(`Syncing ${this.bestBlock - this.currentHeight} blocks for ${chain} ${network}`);
await this.initializeThreads();
const startTime = Date.now();
const oneSecond = 1000;
this.syncInterval = setInterval(() => {
if (this.resolvingGaps) {
logger_1.default.info(`${(0, logger_1.timestamp)()} | Filling gaps... | Chain: ${chain} | Network: ${network} | On gap ${this.gapsLength -
this.syncQueue.length} of ${this.gapsLength} | Height: ${this.syncQueue[0] ? this.syncQueue[0].toString().padStart(7) : this.syncHeight}`);
}
else {
const blocksProcessed = this.currentHeight - startHeight;
const elapsedMinutes = (Date.now() - startTime) / (60 * oneSecond);
logger_1.default.info(`${(0, logger_1.timestamp)()} | Syncing... | Chain: ${chain} | Network: ${network} | ${(blocksProcessed / elapsedMinutes)
.toFixed(2)
.padStart(8)} blocks/min | Height: ${this.currentHeight.toString().padStart(7)}`);
}
}, oneSecond);
this.syncingThreads = this.threads.length;
// Kick off threads syncing
for (let i = 0; i < this.threads.length; i++) {
if (this.syncQueue.length > 0) {
this.threads[i].postMessage({ blockNum: this.syncQueue.shift() });
}
else {
this.threads[i].postMessage({ blockNum: this.currentHeight++ });
}
}
}
catch (err) {
logger_1.default.error(`Error syncing ${chain} ${network} :: ${err.message}`);
await (0, utils_1.wait)(2000);
this.syncing = false;
return this.sync();
}
return true;
}
threadMessageHandler(thread) {
const self = this;
return function (msg) {
logger_1.default.debug('Received sync thread message: ' + JSON.stringify(msg));
switch (msg.message) {
case 'ready':
self.emit('THREADREADY');
break;
case 'sync':
default:
self.threadSync(thread, msg);
}
};
}
async threadSync(thread, msg) {
if (msg.error) {
logger_1.default.warn(`Syncing thread ${thread.threadId} returned an error: ${msg.error}`);
}
const gimmeAnotherBlock = !msg.notFound;
const atTip = await this.areWeAtTheTip();
const moreBlocksToGive = !atTip || this.syncQueue.length > 0;
// If last block was found and there's more to sync
if (gimmeAnotherBlock && moreBlocksToGive) {
// If queue is empty, then !atTip must be true, so add next block to queue
if (this.syncQueue.length === 0) {
this.addBlockToQueue(this.syncHeight++);
}
const blockNum = this.syncQueue.shift();
thread.postMessage({ message: 'sync', blockNum });
this.currentHeight = Math.max(msg.blockNum, this.currentHeight);
// If the thread didn't find the block for some reason, but we know it exists
}
else if (msg.blockNum < this.bestBlock && !atTip) {
logger_1.default.debug('Known block not found by thread: %o. Retrying.', msg.blockNum);
thread.postMessage({ message: 'sync', blockNum: msg.blockNum });
// Otherwise, decrement active syncing threads counter
}
else {
this.syncingThreads--;
if (!this.syncingThreads) {
this.finishSync();
}
}
}
async areWeAtTheTip() {
if (this.bestBlock > this.syncHeight + this.mtSyncTipPad) {
return false;
}
const rpc = this.getRpc();
this.bestBlock = await rpc.web3.eth.getBlockNumber();
if (this.bestBlock > this.syncHeight + this.mtSyncTipPad) {
return false;
}
return true;
}
getWorkerThread(workerData) {
return new worker_threads_1.Worker(__dirname + '/syncWorker.js', {
workerData
});
}
async initializeThreads() {
if (this.threads.length > 0) {
return;
}
const self = this;
let threadCnt = this.config.threads || os.cpus().length - 1; // Subtract 1 for this process/thread
if (threadCnt <= 0) {
throw new Error('Invalid number of syncing threads.');
}
logger_1.default.info(`Initializing ${threadCnt} syncing threads.`);
const workerData = { chain: this.chain, network: this.network };
for (let i = 0; i < threadCnt; i++) {
const thread = this.getWorkerThread(workerData);
this.threads.push(thread);
thread.on('message', this.threadMessageHandler(thread));
thread.on('exit', function (code) {
self.syncingThreads--;
self.threads.splice(self.threads.findIndex(t => t.threadId === thread.threadId), 1);
if (code !== 0) {
logger_1.default.error('Thread exited with non-zero code: %o', code);
}
if (self.threads.length === 0) {
logger_1.default.info('All syncing threads stopped.');
}
});
thread.postMessage({ message: 'start' });
}
await new Promise(resolve => this.once('THREADREADY', resolve));
logger_1.default.info('Syncing threads ready.');
}
async getVerifiedBlockHeight() {
const state = await state_1.StateStorage.collection.findOne({}, { sort: { _id: -1 } });
const savedStartHeight = state?.verifiedBlockHeight?.[this.chain]?.[this.network] || 0;
return Math.max(savedStartHeight, this.config.syncStartHeight || 0);
}
async finishSync() {
clearInterval(this.syncInterval);
if (this.stopping) {
return;
}
const verifiedBlockHeight = await this.getVerifiedBlockHeight();
logger_1.default.info(`Verifying ${this.currentHeight - verifiedBlockHeight} ${this.chain}:${this.network} blocks for consistency.`);
const gaps = await block_1.EVMBlockStorage.getBlockSyncGaps({
chain: this.chain,
network: this.network,
startHeight: verifiedBlockHeight
});
logger_1.default.info(`Verification of ${this.chain}:${this.network} blocks finished.`);
if (gaps.length) {
logger_1.default.info(`Gaps found. Attempting to fill ${gaps.length} block gaps.`);
this.resolvingGaps = true;
this.gapsLength = gaps.length;
this.syncingThreads = this.threads.length;
for (let blockNum of gaps) {
this.addBlockToQueue(blockNum);
}
this.syncing = false;
this.sync();
}
else {
logger_1.default.info(`${this.chain}:${this.network} multi-thread sync is finished. Switching to main process sync.`);
await state_1.StateStorage.setVerifiedBlockHeight({ chain: this.chain, network: this.network, height: this.currentHeight });
this.emit('INITIALSYNCDONE');
this.shutdownThreads();
this.syncing = false;
}
}
shutdownThreads() {
for (let thread of this.threads) {
thread.postMessage({ message: 'shutdown' });
}
clearInterval(this.syncInterval);
}
stop() {
this.shutdownThreads();
this.stopping = true;
}
}
exports.MultiThreadSync = MultiThreadSync;
//# sourceMappingURL=sync.js.map