@lodestar/beacon-node
Version:
A Typescript implementation of the beacon chain
968 lines • 71 kB
JavaScript
import { routes } from "@lodestar/api";
import { ForkSeq } from "@lodestar/params";
import { RequestError, RequestErrorCode } from "@lodestar/reqresp";
import { computeTimeAtSlot } from "@lodestar/state-transition";
import { fromHex, prettyPrintIndices, pruneSetToMax, sleep, toRootHex } from "@lodestar/utils";
import { isBlockInputBlobs, isBlockInputColumns } from "../chain/blocks/blockInput/blockInput.js";
import { BlockInputSource } from "../chain/blocks/blockInput/types.js";
import { PayloadError, PayloadErrorCode } from "../chain/blocks/importExecutionPayload.js";
import { PayloadEnvelopeInputSource } from "../chain/blocks/payloadEnvelopeInput/index.js";
import { BlockError, BlockErrorCode } from "../chain/errors/index.js";
import { ChainEvent } from "../chain/index.js";
import { validateGloasBlockDataColumnSidecars } from "../chain/validation/dataColumnSidecar.js";
import { validateGossipExecutionPayloadEnvelope } from "../chain/validation/executionPayloadEnvelope.js";
import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
import { shuffle } from "../util/shuffle.js";
import { sortBy } from "../util/sortBy.js";
import { wrapError } from "../util/wrapError.js";
import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
import { PendingBlockInputStatus, PendingBlockType, PendingPayloadInputStatus, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, getPayloadSyncCacheItemRootHex, getPayloadSyncCacheItemSlot, isPendingBlockInput, isPendingPayloadEnvelope, isPendingPayloadInput, } from "./types.js";
import { DownloadByRootError, downloadByRoot } from "./utils/downloadByRoot.js";
import { getAllDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
import { getRateLimitedUntilMs } from "./utils/rateLimit.js";
const MAX_ATTEMPTS_PER_BLOCK = 5;
const MAX_KNOWN_BAD_BLOCKS = 500;
const MAX_PENDING_BLOCKS = 100;
var FetchResult;
(function (FetchResult) {
FetchResult["SuccessResolved"] = "success_resolved";
FetchResult["SuccessMissingParent"] = "success_missing_parent";
FetchResult["SuccessLate"] = "success_late";
FetchResult["FailureTriedAllPeers"] = "failure_tried_all_peers";
FetchResult["FailureMaxAttempts"] = "failure_max_attempts";
})(FetchResult || (FetchResult = {}));
class UnknownBlockRateLimitedError extends Error {
constructor(message) {
super(message);
this.name = "UnknownBlockRateLimitedError";
}
}
/**
* BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot. The
* blockRoot may have been found via object gossip, or the API. Gossip objects that can trigger a search are block,
* blobs, columns, attestations, etc. In the case of blocks and data this is generally during the current slot but
* can also be for items that are received late but are not fully verified and thus not in fork-choice (old blocks on
* an unknown fork). It can also be triggered via an attestation (or sync committee message or any other item that
* gets gossiped) that references a blockRoot that is not in fork-choice. In rare (and realistically should not happen)
* situations it can get triggered via the API when the validator attempts to publish a block, attestation, aggregate
* and proof or a sync committee contribution that has unknown information included (parentRoot for instance).
*
* The goal of the class is to make sure that all information that is necessary for import into fork-choice is pulled
* from peers so that the block and data can be processed, and thus the object that triggered the search can be
* referenced and validated.
*
* The most common case for this search is a set of block/data that comes across gossip for the current slot, during
* normal chain operation, but not everything was received before the gossip cutoff window happens so it is necessary
* to pull remaining data via req/resp so that fork-choice can be updated prior to making an attestation for the
* current slot.
*
* Event sources for old UnknownBlock
*
* - publishBlock
* - gossipHandlers
* - searchUnknownBlock
* = produceSyncCommitteeContribution
* = validateGossipFnRetryUnknownRoot
* * submitPoolAttestationsV2
* * publishAggregateAndProofsV2
* = onPendingGossipsubMessage
* * NetworkEvent.pendingGossipsubMessage
* - onGossipsubMessage
*/
export class BlockInputSync {
config;
network;
chain;
logger;
metrics;
opts;
/**
* block RootHex -> PendingBlock. To avoid finding same root at the same time
*/
pendingBlocks = new Map();
// Payload sync is keyed by beacon block root as well, so block and payload queues can unblock each other.
pendingPayloads = new Map();
knownBadBlocks = new Set();
maxPendingBlocks;
subscribedToNetworkEvents = false;
peerBalancer;
rateLimitBackoffTimeout;
constructor(config, network, chain, logger, metrics, opts) {
this.config = config;
this.network = network;
this.chain = chain;
this.logger = logger;
this.metrics = metrics;
this.opts = opts;
this.maxPendingBlocks = opts?.maxPendingBlocks ?? MAX_PENDING_BLOCKS;
this.peerBalancer = new UnknownBlockPeerBalancer();
if (metrics) {
metrics.blockInputSync.pendingBlocks.addCollect(() => metrics.blockInputSync.pendingBlocks.set(this.pendingBlocks.size));
metrics.blockInputSync.pendingPayloads.addCollect(() => metrics.blockInputSync.pendingPayloads.set(this.pendingPayloads.size));
metrics.blockInputSync.knownBadBlocks.addCollect(() => metrics.blockInputSync.knownBadBlocks.set(this.knownBadBlocks.size));
}
}
subscribeToNetwork() {
if (this.opts?.disableBlockInputSync) {
this.logger.verbose("BlockInputSync disabled by disableBlockInputSync option.");
return;
}
// cannot chain to the above if or the log will be incorrect
if (!this.subscribedToNetworkEvents) {
this.logger.verbose("BlockInputSync enabled.");
this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
this.chain.emitter.on(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
this.chain.emitter.on(routes.events.EventType.block, this.onBlockImported);
this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
this.subscribedToNetworkEvents = true;
}
}
unsubscribeFromNetwork() {
this.logger.verbose("BlockInputSync disabled.");
this.clearRateLimitBackoffTimer();
this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
this.chain.emitter.off(routes.events.EventType.block, this.onBlockImported);
this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadImported);
this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
this.subscribedToNetworkEvents = false;
}
close() {
this.unsubscribeFromNetwork();
}
isSubscribedToNetwork() {
return this.subscribedToNetworkEvents;
}
/**
* Process an unknownBlock event and register the block in `pendingBlocks` Map.
*/
onUnknownBlockRoot = (data) => {
try {
this.addByRootHex(data.rootHex, data.peer);
this.triggerUnknownBlockSearch();
this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_BLOCK_ROOT });
this.metrics?.blockInputSync.source.inc({ source: data.source });
}
catch (e) {
this.logger.debug("Error handling unknownBlockRoot event", {}, e);
}
};
/**
* Process an unknownBlockInput event and register the block in `pendingBlocks` Map.
*/
onIncompleteBlockInput = (data) => {
try {
this.addByBlockInput(data.blockInput, data.peer);
this.triggerUnknownBlockSearch();
this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.INCOMPLETE_BLOCK_INPUT });
this.metrics?.blockInputSync.source.inc({ source: data.source });
}
catch (e) {
this.logger.debug("Error handling incompleteBlockInput event", {}, e);
}
};
onUnknownEnvelopeBlockRoot = (data) => {
try {
this.addByPayloadRootHex(data.rootHex, data.peer);
this.triggerUnknownBlockSearch();
this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_DATA });
this.metrics?.blockInputSync.source.inc({ source: data.source });
}
catch (e) {
this.logger.debug("Error handling unknownEnvelopeBlockRoot event", {}, e);
}
};
onIncompletePayloadEnvelope = (data) => {
try {
this.addByPayloadInput(data.payloadInput, data.peer);
this.triggerUnknownBlockSearch();
this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_DATA });
this.metrics?.blockInputSync.source.inc({ source: data.source });
}
catch (e) {
this.logger.debug("Error handling incompletePayloadEnvelope event", {}, e);
}
};
/**
* Process an unknownBlockParent event and register the block in `pendingBlocks` Map.
*/
onUnknownParent = (data) => {
try {
const missingDependency = this.getMissingBlockDependency(data.blockInput);
if (missingDependency.kind === "invalidParentPayload") {
this.addByBlockInput(data.blockInput, data.peer);
const pendingBlock = this.pendingBlocks.get(data.blockInput.blockRootHex);
if (pendingBlock && isPendingBlockInput(pendingBlock)) {
this.logger.debug("Ignoring block with conflicting parent payload hash", {
slot: pendingBlock.blockInput.slot,
root: pendingBlock.blockInput.blockRootHex,
parentRoot: missingDependency.parentRootHex,
parentBlockHash: missingDependency.parentBlockHashHex,
});
this.removeAndDownScoreAllDescendants(pendingBlock);
}
return;
}
if (missingDependency.kind === "parentPayload") {
this.addByPayloadRootHex(missingDependency.rootHex, data.peer);
}
else if (missingDependency.kind === "parentBlock") {
this.addByRootHex(missingDependency.rootHex, data.peer);
}
this.addByBlockInput(data.blockInput, data.peer);
this.triggerUnknownBlockSearch();
this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_PARENT });
this.metrics?.blockInputSync.source.inc({ source: data.source });
}
catch (e) {
this.logger.debug("Error handling unknownParent event", {}, e);
}
};
onBlockImported = () => {
if (this.pendingPayloads.size > 0) {
this.triggerUnknownBlockSearch();
}
};
onPayloadImported = ({ blockRoot, }) => {
this.pendingPayloads.delete(blockRoot);
this.triggerUnknownBlockSearch();
};
addByRootHex = (rootHex, peerIdStr) => {
let pendingBlock = this.pendingBlocks.get(rootHex);
let added = false;
if (!pendingBlock) {
pendingBlock = {
status: PendingBlockInputStatus.pending,
rootHex: rootHex,
peerIdStrings: new Set(),
timeAddedSec: Date.now() / 1000,
};
this.pendingBlocks.set(rootHex, pendingBlock);
added = true;
this.logger.verbose("Added new rootHex to BlockInputSync.pendingBlocks", {
root: pendingBlock.rootHex,
peerIdStr: peerIdStr ?? "unknown peer",
});
}
if (peerIdStr) {
pendingBlock.peerIdStrings.add(peerIdStr);
}
// TODO: check this prune methodology
// Limit pending blocks to prevent DOS attacks that cause OOM
const prunedItemCount = pruneSetToMax(this.pendingBlocks, this.maxPendingBlocks);
if (prunedItemCount > 0) {
this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
}
return added;
};
addByBlockInput = (blockInput, peerIdStr) => {
let pendingBlock = this.pendingBlocks.get(blockInput.blockRootHex);
// if entry is missing or was added via rootHex and now we have more complete information overwrite
// the existing information with the more complete cache entry
if (!pendingBlock || !isPendingBlockInput(pendingBlock)) {
pendingBlock = {
// can be added via unknown parent and we may already have full block input. need to check and set correctly
// so we pull the data if its missing or handle the block correctly in getIncompleteAndAncestorBlocks
status: blockInput.hasBlockAndAllData() ? PendingBlockInputStatus.downloaded : PendingBlockInputStatus.pending,
blockInput,
peerIdStrings: new Set(),
timeAddedSec: Date.now() / 1000,
};
this.pendingBlocks.set(blockInput.blockRootHex, pendingBlock);
this.logger.verbose("Added blockInput to BlockInputSync.pendingBlocks", pendingBlock.blockInput.getLogMeta());
}
if (peerIdStr) {
pendingBlock.peerIdStrings.add(peerIdStr);
}
// TODO: check this prune methodology
// Limit pending blocks to prevent DOS attacks that cause OOM
const prunedItemCount = pruneSetToMax(this.pendingBlocks, this.maxPendingBlocks);
if (prunedItemCount > 0) {
this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
}
};
addByPayloadRootHex = (rootHex, peerIdStr) => {
let pendingPayload = this.pendingPayloads.get(rootHex);
let added = false;
if (!pendingPayload) {
pendingPayload = {
status: PendingPayloadInputStatus.pending,
rootHex,
peerIdStrings: new Set(),
timeAddedSec: Date.now() / 1000,
};
this.pendingPayloads.set(rootHex, pendingPayload);
added = true;
this.logger.verbose("Added new payload rootHex to BlockInputSync.pendingPayloads", {
root: rootHex,
peerIdStr: peerIdStr ?? "unknown peer",
});
}
if (peerIdStr) {
pendingPayload.peerIdStrings.add(peerIdStr);
}
const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
if (prunedItemCount > 0) {
this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
}
return added;
};
addByPayloadInput = (payloadInput, peerIdStr, envelope) => {
const pendingPayload = this.toPendingPayloadInput(payloadInput, this.pendingPayloads.get(payloadInput.blockRootHex), envelope);
if (peerIdStr) {
pendingPayload.peerIdStrings.add(peerIdStr);
}
this.pendingPayloads.set(payloadInput.blockRootHex, pendingPayload);
const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
if (prunedItemCount > 0) {
this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
}
};
onPeerConnected = (data) => {
try {
const peerId = data.peer;
const peerSyncMeta = this.network.getConnectedPeerSyncMeta(peerId);
this.peerBalancer.onPeerConnected(data.peer, peerSyncMeta);
this.triggerUnknownBlockSearch();
}
catch (e) {
this.logger.debug("Error handling peerConnected event", {}, e);
}
};
onPeerDisconnected = (data) => {
const peerId = data.peer;
this.peerBalancer.onPeerDisconnected(peerId);
this.scheduleRateLimitBackoffRetry();
};
/**
* Post-gloas, a locally complete block can still be blocked on its parent's execution payload lineage.
* Distinguish which dependency is missing so the scheduler can enqueue the right follow-up work.
*/
getMissingBlockDependency(blockInput) {
const parentRootHex = blockInput.parentRootHex;
if (!this.chain.forkChoice.hasBlockHex(parentRootHex)) {
return { kind: "parentBlock", rootHex: parentRootHex };
}
if (!blockInput.hasBlock()) {
return { kind: "block", rootHex: blockInput.blockRootHex };
}
if (this.config.getForkSeq(blockInput.slot) < ForkSeq.gloas) {
return { kind: "ready" };
}
const block = blockInput.getBlock();
const parentBlockHashHex = toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash);
if (this.chain.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex) !== null) {
return { kind: "ready" };
}
if (this.chain.forkChoice.hasPayloadHexUnsafe(parentRootHex)) {
return { kind: "invalidParentPayload", parentRootHex, parentBlockHashHex };
}
const parentPayloadInput = this.chain.seenPayloadEnvelopeInputCache.get(parentRootHex);
if (parentPayloadInput) {
if (parentPayloadInput.getBlockHashHex() === parentBlockHashHex) {
return { kind: "parentPayload", rootHex: parentRootHex };
}
return { kind: "invalidParentPayload", parentRootHex, parentBlockHashHex };
}
return { kind: "parentPayload", rootHex: parentRootHex };
}
advancePendingBlock(pendingBlock) {
const missingDependency = this.getMissingBlockDependency(pendingBlock.blockInput);
switch (missingDependency.kind) {
case "ready":
return "ready";
case "block":
pendingBlock.status = PendingBlockInputStatus.pending;
return "queued_block";
case "parentBlock": {
let added = this.addByRootHex(missingDependency.rootHex);
for (const peerIdStr of pendingBlock.peerIdStrings) {
added = this.addByRootHex(missingDependency.rootHex, peerIdStr) || added;
}
return added ? "queued_parent_block" : "blocked";
}
case "parentPayload": {
let added = this.addByPayloadRootHex(missingDependency.rootHex);
for (const peerIdStr of pendingBlock.peerIdStrings) {
added = this.addByPayloadRootHex(missingDependency.rootHex, peerIdStr) || added;
}
return added ? "queued_parent_payload" : "blocked";
}
case "invalidParentPayload":
this.logger.debug("Removing block with conflicting parent payload hash", {
slot: pendingBlock.blockInput.slot,
root: pendingBlock.blockInput.blockRootHex,
parentRoot: missingDependency.parentRootHex,
parentBlockHash: missingDependency.parentBlockHashHex,
});
this.removeAndDownScoreAllDescendants(pendingBlock);
return "removed";
}
}
toPendingPayloadInput(payloadInput, previous, envelope) {
// Normalize every payload queueing path into the same cache shape while preserving first-seen
// timing and peer provenance from any earlier by-root or envelope-only entry.
const queuedEnvelope = envelope ?? (previous && isPendingPayloadEnvelope(previous) ? previous.envelope : undefined);
if (queuedEnvelope && !payloadInput.hasPayloadEnvelope()) {
payloadInput.addPayloadEnvelope({
envelope: queuedEnvelope,
source: PayloadEnvelopeInputSource.byRoot,
seenTimestampSec: Date.now() / 1000,
});
}
return {
status: payloadInput.isComplete() ? PendingPayloadInputStatus.downloaded : PendingPayloadInputStatus.pending,
payloadInput,
timeAddedSec: previous?.timeAddedSec ?? Date.now() / 1000,
timeSyncedSec: payloadInput.isComplete() ? Date.now() / 1000 : undefined,
peerIdStrings: new Set(previous?.peerIdStrings ?? []),
};
}
/**
* Gather tip parent blocks with unknown parent and do a search for all of them
*/
triggerUnknownBlockSearch = () => {
// Cheap early stop to prevent calling the network.getConnectedPeers()
if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
return;
}
// If the node loses all peers with pending unknown blocks or payloads, the sync will stall
const connectedPeers = this.network.getConnectedPeers();
const hasConnectedPeers = connectedPeers.length > 0;
const { unknowns, ancestors } = getUnknownAndAncestorBlocks(this.pendingBlocks);
let processedBlocks = 0;
let shouldRerunBlockSearch = false;
for (const block of ancestors) {
const advanceResult = this.advancePendingBlock(block);
switch (advanceResult) {
case "ready":
processedBlocks++;
this.processReadyBlock(block).catch((e) => {
this.logger.debug("Unexpected error - process old downloaded block", {}, e);
});
break;
case "queued_block":
case "queued_parent_block":
shouldRerunBlockSearch = true;
break;
case "queued_parent_payload":
case "blocked":
case "removed":
break;
}
}
if (unknowns.length > 0) {
if (!hasConnectedPeers) {
this.logger.debug("No connected peers, skipping unknown block download.");
}
else {
// Most of the time there is exactly 1 unknown block
for (const block of unknowns) {
this.downloadBlock(block).catch((e) => {
this.logger.debug("Unexpected error - downloadBlock", { root: getBlockInputSyncCacheItemRootHex(block) }, e);
});
}
}
}
else if (ancestors.length > 0) {
// It's rare when there is no unknown block
// see https://github.com/ChainSafe/lodestar/issues/5649#issuecomment-1594213550
this.logger.verbose("No unknown block, process ancestor downloaded blocks", {
pendingBlocks: this.pendingBlocks.size,
ancestorBlocks: ancestors.length,
processedBlocks,
});
}
// Blocks can unblock payloads and payloads can unblock blocks, so every scheduler pass services both queues.
for (const payload of Array.from(this.pendingPayloads.values())) {
if (isPendingPayloadInput(payload) && payload.status === PendingPayloadInputStatus.downloaded) {
this.processPayload(payload).catch((e) => {
this.logger.debug("Unexpected error - process downloaded payload", {}, e);
});
continue;
}
if (isPendingPayloadEnvelope(payload)) {
this.reconcilePayloadEnvelope(payload).catch((e) => {
this.logger.debug("Unexpected error - reconcile pending payload envelope", {}, e);
});
continue;
}
if (!hasConnectedPeers) {
this.logger.debug("No connected peers, skipping unknown payload download.", {
root: getPayloadSyncCacheItemRootHex(payload),
});
continue;
}
this.downloadPayload(payload).catch((e) => {
this.logger.debug("Unexpected error - downloadPayload", { root: getPayloadSyncCacheItemRootHex(payload) }, e);
});
}
if (shouldRerunBlockSearch) {
this.triggerUnknownBlockSearch();
}
};
scheduleRateLimitBackoffRetry() {
this.clearRateLimitBackoffTimer();
if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
return;
}
const now = Date.now();
const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
if (retryAt === null) {
return;
}
this.rateLimitBackoffTimeout = setTimeout(() => {
this.rateLimitBackoffTimeout = undefined;
this.triggerUnknownBlockSearch();
this.scheduleRateLimitBackoffRetry();
}, Math.max(0, retryAt - now));
}
clearRateLimitBackoffTimer() {
if (this.rateLimitBackoffTimeout !== undefined) {
clearTimeout(this.rateLimitBackoffTimeout);
this.rateLimitBackoffTimeout = undefined;
}
}
async downloadBlock(block) {
if (block.status !== PendingBlockInputStatus.pending) {
return;
}
const rootHex = getBlockInputSyncCacheItemRootHex(block);
const logCtx = {
slot: getBlockInputSyncCacheItemSlot(block),
root: rootHex,
pendingBlocks: this.pendingBlocks.size,
};
this.logger.verbose("BlockInputSync.downloadBlock()", logCtx);
block.status = PendingBlockInputStatus.fetching;
const res = await wrapError(this.fetchBlockInput(block));
if (!res.err) {
this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
const pending = res.result;
this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
const blockSlot = pending.blockInput.slot;
const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
const delaySec = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.chain.genesisTime);
this.metrics?.blockInputSync.elapsedTimeTillReceived.observe(delaySec);
const parentInForkChoice = this.chain.forkChoice.hasBlockHex(pending.blockInput.parentRootHex);
const logCtx2 = {
...logCtx,
slot: blockSlot,
parentInForkChoice,
};
this.logger.verbose("Downloaded unknown block", logCtx2);
if (parentInForkChoice) {
// If the direct parent is already in fork choice, let the block state machine decide if
// the next step is block import, parent payload download, or branch removal.
const advanceResult = this.advancePendingBlock(pending);
switch (advanceResult) {
case "ready":
this.processReadyBlock(pending).catch((e) => {
this.logger.debug("Unexpected error - process newly downloaded block", logCtx2, e);
});
break;
case "queued_block":
case "queued_parent_block":
case "queued_parent_payload":
this.triggerUnknownBlockSearch();
break;
case "blocked":
case "removed":
break;
}
}
else if (blockSlot <= finalizedSlot) {
// the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
// we should found it through forkchoice. If not, we should penalize all peers sending us this block chain
// 0 - 1 - ... - n - finalizedSlot
// \
// parent 1 - parent 2 - ... - unknownParent block
this.logger.debug("Downloaded block is before finalized slot", {
...logCtx2,
finalizedSlot,
});
this.removeAndDownScoreAllDescendants(block);
}
else {
this.onUnknownBlockRoot({ rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot });
}
}
else {
if (res.err instanceof UnknownBlockRateLimitedError) {
const pendingBlock = this.pendingBlocks.get(rootHex);
if (pendingBlock) {
pendingBlock.status = PendingBlockInputStatus.pending;
}
this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
this.scheduleRateLimitBackoffRetry();
return;
}
this.metrics?.blockInputSync.downloadedBlocksError.inc();
this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
this.removeAndDownScoreAllDescendants(block);
}
}
/**
* Import a block that has already passed the local dependency checks in BlockInputSync.
* On error, remove and downscore descendants as appropriate for the failure type.
*/
async processReadyBlock(pendingBlock) {
if (pendingBlock.status !== PendingBlockInputStatus.downloaded) {
return;
}
pendingBlock.status = PendingBlockInputStatus.processing;
// this prevents unbundling attack
// see https://lighthouse-blog.sigmaprime.io/mev-unbundling-rpc.html
const { slot: blockSlot, proposerIndex } = pendingBlock.blockInput.getBlock().message;
const fork = this.config.getForkName(blockSlot);
const proposerBoostWindowMs = this.config.getAttestationDueMs(fork);
if (this.chain.clock.msFromSlot(blockSlot) < proposerBoostWindowMs &&
this.chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
// proposer is known by a gossip block already, wait a bit to make sure this block is not
// eligible for proposer boost to prevent unbundling attack
this.logger.verbose("Avoid proposer boost for this block of known proposer", {
slot: blockSlot,
root: pendingBlock.blockInput.blockRootHex,
proposerIndex,
});
await sleep(proposerBoostWindowMs);
}
// At gossip time, it's critical to keep a good number of mesh peers.
// To do that, the Gossip Job Wait Time should be consistently <3s to avoid the behavior penalties in gossip
// Gossip Job Wait Time depends on the BLS Job Wait Time
// so `blsVerifyOnMainThread = true`: we want to verify signatures immediately without affecting the bls thread pool.
// otherwise we can't utilize bls thread pool capacity and Gossip Job Wait Time can't be kept low consistently.
// See https://github.com/ChainSafe/lodestar/issues/3792
const res = await wrapError(this.chain.processBlock(pendingBlock.blockInput, {
ignoreIfKnown: true,
// there could be finalized/head sync at the same time so we need to ignore if finalized
// see https://github.com/ChainSafe/lodestar/issues/5650
ignoreIfFinalized: true,
blsVerifyOnMainThread: true,
}));
if (res.err)
this.metrics?.blockInputSync.processedBlocksError.inc();
else
this.metrics?.blockInputSync.processedBlocksSuccess.inc();
if (!res.err) {
// no need to update status to "processed", delete anyway
this.pendingBlocks.delete(pendingBlock.blockInput.blockRootHex);
// Re-enter the scheduler so descendants blocked on either parent blocks or parent payloads
// are advanced through the same dependency checks as every other pending item.
this.triggerUnknownBlockSearch();
}
else {
const errorData = { slot: pendingBlock.blockInput.slot, root: pendingBlock.blockInput.blockRootHex };
if (res.err instanceof BlockError) {
switch (res.err.type.code) {
// This cases are already handled with `{ignoreIfKnown: true}`
// case BlockErrorCode.ALREADY_KNOWN:
// case BlockErrorCode.GENESIS_BLOCK:
case BlockErrorCode.PARENT_UNKNOWN:
case BlockErrorCode.PRESTATE_MISSING:
// Should not happen, mark as downloaded to try again latter
this.logger.debug("Attempted to process block but its parent was still unknown", errorData, res.err);
pendingBlock.status = PendingBlockInputStatus.downloaded;
break;
case BlockErrorCode.PARENT_PAYLOAD_UNKNOWN:
this.logger.error("processReadyBlock() hit unexpected parent payload dependency after readiness checks", {
...errorData,
parentRoot: pendingBlock.blockInput.parentRootHex,
parentBlockHash: res.err.type.parentBlockHash,
}, res.err);
pendingBlock.status = PendingBlockInputStatus.downloaded;
break;
case BlockErrorCode.EXECUTION_ENGINE_ERROR:
// Removing the block(s) without penalizing the peers, hoping for EL to
// recover on a latter download + verify attempt
this.removeAllDescendants(pendingBlock);
break;
default:
// Block is not correct with respect to our chain. Log error loudly
this.logger.debug("Error processing block from unknown parent sync", errorData, res.err);
this.removeAndDownScoreAllDescendants(pendingBlock);
}
}
// Probably a queue error or something unwanted happened, mark as pending to try again latter
else {
this.logger.debug("Unknown error processing block from unknown block sync", errorData, res.err);
pendingBlock.status = PendingBlockInputStatus.downloaded;
}
}
}
/**
* Reconcile an envelope-first payload entry once the block import path has seeded its
* PayloadEnvelopeInput. This may queue block download, validate the speculative envelope, or
* downgrade back to by-root fetching when the cached envelope does not match the imported block.
*/
async reconcilePayloadEnvelope(pendingPayload) {
const rootHex = getPayloadSyncCacheItemRootHex(pendingPayload);
if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
this.pendingPayloads.delete(rootHex);
return;
}
const payloadInput = this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
if (!payloadInput) {
if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
// Column commitments live on the block body, so an envelope-only entry has to pull the block first.
if (!this.pendingBlocks.has(rootHex)) {
this.addByRootHex(rootHex);
}
const pendingBlock = this.pendingBlocks.get(rootHex);
if (pendingBlock && this.network.getConnectedPeers().length > 0) {
await this.downloadBlock(pendingBlock);
}
}
else {
this.logger.debug("Missing PayloadEnvelopeInput for known block while reconciling payload envelope", {
root: rootHex,
});
}
return;
}
if (!payloadInput.hasPayloadEnvelope()) {
const validationResult = await wrapError(validateGossipExecutionPayloadEnvelope(this.chain, pendingPayload.envelope));
if (validationResult.err) {
this.logger.debug("Pending payload envelope failed validation after block import, refetching by root", { slot: pendingPayload.envelope.message.payload.slotNumber, root: rootHex }, validationResult.err);
const pendingPayloadByRoot = {
status: PendingPayloadInputStatus.pending,
rootHex,
timeAddedSec: pendingPayload.timeAddedSec,
peerIdStrings: new Set(pendingPayload.peerIdStrings),
};
this.pendingPayloads.set(rootHex, pendingPayloadByRoot);
if (this.network.getConnectedPeers().length > 0) {
await this.downloadPayload(pendingPayloadByRoot);
}
return;
}
}
const upgradedPayload = this.toPendingPayloadInput(payloadInput, pendingPayload, pendingPayload.envelope);
this.pendingPayloads.set(rootHex, upgradedPayload);
if (upgradedPayload.status === PendingPayloadInputStatus.downloaded) {
await this.processPayload(upgradedPayload);
return;
}
await this.downloadPayload(upgradedPayload);
}
async downloadPayload(payload) {
if (isPendingPayloadEnvelope(payload)) {
await this.reconcilePayloadEnvelope(payload);
return;
}
const rootHex = getPayloadSyncCacheItemRootHex(payload);
if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
this.pendingPayloads.delete(rootHex);
return;
}
if (payload.status !== PendingPayloadInputStatus.pending) {
return;
}
const logCtx = {
slot: getPayloadSyncCacheItemSlot(payload),
root: rootHex,
pendingPayloads: this.pendingPayloads.size,
};
this.logger.verbose("BlockInputSync.downloadPayload()", logCtx);
payload.status = PendingPayloadInputStatus.fetching;
const res = await wrapError(this.fetchPayloadInput(payload));
if (!res.err) {
const pendingPayload = res.result;
this.pendingPayloads.set(getPayloadSyncCacheItemRootHex(pendingPayload), pendingPayload);
if (isPendingPayloadEnvelope(pendingPayload)) {
await this.reconcilePayloadEnvelope(pendingPayload);
}
else if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
await this.processPayload(pendingPayload);
}
return;
}
this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
if (!isPendingPayloadEnvelope(payload)) {
payload.status = PendingPayloadInputStatus.pending;
}
}
async processPayload(pendingPayload) {
const rootHex = pendingPayload.payloadInput.blockRootHex;
const logCtx = { slot: pendingPayload.payloadInput.slot, root: rootHex };
if (pendingPayload.status !== PendingPayloadInputStatus.downloaded) {
this.logger.debug("Skipping payload processing before payload input is downloaded", {
...logCtx,
status: pendingPayload.status,
});
return;
}
if (this.chain.forkChoice.hasPayloadHexUnsafe(rootHex)) {
this.logger.debug("Payload already imported while processing unknown payload", logCtx);
this.pendingPayloads.delete(rootHex);
return;
}
if (!this.chain.forkChoice.hasBlockHex(rootHex)) {
this.logger.debug("Payload input is ready before its block is in fork choice", logCtx);
const added = this.addByRootHex(rootHex);
pendingPayload.status = PendingPayloadInputStatus.downloaded;
if (added) {
this.triggerUnknownBlockSearch();
}
return;
}
pendingPayload.status = PendingPayloadInputStatus.processing;
const res = await wrapError(this.chain.processExecutionPayload(pendingPayload.payloadInput));
if (!res.err) {
this.logger.debug("Processed payload from unknown sync", logCtx);
this.pendingPayloads.delete(rootHex);
this.triggerUnknownBlockSearch();
return;
}
if (res.err instanceof PayloadError) {
switch (res.err.type.code) {
case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:
// Payload sync discovered the block dependency before the block queue did. Re-enqueue the
// block and keep the payload ready so the scheduler can retry once the block reaches fork choice.
if (this.addByRootHex(rootHex)) {
this.triggerUnknownBlockSearch();
}
// Keep the payload out of any synchronous requeue pass; a later scheduler pass will retry it.
pendingPayload.status = PendingPayloadInputStatus.downloaded;
break;
case PayloadErrorCode.EXECUTION_ENGINE_ERROR:
this.logger.debug("Execution engine error while processing payload from unknown sync", logCtx, res.err);
pendingPayload.status = PendingPayloadInputStatus.downloaded;
break;
case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
case PayloadErrorCode.INVALID_SIGNATURE:
// TODO GLOAS: Decide how invalid payload inputs should eventually leave memory without
// reintroducing envelope replacement / recreation flows.
this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
this.removePendingPayloadAndDescendants(rootHex);
break;
default:
this.logger.debug("Error processing payload from unknown sync", logCtx, res.err);
this.pendingPayloads.delete(rootHex);
}
return;
}
this.logger.debug("Unknown error processing payload from unknown sync", logCtx, res.err);
pendingPayload.status = PendingPayloadInputStatus.downloaded;
}
/**
* Download payload material keyed by beacon block root. Unlike block download, payload sync may
* already have a locally cached envelope or partial columns, so each attempt starts from local state
* and only asks peers for the remaining pieces.
*/
async fetchPayloadInput(cacheItem) {
const rootHex = getPayloadSyncCacheItemRootHex(cacheItem);
const blockRoot = fromHex(rootHex);
const excludedPeers = new Set();
let slot = getPayloadSyncCacheItemSlot(cacheItem);
let payloadInput = isPendingPayloadInput(cacheItem)
? cacheItem.payloadInput
: this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
let i = 0;
let deferredByRateLimit = false;
while (i++ < this.getMaxDownloadAttempts()) {
const pendingColumns = payloadInput?.hasAllData()
? new Set()
: new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
if (peerMeta === null) {
if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
throw new UnknownBlockRateLimitedError(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`);
}
throw Error(`Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`);
}
const { peerId, client: peerClient } = peerMeta;
cacheItem.peerIdStrings.add(peerId);
try {
if (!envelope) {
envelope = await this.fetchExecutionPayloadEnvelope(peerId, blockRoot, rootHex);
slot = envelope.message.payload.slotNumber;
}
payloadInput ??= this.chain.seenPayloadEnvelopeInputCache.get(rootHex);
if (!payloadInput) {
if (this.chain.forkChoice.hasBlockHex(rootHex)) {
throw new Error(`Missing PayloadEnvelopeInput for known block ${rootHex}`);
}
// Keep the validated envelope around, but wait for the block body before turning it into a full payload input.
return {
status: PendingPayloadInputStatus.waitingForBlock,
envelope,
timeAddedSec: cacheItem.timeAddedSec,
peerIdStrings: cacheItem.peerIdStrings,
};
}
if (!payloadInput.hasPayloadEnvelope()) {
await validateGossipExecutionPayloadEnvelope(this.chain, envelope);
}
let pendingPayload = this.toPendingPayloadInput(payloadInput, cacheItem, envelope);
if (!pendingPayload.payloadInput.hasAllData()) {
const missing = pendingPayload.payloadInput.getMissingSampledColumnMeta().missing;
if (missing.length > 0) {
const columnSidecars = await this.fetchPayloadColumns(peerMeta, pendingPayload.payloadInput, missing);
const seenTimestampSec = Date.now() / 1000;
for (const columnSidecar of columnSidecars) {
if (pendingPayload.payloadInput.hasColumn(columnSidecar.index)) {
continue;
}
pendingPayload.payloadInput.addColumn({
columnSidecar,
source: PayloadEnvelopeInputSource.byRoot,
seenTimestampSec,
peerIdStr: peerId,
});
}
pendingPayload = this.toPendingPayloadInput(pendingPayload.payloadInput, pendingPayload);
}
}
this.logger.verbose("BlockInputSync.fetchPayloadInput: successful download", {
slot,
rootHex,
peerId,
peerClient,
hasPayload: pendingPayload.payloadInput.hasPayloadEnvelope(),
hasAllData: pendingPayload.payloadInput.hasAllData(),
});
if (pendingPayload.status === PendingPayloadInputStatus.downloaded) {
return pendingPayload;
}
cacheItem = pendingPayload;
payloadInput = pendingPayload.payloadInput;
}
catch (e) {
this.logger.debug("Error downloading payload in BlockInputSync.fetchPayloadInput", { slot, rootHex, attempt: i, peer: peerId, peerClient }, e);
const rateLimitedUntilMs = getRateLimitedUntilMs(e);
if (rateLimitedUntilMs !== null) {
deferredByRateLimit = true;
this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
this.scheduleRateLimitBackoffRetry();
}
else if (e instanceof RequestError) {
switch (e.type.code) {
case RequestErrorCode.REQUEST_RATE_LIMITED:
case RequestErrorCode.REQUEST_TIMEOUT:
break;
default:
excludedPeers.add(peerId);
break;
}
}
else {
excludedPeers.add(peerId);
}
}
finally {
this.peerBalancer.onRequestCompleted(peerId);
}
}
if (deferredByRateLimit && this.peerBalancer.get