@lodestar/beacon-node
Version:
A Typescript implementation of the beacon chain
596 lines (521 loc) • 23.3 kB
text/typescript
import {
type GossipSub,
type GossipSubEvents,
type PublishResult,
StrictNoSign,
type TopicValidatorResult,
gossipsub,
} from "@libp2p/gossipsub";
import type {MetricsRegister, TopicLabel, TopicStrToLabel} from "@libp2p/gossipsub/metrics";
import type {PeerScoreParams, PeerScoreStatsDump} from "@libp2p/gossipsub/score";
import type {AddrInfo, PublishOpts, TopicStr} from "@libp2p/gossipsub/types";
import type {PeerId} from "@libp2p/interface";
import {peerIdFromString} from "@libp2p/peer-id";
import {type Multiaddr, multiaddr} from "@multiformats/multiaddr";
import {ENR} from "@chainsafe/enr";
import {routes} from "@lodestar/api";
import {BeaconConfig, ForkBoundary} from "@lodestar/config";
import {ATTESTATION_SUBNET_COUNT, SLOTS_PER_EPOCH, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params";
import {SubnetID} from "@lodestar/types";
import {Logger, Map2d, Map2dArr} from "@lodestar/utils";
import {RegistryMetricCreator} from "../../metrics/index.js";
import {callInNextEventLoop} from "../../util/eventLoop.js";
import {NetworkEvent, NetworkEventBus, NetworkEventData} from "../events.js";
import {Libp2p} from "../interface.js";
import {NetworkConfig} from "../networkConfig.js";
import {ClientKind} from "../peers/client.js";
import {PeersData} from "../peers/peersData.js";
import {DataTransformSnappy, fastMsgIdFn, msgIdFn, msgIdToStrFn} from "./encoding.js";
import {GossipTopic, GossipType} from "./interface.js";
import {Eth2GossipsubMetrics, createEth2GossipsubMetrics} from "./metrics.js";
import {
GOSSIP_D,
GOSSIP_D_HIGH,
GOSSIP_D_LOW,
computeGossipPeerScoreParams,
gossipScoreThresholds,
} from "./scoringParameters.js";
import {GossipTopicCache, getCoreTopicsAtFork, stringifyGossipTopic} from "./topic.js";
/** As specified in https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md */
const GOSSIPSUB_HEARTBEAT_INTERVAL = 0.7 * 1000;
const MAX_OUTBOUND_BUFFER_SIZE = 2 ** 24; // 16MB
export type Eth2Context = {
activeValidatorCount: number;
currentSlot: number;
currentEpoch: number;
};
export type Eth2GossipsubModules = {
networkConfig: NetworkConfig;
libp2p: Libp2p;
logger: Logger;
metricsRegister: RegistryMetricCreator | null;
eth2Context: Eth2Context;
peersData: PeersData;
events: NetworkEventBus;
};
export type Eth2GossipsubOpts = {
allowPublishToZeroPeers?: boolean;
gossipsubD?: number;
gossipsubDLow?: number;
gossipsubDHigh?: number;
gossipsubAwaitHandler?: boolean;
disableFloodPublish?: boolean;
skipParamsLog?: boolean;
disableLightClientServer?: boolean;
/**
* Direct peers for GossipSub - these peers maintain permanent mesh connections without GRAFT/PRUNE.
* Supports multiaddr strings with peer ID (e.g., "/ip4/192.168.1.1/tcp/9000/p2p/16Uiu2HAmKLhW7...")
* or ENR strings (e.g., "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOo...")
*/
directPeers?: string[];
};
export type ForkBoundaryLabel = string;
// Many of the internal properties we need are not available on the public interface,
// so we create an extended type here to avoid excessive type assertions throughout the codebase.
// Mind that any updates to the gossipsub package may require updates to this type.
type GossipSubInternal = GossipSub & {
mesh: Map<string, Set<string>>;
peers: Map<string, PeerId>;
score: {score: (peerIdStr: string) => number};
direct: Set<string>;
topics: Map<string, Set<string>>;
start: () => Promise<void>;
stop: () => Promise<void>;
publish: (topic: TopicStr, data: Uint8Array, opts?: PublishOpts) => Promise<PublishResult>;
getMeshPeers: (topic: TopicStr) => string[];
dumpPeerScoreStats: () => PeerScoreStatsDump;
getScore: (peerIdStr: string) => number;
reportMessageValidationResult: (msgId: string, propagationSource: string, acceptance: TopicValidatorResult) => void;
};
/**
* Wrapper around js-libp2p-gossipsub with the following extensions:
* - Eth2 message id
* - Emits `GossipObject`, not `InMessage`
* - Provides convenience interface:
* - `publishObject`
* - `subscribeTopic`
* - `unsubscribeTopic`
* - `handleTopic`
* - `unhandleTopic`
*
* See https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub
*/
export class Eth2Gossipsub {
readonly scoreParams: Partial<PeerScoreParams>;
private readonly config: BeaconConfig;
private readonly logger: Logger;
private readonly peersData: PeersData;
private readonly events: NetworkEventBus;
private readonly libp2p: Libp2p;
private readonly gossipsub: GossipSubInternal;
// Internal caches
private readonly gossipTopicCache: GossipTopicCache;
constructor(opts: Eth2GossipsubOpts, modules: Eth2GossipsubModules) {
const {allowPublishToZeroPeers, gossipsubD, gossipsubDLow, gossipsubDHigh} = opts;
const {networkConfig, logger, metricsRegister, peersData, events} = modules;
const {config} = networkConfig;
const gossipTopicCache = new GossipTopicCache(config);
const scoreParams = computeGossipPeerScoreParams({config, eth2Context: modules.eth2Context});
let metrics: Eth2GossipsubMetrics | null = null;
if (metricsRegister) {
metrics = createEth2GossipsubMetrics(metricsRegister);
}
// Parse direct peers from multiaddr strings to AddrInfo objects
const directPeers = parseDirectPeers(opts.directPeers ?? [], logger);
// Gossipsub parameters defined here:
// https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub
const gossipsubInstance = gossipsub({
globalSignaturePolicy: StrictNoSign,
allowPublishToZeroTopicPeers: allowPublishToZeroPeers,
D: gossipsubD ?? GOSSIP_D,
Dlo: gossipsubDLow ?? GOSSIP_D_LOW,
Dhi: gossipsubDHigh ?? GOSSIP_D_HIGH,
Dlazy: 6,
directPeers,
heartbeatInterval: GOSSIPSUB_HEARTBEAT_INTERVAL,
fanoutTTL: 60 * 1000,
mcacheLength: 6,
mcacheGossip: 3,
// this should be in ms
seenTTL: config.SLOT_DURATION_MS * SLOTS_PER_EPOCH * 2,
scoreParams,
scoreThresholds: gossipScoreThresholds,
// For a single stream, await processing each RPC before processing the next
awaitRpcHandler: opts.gossipsubAwaitHandler,
// For a single RPC, await processing each message before processing the next
awaitRpcMessageHandler: opts.gossipsubAwaitHandler,
// the default in gossipsub is 3s is not enough since lodestar suffers from I/O lag
gossipsubIWantFollowupMs: 12 * 1000, // 12s
fastMsgIdFn: fastMsgIdFn,
msgIdFn: msgIdFn.bind(msgIdFn, gossipTopicCache),
msgIdToStrFn: msgIdToStrFn,
dataTransform: new DataTransformSnappy(gossipTopicCache, config.MAX_PAYLOAD_SIZE, metrics),
metricsRegister: metricsRegister as MetricsRegister | null,
metricsTopicStrToLabel: metricsRegister
? getMetricsTopicStrToLabel(networkConfig, {disableLightClientServer: opts.disableLightClientServer ?? false})
: undefined,
asyncValidation: true,
maxOutboundBufferSize: MAX_OUTBOUND_BUFFER_SIZE,
// serialize message once and send to all peers when publishing
batchPublish: true,
// if this is false, only publish to mesh peers. If there is not enough GOSSIP_D mesh peers,
// publish to some more topic peers to make sure we always publish to at least GOSSIP_D peers
floodPublish: !opts?.disableFloodPublish,
// Only send IDONTWANT messages if the message size is larger than this
// This should be large enough to not send IDONTWANT for "small" messages
// See https://github.com/ChainSafe/lodestar/pull/7077#issuecomment-2383679472
idontwantMinDataSize: 16829,
})(modules.libp2p.services.components) as GossipSubInternal;
if (metrics) {
metrics.gossipMesh.peersByType.addCollect(() => this.onScrapeLodestarMetrics(metrics, networkConfig));
}
this.gossipsub = gossipsubInstance;
this.scoreParams = scoreParams;
this.config = config;
this.logger = logger;
this.peersData = peersData;
this.events = events;
this.libp2p = modules.libp2p;
this.gossipTopicCache = gossipTopicCache;
this.gossipsub.addEventListener("gossipsub:message", this.onGossipsubMessage.bind(this));
this.events.on(NetworkEvent.gossipMessageValidationResult, this.onValidationResult.bind(this));
// Having access to this data is CRUCIAL for debugging. While this is a massive log, it must not be deleted.
// Scoring issues require this dump + current peer score stats to re-calculate scores.
if (!opts.skipParamsLog) {
this.logger.debug("Gossipsub score params", {params: JSON.stringify(scoreParams)});
}
}
async start(): Promise<void> {
await this.gossipsub.start();
}
async stop(): Promise<void> {
await this.gossipsub.stop();
}
get mesh(): Map<string, Set<string>> {
return this.gossipsub.mesh;
}
getTopics(): TopicStr[] {
return this.gossipsub.getTopics();
}
getMeshPeers(topic: TopicStr): string[] {
return this.gossipsub.getMeshPeers(topic);
}
publish(topic: TopicStr, data: Uint8Array, opts?: PublishOpts): Promise<PublishResult> {
return this.gossipsub.publish(topic, data, opts);
}
dumpPeerScoreStats(): PeerScoreStatsDump {
return this.gossipsub.dumpPeerScoreStats();
}
getScore(peerIdStr: string): number {
return this.gossipsub.getScore(peerIdStr);
}
/**
* Subscribe to a `GossipTopic`
*/
subscribeTopic(topic: GossipTopic): void {
const topicStr = stringifyGossipTopic(this.config, topic);
// Register known topicStr
this.gossipTopicCache.setTopic(topicStr, topic);
this.logger.verbose("Subscribe to gossipsub topic", {topic: topicStr});
this.gossipsub.subscribe(topicStr);
}
/**
* Unsubscribe to a `GossipTopic`
*/
unsubscribeTopic(topic: GossipTopic): void {
const topicStr = stringifyGossipTopic(this.config, topic);
this.logger.verbose("Unsubscribe to gossipsub topic", {topic: topicStr});
this.gossipsub.unsubscribe(topicStr);
}
private onScrapeLodestarMetrics(metrics: Eth2GossipsubMetrics, networkConfig: NetworkConfig): void {
const mesh = this.gossipsub.mesh;
const topics = this.gossipsub.topics;
const peers = this.gossipsub.peers;
const score = this.gossipsub.score;
const meshPeersByClient = new Map<string, number>();
const meshPeerIdStrs = new Set<string>();
for (const {peersMap, metricsGossip, type} of [
{peersMap: mesh, metricsGossip: metrics.gossipMesh, type: "mesh"},
{peersMap: topics, metricsGossip: metrics.gossipTopic, type: "topics"},
]) {
// Pre-aggregate results by fork so we can fill the remaining metrics with 0
const peersByTypeByBoundary = new Map2d<ForkBoundaryLabel, GossipType, number>();
const peersByBeaconAttSubnetByBoundary = new Map2dArr<ForkBoundaryLabel, number>();
const peersByBeaconSyncSubnetByBoundary = new Map2dArr<ForkBoundaryLabel, number>();
const peersByDataColumnSubnetByBoundary = new Map2dArr<ForkBoundaryLabel, number>();
// loop through all mesh entries, count each set size
for (const [topicString, peers] of peersMap) {
// Ignore topics with 0 peers. May prevent overriding after a fork
if (peers.size === 0) continue;
// there are some new topics in the network so `getKnownTopic()` returns undefined
// for example in prater: /eth2/82f4a72b/optimistic_light_client_update_v0/ssz_snappy
const topic = this.gossipTopicCache.getKnownTopic(topicString);
if (topic !== undefined) {
const boundary = getForkBoundaryLabel(topic.boundary);
if (topic.type === GossipType.beacon_attestation) {
peersByBeaconAttSubnetByBoundary.set(boundary, topic.subnet, peers.size);
} else if (topic.type === GossipType.sync_committee) {
peersByBeaconSyncSubnetByBoundary.set(boundary, topic.subnet, peers.size);
} else if (topic.type === GossipType.data_column_sidecar) {
peersByDataColumnSubnetByBoundary.set(boundary, topic.subnet, peers.size);
} else {
peersByTypeByBoundary.set(boundary, topic.type, peers.size);
}
}
if (type === "mesh") {
for (const peer of peers) {
if (!meshPeerIdStrs.has(peer)) {
meshPeerIdStrs.add(peer);
const client = this.peersData.connectedPeers.get(peer)?.agentClient?.toString() ?? ClientKind.Unknown;
meshPeersByClient.set(client, (meshPeersByClient.get(client) ?? 0) + 1);
}
}
}
}
// beacon attestation mesh gets counted separately so we can track mesh peers by subnet
// zero out all gossip type & subnet choices, so the dashboard will register them
for (const [boundary, peersByType] of peersByTypeByBoundary.map) {
for (const type of Object.values(GossipType)) {
metricsGossip.peersByType.set({boundary, type}, peersByType.get(type) ?? 0);
}
}
for (const [boundary, peersByBeaconAttSubnet] of peersByBeaconAttSubnetByBoundary.map) {
for (let subnet = 0; subnet < ATTESTATION_SUBNET_COUNT; subnet++) {
metricsGossip.peersByBeaconAttestationSubnet.set(
{boundary, subnet: attSubnetLabel(subnet)},
peersByBeaconAttSubnet[subnet] ?? 0
);
}
}
for (const [boundary, peersByBeaconSyncSubnet] of peersByBeaconSyncSubnetByBoundary.map) {
for (let subnet = 0; subnet < SYNC_COMMITTEE_SUBNET_COUNT; subnet++) {
// SYNC_COMMITTEE_SUBNET_COUNT is < 9, no need to prepend a 0 to the label
metricsGossip.peersBySyncCommitteeSubnet.set({boundary, subnet}, peersByBeaconSyncSubnet[subnet] ?? 0);
}
}
for (const [boundary, peersByDataColumnSubnet] of peersByDataColumnSubnetByBoundary.map) {
for (const subnet of networkConfig.custodyConfig.sampleGroups) {
metricsGossip.peersByDataColumnSubnet.set({boundary, subnet}, peersByDataColumnSubnet[subnet] ?? 0);
}
}
}
for (const [client, peers] of meshPeersByClient.entries()) {
metrics.gossipPeer.meshPeersByClient.set({client}, peers);
}
// track gossip peer score
let peerCountScoreGraylist = 0;
let peerCountScorePublish = 0;
let peerCountScoreGossip = 0;
let peerCountScoreMesh = 0;
const {graylistThreshold, publishThreshold, gossipThreshold} = gossipScoreThresholds;
const gossipScores: number[] = [];
for (const peerIdStr of peers.keys()) {
const s = score.score(peerIdStr);
if (s >= graylistThreshold) peerCountScoreGraylist++;
if (s >= publishThreshold) peerCountScorePublish++;
if (s >= gossipThreshold) peerCountScoreGossip++;
if (s >= 0) peerCountScoreMesh++;
gossipScores.push(s);
}
// Access once for all calls below
metrics.gossipPeer.scoreByThreshold.set({threshold: "graylist"}, peerCountScoreGraylist);
metrics.gossipPeer.scoreByThreshold.set({threshold: "publish"}, peerCountScorePublish);
metrics.gossipPeer.scoreByThreshold.set({threshold: "gossip"}, peerCountScoreGossip);
metrics.gossipPeer.scoreByThreshold.set({threshold: "mesh"}, peerCountScoreMesh);
// Register full score too
metrics.gossipPeer.score.set(gossipScores);
}
private onGossipsubMessage(event: GossipSubEvents["gossipsub:message"]): void {
const {propagationSource, msgId, msg} = event.detail;
// Also validates that the topicStr is known
const topic = this.gossipTopicCache.getTopic(msg.topic);
// Get seenTimestamp before adding the message to the queue or add async delays
const seenTimestampSec = Date.now() / 1000;
const peerIdStr = propagationSource.toString();
const clientAgent = this.peersData.getPeerKind(peerIdStr) ?? "Unknown";
const clientVersion = this.peersData.getAgentVersion(peerIdStr);
// Use setTimeout to yield to the macro queue
// Without this we'll have huge event loop lag
// See https://github.com/ChainSafe/lodestar/issues/5604
callInNextEventLoop(() => {
this.events.emit(NetworkEvent.pendingGossipsubMessage, {
topic,
msg,
msgId,
// Hot path, use cached .toString() version
propagationSource: peerIdStr,
clientVersion,
clientAgent,
seenTimestampSec,
startProcessUnixSec: null,
});
});
}
private onValidationResult(data: NetworkEventData[NetworkEvent.gossipMessageValidationResult]): void {
// Use setTimeout to yield to the macro queue
// Without this we'll have huge event loop lag
// See https://github.com/ChainSafe/lodestar/issues/5604
callInNextEventLoop(() => {
this.gossipsub.reportMessageValidationResult(data.msgId, data.propagationSource, data.acceptance);
});
}
/**
* Add a peer as a direct peer at runtime. Accepts multiaddr with peer ID or ENR string.
* Direct peers maintain permanent mesh connections without GRAFT/PRUNE negotiation.
*/
async addDirectPeer(peerStr: routes.lodestar.DirectPeer): Promise<string | null> {
const parsed = parseDirectPeers([peerStr], this.logger);
if (parsed.length === 0) {
return null;
}
const {id: peerId, addrs} = parsed[0];
const peerIdStr = peerId.toString();
// Prevent adding self as a direct peer
if (peerId.equals(this.libp2p.peerId)) {
this.logger.warn("Cannot add self as a direct peer", {peerId: peerIdStr});
return null;
}
// Direct peers need addresses to connect - reject if none provided
if (addrs.length === 0) {
this.logger.warn("Cannot add direct peer without addresses", {peerId: peerIdStr});
return null;
}
// Add addresses to peer store first so we can connect
try {
await this.libp2p.peerStore.merge(peerId, {multiaddrs: addrs});
} catch (e) {
this.logger.warn("Failed to add direct peer addresses to peer store", {peerId: peerIdStr}, e as Error);
return null;
}
// Add to direct peers set only after addresses are stored
this.gossipsub.direct.add(peerIdStr);
this.logger.info("Added direct peer via API", {peerId: peerIdStr});
return peerIdStr;
}
/**
* Remove a peer from direct peers.
*/
removeDirectPeer(peerIdStr: string): boolean {
const removed = this.gossipsub.direct.delete(peerIdStr);
if (removed) {
this.logger.info("Removed direct peer via API", {peerId: peerIdStr});
}
return removed;
}
/**
* Get list of current direct peer IDs.
*/
getDirectPeers(): string[] {
return Array.from(this.gossipsub.direct);
}
}
/**
* Left pad subnets to two characters. Assumes ATTESTATION_SUBNET_COUNT < 99
* Otherwise grafana sorts the mesh peers chart as: [1,11,12,13,...]
*/
function attSubnetLabel(subnet: SubnetID): string {
if (subnet > 9) return String(subnet);
return `0${subnet}`;
}
function getMetricsTopicStrToLabel(
networkConfig: NetworkConfig,
opts: {disableLightClientServer: boolean}
): TopicStrToLabel {
const {config} = networkConfig;
const metricsTopicStrToLabel = new Map<TopicStr, TopicLabel>();
const {forkBoundariesAscendingEpochOrder} = config;
for (let i = 0; i < forkBoundariesAscendingEpochOrder.length; i++) {
const currentForkBoundary = forkBoundariesAscendingEpochOrder[i];
const nextForkBoundary = forkBoundariesAscendingEpochOrder[i + 1];
// Edge case: If multiple fork boundaries start at the same epoch, only consider the latest one
if (nextForkBoundary && currentForkBoundary.epoch === nextForkBoundary.epoch) {
continue;
}
const topics = getCoreTopicsAtFork(networkConfig, currentForkBoundary.fork, {
subscribeAllSubnets: true,
disableLightClientServer: opts.disableLightClientServer,
});
for (const topic of topics) {
metricsTopicStrToLabel.set(stringifyGossipTopic(config, {...topic, boundary: currentForkBoundary}), topic.type);
}
}
return metricsTopicStrToLabel;
}
// Topics of the same ForkBoundary should have the same ForkBoundary object
// we don't want to create a new string for every topic
const boundaryLabelMap = new Map<ForkBoundary, ForkBoundaryLabel>();
function getForkBoundaryLabel(boundary: ForkBoundary): ForkBoundaryLabel {
let label = boundaryLabelMap.get(boundary);
if (label === undefined) {
label = `${boundary.fork}_${boundary.epoch}`;
boundaryLabelMap.set(boundary, label);
}
return label;
}
/**
* Parse direct peer strings into AddrInfo objects for GossipSub.
* Direct peers maintain permanent mesh connections without GRAFT/PRUNE negotiation.
*
* Supported formats:
* - Multiaddr with peer ID: `/ip4/192.168.1.1/tcp/9000/p2p/16Uiu2HAmKLhW7...`
* - ENR: `enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOo...`
*
* For multiaddrs, the string must contain a /p2p/ component with the peer ID.
* For ENRs, the TCP multiaddr and peer ID are extracted from the encoded record.
*/
export function parseDirectPeers(directPeerStrs: routes.lodestar.DirectPeer[], logger: Logger): AddrInfo[] {
const directPeers: AddrInfo[] = [];
for (const peerStr of directPeerStrs) {
// Check if this is an ENR (starts with "enr:")
if (peerStr.startsWith("enr:")) {
try {
const enr = ENR.decodeTxt(peerStr);
const peerId = enr.peerId;
// Get all available transport multiaddrs from ENR
const addrs = [enr.getLocationMultiaddr("quic"), enr.getLocationMultiaddr("tcp")].filter(
(a): a is Multiaddr => a != null
);
if (addrs.length === 0) {
logger.warn("ENR does not contain any transport multiaddr", {enr: peerStr});
continue;
}
directPeers.push({
id: peerId,
addrs,
});
logger.info("Added direct peer from ENR", {
peerId: peerId.toString(),
addrs: addrs.map((a) => a.toString()).join(", "),
});
} catch (e) {
logger.warn("Failed to parse direct peer ENR", {enr: peerStr}, e as Error);
}
} else {
// Parse as multiaddr
try {
const ma = multiaddr(peerStr);
const peerIdComponent = ma.getComponents().findLast((component) => component.name === "p2p");
const peerIdStr = peerIdComponent?.value;
if (!peerIdStr) {
logger.warn("Direct peer multiaddr must contain /p2p/ component with peer ID", {multiaddr: peerStr});
continue;
}
try {
const peerId = peerIdFromString(peerIdStr);
// Get the address without the /p2p/ component
const addr = ma.decapsulate("/p2p/" + peerIdStr);
directPeers.push({
id: peerId,
addrs: [addr],
});
logger.info("Added direct peer", {peerId: peerIdStr, addr: addr.toString()});
} catch (e) {
logger.warn("Invalid peer ID in direct peer multiaddr", {multiaddr: peerStr, peerId: peerIdStr}, e as Error);
}
} catch (e) {
logger.warn("Failed to parse direct peer multiaddr", {multiaddr: peerStr}, e as Error);
}
}
}
return directPeers;
}