@lodestar/beacon-node
Version:
A Typescript implementation of the beacon chain
924 lines (923 loc) • 51.1 kB
JavaScript
import { ForkSeq, INTERVALS_PER_SLOT, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH } from "@lodestar/params";
import { computeEpochAtSlot, computeStartSlotAtEpoch, getBlockRootAtSlot, getCurrentSlot, parseAttesterFlags, parseParticipationFlags, } from "@lodestar/state-transition";
import { LogLevel, MapDef, MapDefMax, toRootHex } from "@lodestar/utils";
import { GENESIS_SLOT } from "../constants/constants.js";
/** The validator monitor collects per-epoch data about each monitored validator.
* Historical data will be kept around for `HISTORIC_EPOCHS` before it is pruned.
*/
const MAX_CACHED_EPOCHS = 4;
const MAX_CACHED_DISTINCT_TARGETS = 4;
const INTERVALS_LATE_ATTESTATION_SUBMISSION = 1.5;
const INTERVALS_LATE_BLOCK_SUBMISSION = 0.75;
const RETAIN_REGISTERED_VALIDATORS_MS = 1 * 3600 * 1000; // 1 hour
export var OpSource;
(function (OpSource) {
OpSource["api"] = "api";
OpSource["gossip"] = "gossip";
})(OpSource || (OpSource = {}));
export const defaultValidatorMonitorOpts = {
validatorMonitorLogs: false,
};
function statusToSummary(inclusionDelay, flag, isActiveInCurrentEpoch, isActiveInPreviousEpoch) {
const flags = parseAttesterFlags(flag);
return {
isSlashed: flags.unslashed,
isActiveInCurrentEpoch,
isActiveInPreviousEpoch,
// TODO: Implement
currentEpochEffectiveBalance: 0,
isPrevSourceAttester: flags.prevSourceAttester,
isPrevTargetAttester: flags.prevTargetAttester,
isPrevHeadAttester: flags.prevHeadAttester,
isCurrSourceAttester: flags.currSourceAttester,
isCurrTargetAttester: flags.currTargetAttester,
isCurrHeadAttester: flags.currHeadAttester,
inclusionDistance: inclusionDelay,
};
}
function getEpochSummary(validator, epoch) {
let summary = validator.summaries.get(epoch);
if (!summary) {
summary = {
attestations: 0,
attestationMinDelay: null,
attestationAggregateInclusions: 0,
attestationBlockInclusions: 0,
attestationMinBlockInclusionDistance: null,
blocks: 0,
blockMinDelay: null,
aggregates: 0,
aggregateMinDelay: null,
attestationCorrectHead: null,
syncCommitteeHits: 0,
syncCommitteeMisses: 0,
syncSignatureAggregateInclusions: 0,
blockProposals: [],
};
validator.summaries.set(epoch, summary);
}
// Prune
const toPrune = validator.summaries.size - MAX_CACHED_EPOCHS;
if (toPrune > 0) {
let pruned = 0;
for (const idx of validator.summaries.keys()) {
validator.summaries.delete(idx);
if (++pruned >= toPrune)
break;
}
}
return summary;
}
export function createValidatorMonitor(metricsRegister, config, genesisTime, logger, opts) {
const logLevel = opts.validatorMonitorLogs ? LogLevel.info : LogLevel.debug;
const log = (message, context) => {
logger[logLevel](message, context);
};
/** The validators that require additional monitoring. */
const validators = new MapDef(() => ({
summaries: new Map(),
inSyncCommitteeUntilEpoch: -1,
attestations: new MapDefMax(() => new MapDefMax(() => ({
poolSubmitDelayMinSec: null,
poolSubmitSentPeers: null,
aggregateInclusionDelaysSec: [],
blockInclusions: [],
}), MAX_CACHED_DISTINCT_TARGETS), MAX_CACHED_EPOCHS),
lastRegisteredTimeMs: 0,
}));
let lastRegisteredStatusEpoch = -1;
const validatorMonitorMetrics = metricsRegister ? createValidatorMonitorMetrics(metricsRegister) : null;
const validatorMonitor = {
registerLocalValidator(index) {
validators.getOrDefault(index).lastRegisteredTimeMs = Date.now();
},
registerLocalValidatorInSyncCommittee(index, untilEpoch) {
const validator = validators.get(index);
if (validator) {
validator.inSyncCommitteeUntilEpoch = Math.max(untilEpoch, validator.inSyncCommitteeUntilEpoch ?? -1);
}
},
registerValidatorStatuses(currentEpoch, inclusionDelays, flags, isActiveCurrEpoch, isActiveInPrevEpoch, balances) {
// Prevent registering status for the same epoch twice. processEpoch() may be ran more than once for the same epoch.
if (currentEpoch <= lastRegisteredStatusEpoch) {
return;
}
lastRegisteredStatusEpoch = currentEpoch;
const previousEpoch = currentEpoch - 1;
// There won't be any validator activity in epoch -1
if (previousEpoch === -1) {
return;
}
for (const [index, monitoredValidator] of validators.entries()) {
// We subtract two from the state of the epoch that generated these summaries.
//
// - One to account for it being the previous epoch.
// - One to account for the state advancing an epoch whilst generating the validator
// statuses.
const summary = statusToSummary(inclusionDelays[index], flags[index], isActiveCurrEpoch[index], isActiveInPrevEpoch[index]);
if (summary.isPrevSourceAttester) {
validatorMonitorMetrics?.prevEpochOnChainSourceAttesterHit.inc();
}
else {
validatorMonitorMetrics?.prevEpochOnChainSourceAttesterMiss.inc();
}
if (summary.isPrevHeadAttester) {
validatorMonitorMetrics?.prevEpochOnChainHeadAttesterHit.inc();
}
else {
validatorMonitorMetrics?.prevEpochOnChainHeadAttesterMiss.inc();
}
if (summary.isPrevTargetAttester) {
validatorMonitorMetrics?.prevEpochOnChainTargetAttesterHit.inc();
}
else {
validatorMonitorMetrics?.prevEpochOnChainTargetAttesterMiss.inc();
}
const prevEpochSummary = monitoredValidator.summaries.get(previousEpoch);
const attestationCorrectHead = prevEpochSummary?.attestationCorrectHead;
if (attestationCorrectHead !== null && attestationCorrectHead !== undefined) {
if (attestationCorrectHead) {
validatorMonitorMetrics?.prevOnChainAttesterCorrectHead.inc();
}
else {
validatorMonitorMetrics?.prevOnChainAttesterIncorrectHead.inc();
}
}
const attestationMinBlockInclusionDistance = prevEpochSummary?.attestationMinBlockInclusionDistance;
const inclusionDistance = attestationMinBlockInclusionDistance != null && attestationMinBlockInclusionDistance > 0
? // altair, attestation is not missed
attestationMinBlockInclusionDistance
: summary.inclusionDistance
? // phase0, this is from the state transition
summary.inclusionDistance
: null;
if (inclusionDistance !== null) {
validatorMonitorMetrics?.prevEpochOnChainInclusionDistance.observe(inclusionDistance);
validatorMonitorMetrics?.prevEpochOnChainAttesterHit.inc();
}
else {
validatorMonitorMetrics?.prevEpochOnChainAttesterMiss.inc();
}
const balance = balances?.[index];
if (balance !== undefined) {
validatorMonitorMetrics?.prevEpochOnChainBalance.set({ index }, balance);
}
if (!summary.isPrevSourceAttester || !summary.isPrevTargetAttester || !summary.isPrevHeadAttester) {
log("Failed attestation in previous epoch", {
validator: index,
prevEpoch: currentEpoch - 1,
isPrevSourceAttester: summary.isPrevSourceAttester,
isPrevHeadAttester: summary.isPrevHeadAttester,
isPrevTargetAttester: summary.isPrevTargetAttester,
// inclusionDistance is not available in summary since altair
inclusionDistance,
});
}
}
},
registerBeaconBlock(src, delaySec, block) {
const validator = validators.get(block.proposerIndex);
// Returns the delay between the start of `block.slot` and `seenTimestamp`.
if (validator) {
validatorMonitorMetrics?.beaconBlockTotal.inc({ src });
validatorMonitorMetrics?.beaconBlockDelaySeconds.observe({ src }, delaySec);
const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot));
summary.blockProposals.push({
blockRoot: toRootHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)),
blockSlot: block.slot,
poolSubmitDelaySec: delaySec,
successfullyImported: false,
});
}
},
registerBlobSidecar(_src, _seenTimestampSec, _blob) {
//TODO: freetheblobs
},
registerImportedBlock(block, { proposerBalanceDelta }) {
const validator = validators.get(block.proposerIndex);
if (validator) {
validatorMonitorMetrics?.proposerBalanceDeltaKnown.observe(proposerBalanceDelta);
// There should be alredy a summary for the block. Could be missing when using one VC multiple BNs
const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot));
const proposal = summary.blockProposals.find((p) => p.blockSlot === block.slot);
if (proposal) {
proposal.successfullyImported = true;
}
else {
summary.blockProposals.push({
blockRoot: toRootHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)),
blockSlot: block.slot,
poolSubmitDelaySec: null,
successfullyImported: true,
});
}
}
},
onPoolSubmitUnaggregatedAttestation(seenTimestampSec, indexedAttestation, subnet, sentPeers) {
const data = indexedAttestation.data;
// Returns the duration between when the attestation `data` could be produced (1/3rd through the slot) and `seenTimestamp`.
const delaySec = seenTimestampSec - (genesisTime + (data.slot + 1 / 3) * config.SECONDS_PER_SLOT);
for (const index of indexedAttestation.attestingIndices) {
const validator = validators.get(index);
if (validator) {
validatorMonitorMetrics?.unaggregatedAttestationSubmittedSentPeers.observe(sentPeers);
validatorMonitorMetrics?.unaggregatedAttestationDelaySeconds.observe({ src: OpSource.api }, delaySec);
log("Published unaggregated attestation", {
validator: index,
slot: data.slot,
committeeIndex: data.index,
subnet,
sentPeers,
delaySec: delaySec.toFixed(4),
});
const attestationSummary = validator.attestations
.getOrDefault(indexedAttestation.data.target.epoch)
.getOrDefault(toRootHex(indexedAttestation.data.target.root));
if (attestationSummary.poolSubmitDelayMinSec === null ||
attestationSummary.poolSubmitDelayMinSec > delaySec) {
attestationSummary.poolSubmitDelayMinSec = delaySec;
}
}
}
},
registerGossipUnaggregatedAttestation(seenTimestampSec, indexedAttestation) {
const src = OpSource.gossip;
const data = indexedAttestation.data;
const epoch = computeEpochAtSlot(data.slot);
// Returns the duration between when the attestation `data` could be produced (1/3rd through the slot) and `seenTimestamp`.
const delaySec = seenTimestampSec - (genesisTime + (data.slot + 1 / 3) * config.SECONDS_PER_SLOT);
for (const index of indexedAttestation.attestingIndices) {
const validator = validators.get(index);
if (validator) {
validatorMonitorMetrics?.unaggregatedAttestationTotal.inc({ src });
validatorMonitorMetrics?.unaggregatedAttestationDelaySeconds.observe({ src }, delaySec);
const summary = getEpochSummary(validator, epoch);
summary.attestations += 1;
summary.attestationMinDelay = Math.min(delaySec, summary.attestationMinDelay ?? Infinity);
}
}
},
onPoolSubmitAggregatedAttestation(seenTimestampSec, indexedAttestation, sentPeers) {
const data = indexedAttestation.data;
// Returns the duration between when a `AggregateAndproof` with `data` could be produced (2/3rd through the slot) and `seenTimestamp`.
const delaySec = seenTimestampSec - (genesisTime + (data.slot + 2 / 3) * config.SECONDS_PER_SLOT);
for (const index of indexedAttestation.attestingIndices) {
const validator = validators.get(index);
if (validator) {
validatorMonitorMetrics?.aggregatedAttestationDelaySeconds.observe({ src: OpSource.api }, delaySec);
log("Published aggregated attestation", {
validator: index,
slot: data.slot,
committeeIndex: data.index,
sentPeers,
delaySec: delaySec.toFixed(4),
});
validator.attestations
.getOrDefault(indexedAttestation.data.target.epoch)
.getOrDefault(toRootHex(indexedAttestation.data.target.root))
.aggregateInclusionDelaysSec.push(delaySec);
}
}
},
registerGossipAggregatedAttestation(seenTimestampSec, signedAggregateAndProof, indexedAttestation) {
const src = OpSource.gossip;
const data = indexedAttestation.data;
const epoch = computeEpochAtSlot(data.slot);
// Returns the duration between when a `AggregateAndProof` with `data` could be produced (2/3rd through the slot) and `seenTimestamp`.
const delaySec = seenTimestampSec - (genesisTime + (data.slot + 2 / 3) * config.SECONDS_PER_SLOT);
const aggregatorIndex = signedAggregateAndProof.message.aggregatorIndex;
const validatorAggregator = validators.get(aggregatorIndex);
if (validatorAggregator) {
validatorMonitorMetrics?.aggregatedAttestationTotal.inc({ src });
validatorMonitorMetrics?.aggregatedAttestationDelaySeconds.observe({ src }, delaySec);
const summary = getEpochSummary(validatorAggregator, epoch);
summary.aggregates += 1;
summary.aggregateMinDelay = Math.min(delaySec, summary.aggregateMinDelay ?? Infinity);
}
for (const index of indexedAttestation.attestingIndices) {
const validator = validators.get(index);
if (validator) {
validatorMonitorMetrics?.attestationInAggregateTotal.inc({ src });
validatorMonitorMetrics?.attestationInAggregateDelaySeconds.observe({ src }, delaySec);
const summary = getEpochSummary(validator, epoch);
summary.attestationAggregateInclusions += 1;
log("Attestation is included in aggregate", {
validator: index,
slot: data.slot,
committeeIndex: data.index,
aggregatorIndex,
});
validator.attestations
.getOrDefault(indexedAttestation.data.target.epoch)
.getOrDefault(toRootHex(indexedAttestation.data.target.root))
.aggregateInclusionDelaysSec.push(delaySec);
}
}
},
// Register that the `indexed_attestation` was included in a *valid* `BeaconBlock`.
registerAttestationInBlock(indexedAttestation, parentSlot, correctHead, missedSlotVote, inclusionBlockRoot, inclusionBlockSlot) {
const data = indexedAttestation.data;
// optimal inclusion distance, not to count skipped slots between data.slot and blockSlot
const inclusionDistance = Math.max(parentSlot - data.slot, 0) + 1;
const delay = inclusionDistance - MIN_ATTESTATION_INCLUSION_DELAY;
const epoch = computeEpochAtSlot(data.slot);
const participants = indexedAttestation.attestingIndices.length;
for (const index of indexedAttestation.attestingIndices) {
const validator = validators.get(index);
if (validator) {
validatorMonitorMetrics?.attestationInBlockTotal.inc();
validatorMonitorMetrics?.attestationInBlockDelaySlots.observe(delay);
validatorMonitorMetrics?.attestationInBlockParticipants.observe(participants);
const summary = getEpochSummary(validator, epoch);
summary.attestationBlockInclusions += 1;
if (summary.attestationMinBlockInclusionDistance !== null) {
summary.attestationMinBlockInclusionDistance = Math.min(summary.attestationMinBlockInclusionDistance, inclusionDistance);
}
else {
summary.attestationMinBlockInclusionDistance = inclusionDistance;
}
summary.attestationCorrectHead = correctHead;
validator.attestations
.getOrDefault(indexedAttestation.data.target.epoch)
.getOrDefault(toRootHex(indexedAttestation.data.target.root))
.blockInclusions.push({
blockRoot: inclusionBlockRoot,
blockSlot: inclusionBlockSlot,
votedCorrectHeadRoot: correctHead,
votedForMissedSlot: missedSlotVote,
attestationSlot: indexedAttestation.data.slot,
});
log("Attestation is included in block", {
validator: index,
slot: data.slot,
committeeIndex: data.index,
inclusionDistance,
correctHead,
participants,
});
}
}
},
registerGossipSyncContributionAndProof(syncContributionAndProof, syncCommitteeParticipantIndices) {
const epoch = computeEpochAtSlot(syncContributionAndProof.contribution.slot);
for (const index of syncCommitteeParticipantIndices) {
const validator = validators.get(index);
if (validator) {
validatorMonitorMetrics?.syncSignatureInAggregateTotal.inc();
const summary = getEpochSummary(validator, epoch);
summary.syncSignatureAggregateInclusions += 1;
}
}
},
registerSyncAggregateInBlock(epoch, syncAggregate, syncCommitteeIndices) {
for (let i = 0; i < syncCommitteeIndices.length; i++) {
const validator = validators.get(syncCommitteeIndices[i]);
if (validator) {
const summary = getEpochSummary(validator, epoch);
if (syncAggregate.syncCommitteeBits.get(i)) {
summary.syncCommitteeHits++;
}
else {
summary.syncCommitteeMisses++;
}
}
}
},
// Validator monitor tracks performance of validators in healthy network conditions.
// It does not attempt to track correctly duties on forking conditions deeper than 1 epoch.
// To guard against short re-orgs it will track the status of epoch N at the end of epoch N+1.
// This function **SHOULD** be called at the last slot of an epoch to have max possible information.
onceEveryEndOfEpoch(headState) {
if (headState.slot <= GENESIS_SLOT) {
// Before genesis, there won't be any validator activity
return;
}
// Prune validators not seen in a while
for (const [index, validator] of validators.entries()) {
if (Date.now() - validator.lastRegisteredTimeMs > RETAIN_REGISTERED_VALIDATORS_MS) {
validators.delete(index);
}
}
// Compute summaries of previous epoch attestation performance
const prevEpoch = computeEpochAtSlot(headState.slot) - 1;
// During the end of first epoch, the prev epoch with be -1
// Skip this as there is no attestation and block proposal summary in epoch -1
if (prevEpoch === -1) {
return;
}
const rootCache = new RootHexCache(headState);
if (config.getForkSeq(headState.slot) >= ForkSeq.altair) {
const { previousEpochParticipation } = headState;
const prevEpochStartSlot = computeStartSlotAtEpoch(prevEpoch);
const prevEpochTargetRoot = toRootHex(getBlockRootAtSlot(headState, prevEpochStartSlot));
// Check attestation performance
for (const [index, validator] of validators.entries()) {
const flags = parseParticipationFlags(previousEpochParticipation.get(index));
const attestationSummary = validator.attestations.get(prevEpoch)?.get(prevEpochTargetRoot);
const summary = renderAttestationSummary(config, rootCache, attestationSummary, flags);
validatorMonitorMetrics?.prevEpochAttestationSummary.inc({ summary });
log("Previous epoch attestation", {
validator: index,
epoch: prevEpoch,
summary,
});
}
}
if (headState.epochCtx.proposersPrevEpoch !== null) {
// proposersPrevEpoch is null on the first epoch of `headState` being generated
for (const [slotIndex, validatorIndex] of headState.epochCtx.proposersPrevEpoch.entries()) {
const validator = validators.get(validatorIndex);
if (validator) {
// If expected proposer is a tracked validator
const epochSummary = validator.summaries.get(prevEpoch);
const proposalSlot = SLOTS_PER_EPOCH * prevEpoch + slotIndex;
const summary = renderBlockProposalSummary(config, rootCache, epochSummary, proposalSlot);
validatorMonitorMetrics?.prevEpochBlockProposalSummary.inc({ summary });
log("Previous epoch block proposal", {
validator: validatorIndex,
slot: proposalSlot,
epoch: prevEpoch,
summary,
});
}
}
}
},
/**
* Scrape `self` for metrics.
* Should be called whenever Prometheus is scraping.
*/
scrapeMetrics(slotClock) {
validatorMonitorMetrics?.validatorsConnected.set(validators.size);
const epoch = computeEpochAtSlot(slotClock);
const slotInEpoch = slotClock % SLOTS_PER_EPOCH;
// Only start to report on the current epoch once we've progressed past the point where
// all attestation should be included in a block.
//
// This allows us to set alarms on Grafana to detect when an attestation has been
// missed. If we didn't delay beyond the attestation inclusion period then we could
// expect some occasional false-positives on attestation misses.
//
// I have chosen 3 as an arbitrary number where we *probably* shouldn't see that many
// skip slots on mainnet.
const previousEpoch = slotInEpoch > MIN_ATTESTATION_INCLUSION_DELAY + 3 ? epoch - 1 : epoch - 2;
// reset() to mimic the behaviour of an aggregated .set({index})
validatorMonitorMetrics?.prevEpochAttestations.reset();
validatorMonitorMetrics?.prevEpochAttestationsMinDelaySeconds.reset();
validatorMonitorMetrics?.prevEpochAttestationAggregateInclusions.reset();
validatorMonitorMetrics?.prevEpochAttestationBlockInclusions.reset();
validatorMonitorMetrics?.prevEpochAttestationBlockMinInclusionDistance.reset();
validatorMonitorMetrics?.prevEpochSyncSignatureAggregateInclusions.reset();
let validatorsInSyncCommittee = 0;
let prevEpochSyncCommitteeHits = 0;
let prevEpochSyncCommitteeMisses = 0;
for (const validator of validators.values()) {
// Participation in sync committee
const validatorInSyncCommittee = validator.inSyncCommitteeUntilEpoch >= epoch;
if (validatorInSyncCommittee) {
validatorsInSyncCommittee++;
}
// Prev-epoch summary
const summary = validator.summaries.get(previousEpoch);
if (!summary) {
continue;
}
// Attestations
validatorMonitorMetrics?.prevEpochAttestations.observe(summary.attestations);
if (summary.attestationMinDelay !== null)
validatorMonitorMetrics?.prevEpochAttestationsMinDelaySeconds.observe(summary.attestationMinDelay);
validatorMonitorMetrics?.prevEpochAttestationAggregateInclusions.observe(summary.attestationAggregateInclusions);
validatorMonitorMetrics?.prevEpochAttestationBlockInclusions.observe(summary.attestationBlockInclusions);
if (summary.attestationMinBlockInclusionDistance !== null) {
validatorMonitorMetrics?.prevEpochAttestationBlockMinInclusionDistance.observe(summary.attestationMinBlockInclusionDistance);
}
// Blocks
validatorMonitorMetrics?.prevEpochBeaconBlocks.observe(summary.blocks);
if (summary.blockMinDelay !== null)
validatorMonitorMetrics?.prevEpochBeaconBlocksMinDelaySeconds.observe(summary.blockMinDelay);
// Aggregates
validatorMonitorMetrics?.prevEpochAggregatesTotal.observe(summary.aggregates);
if (summary.aggregateMinDelay !== null)
validatorMonitorMetrics?.prevEpochAggregatesMinDelaySeconds.observe(summary.aggregateMinDelay);
// Sync committee
prevEpochSyncCommitteeHits += summary.syncCommitteeHits;
prevEpochSyncCommitteeMisses += summary.syncCommitteeMisses;
// Only observe if included in sync committee to prevent distorting metrics
if (validatorInSyncCommittee) {
validatorMonitorMetrics?.prevEpochSyncSignatureAggregateInclusions.observe(summary.syncSignatureAggregateInclusions);
}
}
validatorMonitorMetrics?.validatorsInSyncCommittee.set(validatorsInSyncCommittee);
validatorMonitorMetrics?.prevEpochSyncCommitteeHits.set(prevEpochSyncCommitteeHits);
validatorMonitorMetrics?.prevEpochSyncCommitteeMisses.set(prevEpochSyncCommitteeMisses);
},
};
// Register a single collect() function to run all validatorMonitor metrics
validatorMonitorMetrics?.validatorsConnected.addCollect(() => {
const clockSlot = getCurrentSlot(config, genesisTime);
validatorMonitor.scrapeMetrics(clockSlot);
});
return validatorMonitor;
}
/**
* Best guess to automatically debug why validators do not achieve expected rewards.
* Tries to answer common questions such as:
* - Did the validator submit the attestation to this block?
* - Was the attestation seen in an aggregate?
* - Was the attestation seen in a block?
*/
function renderAttestationSummary(config, rootCache, summary, flags) {
// Reference https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices
//
// is_matching_source = data.source == justified_checkpoint
// is_matching_target = is_matching_source and data.target.root == get_block_root(state, data.target.epoch)
// is_matching_head = is_matching_target and data.beacon_block_root == get_block_root_at_slot(state, data.slot)
//
// is_matching_source MUST be true for the attestation to be included in a block
//
// timely_source = is_matching_source and inclusion_delay <= integer_squareroot(SLOTS_PER_EPOCH):
// timely_target = is_matching_target and inclusion_delay <= SLOTS_PER_EPOCH:
// timely_head = is_matching_head and inclusion_delay == MIN_ATTESTATION_INCLUSION_DELAY:
if (flags.timelyHead) {
// NOTE: If timelyHead everything else MUST be true also
return "timely_head";
}
//
if (flags.timelyTarget) {
// timelyHead == false, means at least one is true
// - attestation voted incorrect head
// - attestation was included late
// Note: the same attestation can be included in multiple blocks. For example, block with parent A at slot N can
// include the attestation. Then block as slot N+1 re-orgs slot N setting as parent A and includes the attestations
// from block at slot N.
//
// TODO: Track block inclusions, and then check which ones are canonical
if (!summary) {
// In normal conditions should never happen, validator is expected to submit an attestation to the tracking node.
// If the validator is using multiple beacon nodes as fallback, this condition may be triggered.
return "unexpected_timely_target_without_summary";
}
const canonicalBlockInclusion = summary.blockInclusions.find((block) => isCanonical(rootCache, block));
if (!canonicalBlockInclusion) {
// Should never happen, because for a state to exist that registers a validator's participation this specific
// beacon node must have imported a block with the attestation that caused the change in participation.
return "unexpected_timely_target_without_canonical_inclusion";
}
const { votedCorrectHeadRoot, blockSlot, attestationSlot } = canonicalBlockInclusion;
const inclusionDistance = Math.max(blockSlot - attestationSlot - MIN_ATTESTATION_INCLUSION_DELAY, 0);
if (votedCorrectHeadRoot && inclusionDistance === 0) {
// Should never happen, in this case timelyHead must be true
return "unexpected_timely_head_as_timely_target";
}
// Why is the distance > 0?
// - Block that should have included the attestation was missed
// - Attestation was not included in any aggregate
// - Attestation was sent late
// Why is the head vote wrong?
// - We processed a block late and voted for the parent
// - We voted for a block that latter was missed
// - We voted for a block that was re-org for another chain
let out = "timely_target";
if (!votedCorrectHeadRoot) {
out += "_" + whyIsHeadVoteWrong(rootCache, canonicalBlockInclusion);
}
if (inclusionDistance > 0) {
out += "_" + whyIsDistanceNotOk(rootCache, canonicalBlockInclusion, summary);
}
return out;
}
//
if (flags.timelySource) {
// timelyTarget == false && timelySource == true means that
// - attestation voted the wrong target but distance is <= integer_squareroot(SLOTS_PER_EPOCH)
return "wrong_target_timely_source";
}
//
// timelySource == false, either:
// - attestation was not included in the block
// - included in block with wrong target (very unlikely)
// - included in block with distance > SLOTS_PER_EPOCH (very unlikely)
// Validator failed to submit an attestation for this epoch, validator client is probably offline
if (!summary || summary.poolSubmitDelayMinSec === null) {
return "no_submission";
}
const canonicalBlockInclusion = summary.blockInclusions.find((block) => isCanonical(rootCache, block));
if (canonicalBlockInclusion) {
// Canonical block inclusion with no participation flags set means wrong target + late source
return "wrong_target_late_source";
}
const submittedLate = summary.poolSubmitDelayMinSec >
(INTERVALS_LATE_ATTESTATION_SUBMISSION * config.SECONDS_PER_SLOT) / INTERVALS_PER_SLOT;
const aggregateInclusion = summary.aggregateInclusionDelaysSec.length > 0;
if (submittedLate && aggregateInclusion) {
return "late_submit";
}
if (submittedLate && !aggregateInclusion) {
return "late_submit_no_aggregate_inclusion";
}
if (!submittedLate && aggregateInclusion) {
// TODO: Why was it missed then?
if (summary.blockInclusions.length) {
return "block_inclusion_but_orphan";
}
return "aggregate_inclusion_but_missed";
// } else if (!submittedLate && !aggregateInclusion) {
}
// Did the node had enough peers?
if (summary.poolSubmitSentPeers === 0) {
return "sent_to_zero_peers";
}
return "no_aggregate_inclusion";
}
function whyIsHeadVoteWrong(rootCache, canonicalBlockInclusion) {
const { votedForMissedSlot, attestationSlot } = canonicalBlockInclusion;
const canonicalAttestationSlotMissed = isMissedSlot(rootCache, attestationSlot);
// __A_______C
// \_B1
// ^^ attestation slot
//
// We vote for B1, but the next proposer skips our voted block.
// This scenario happens sometimes when blocks are published late
// __A____________E
// \_B1__C__D
// ^^ attestation slot
//
// We vote for B1, and due to some issue a longer reorg happens orphaning our vote.
// This scenario is considered in the above
if (!votedForMissedSlot && canonicalAttestationSlotMissed) {
// TODO: Did the block arrive late?
return "vote_orphaned";
}
// __A__B1___C
// \_(A)
// ^^ attestation slot
//
// We vote for A assuming skip block, next proposer's view differs
// This scenario happens sometimes when blocks are published late
if (votedForMissedSlot && !canonicalAttestationSlotMissed) {
// TODO: Did the block arrive late?
return "wrong_skip_vote";
}
// __A__B2___C
// \_B1
// ^^ attestation slot
//
// We vote for B1, but the next proposer continues the chain on a competing block
// This scenario is unlikely to happen in short re-orgs given no slashings, won't consider.
//
// __A____B_______C
// \ \_(B)
// \_(A)_(A)
//
// Vote for different heads on skipped slot
return "wrong_head_vote";
}
function whyIsDistanceNotOk(rootCache, canonicalBlockInclusion, summary) {
// If the attestation is not included in any aggregate it's likely because it was sent late.
if (summary.aggregateInclusionDelaysSec.length === 0) {
return "no_aggregate_inclusion";
}
// If the next slot of an attestation is missed, distance will be > 0 even if everything else was timely
if (isMissedSlot(rootCache, canonicalBlockInclusion.attestationSlot + 1)) {
return "next_slot_missed";
}
//
return "late_unknown";
}
/** Returns true if the state's root record includes `block` */
function isCanonical(rootCache, block) {
return rootCache.getBlockRootAtSlot(block.blockSlot) === block.blockRoot;
}
/** Returns true if root at slot is the same at slot - 1 == there was no new block at slot */
function isMissedSlot(rootCache, slot) {
return slot > 0 && rootCache.getBlockRootAtSlot(slot) === rootCache.getBlockRootAtSlot(slot - 1);
}
function renderBlockProposalSummary(config, rootCache, summary, proposalSlot) {
const proposal = summary?.blockProposals.find((proposal) => proposal.blockSlot === proposalSlot);
if (!proposal) {
return "not_submitted";
}
if (rootCache.getBlockRootAtSlot(proposalSlot) === proposal.blockRoot) {
// Canonical state includes our block
return "canonical";
}
let out = "orphaned";
if (isMissedSlot(rootCache, proposalSlot)) {
out += "_missed";
}
if (proposal.poolSubmitDelaySec !== null &&
proposal.poolSubmitDelaySec > (INTERVALS_LATE_BLOCK_SUBMISSION * config.SECONDS_PER_SLOT) / INTERVALS_PER_SLOT) {
out += "_late";
}
return out;
}
/**
* Cache to prevent accessing the state tree to fetch block roots repeteadly.
* In normal network conditions the same root is read multiple times, specially the target.
*/
export class RootHexCache {
constructor(state) {
this.state = state;
this.blockRootSlotCache = new Map();
}
getBlockRootAtSlot(slot) {
let root = this.blockRootSlotCache.get(slot);
if (!root) {
root = toRootHex(getBlockRootAtSlot(this.state, slot));
this.blockRootSlotCache.set(slot, root);
}
return root;
}
}
function createValidatorMonitorMetrics(register) {
return {
validatorsConnected: register.gauge({
name: "validator_monitor_validators",
help: "Count of validators that are specifically monitored by this beacon node",
}),
validatorsInSyncCommittee: register.gauge({
name: "validator_monitor_validators_in_sync_committee",
help: "Count of validators monitored by this beacon node that are part of sync committee",
}),
// Validator Monitor Metrics (per-epoch summaries)
// Only track prevEpochOnChainBalance per index
prevEpochOnChainBalance: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_balance",
help: "Balance of validator after an epoch",
labelNames: ["index"],
}),
prevEpochOnChainAttesterHit: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_attester_hit_total",
help: "Incremented if validator's submitted attestation is included in some blocks",
}),
prevEpochOnChainAttesterMiss: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_attester_miss_total",
help: "Incremented if validator's submitted attestation is not included in any blocks",
}),
prevEpochOnChainSourceAttesterHit: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_source_attester_hit_total",
help: "Incremented if the validator is flagged as a previous epoch source attester during per epoch processing",
}),
prevEpochOnChainSourceAttesterMiss: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_source_attester_miss_total",
help: "Incremented if the validator is not flagged as a previous epoch source attester during per epoch processing",
}),
prevEpochOnChainHeadAttesterHit: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_head_attester_hit_total",
help: "Incremented if the validator is flagged as a previous epoch head attester during per epoch processing",
}),
prevEpochOnChainHeadAttesterMiss: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_head_attester_miss_total",
help: "Incremented if the validator is not flagged as a previous epoch head attester during per epoch processing",
}),
prevOnChainAttesterCorrectHead: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_attester_correct_head_total",
help: "Total count of times a validator votes correct head",
}),
prevOnChainAttesterIncorrectHead: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_attester_incorrect_head_total",
help: "Total count of times a validator votes incorrect head",
}),
prevEpochOnChainTargetAttesterHit: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_target_attester_hit_total",
help: "Incremented if the validator is flagged as a previous epoch target attester during per epoch processing",
}),
prevEpochOnChainTargetAttesterMiss: register.gauge({
name: "validator_monitor_prev_epoch_on_chain_target_attester_miss_total",
help: "Incremented if the validator is not flagged as a previous epoch target attester during per epoch processing",
}),
prevEpochOnChainInclusionDistance: register.histogram({
name: "validator_monitor_prev_epoch_on_chain_inclusion_distance",
help: "The attestation inclusion distance calculated during per epoch processing",
// min inclusion distance is 1, usual values are 1,2,3 max is 32 (1 epoch)
buckets: [1, 2, 3, 5, 10, 32],
}),
prevEpochAttestations: register.histogram({
name: "validator_monitor_prev_epoch_attestations",
help: "The number of unagg. attestations seen in the previous epoch",
buckets: [0, 1, 2, 3],
}),
prevEpochAttestationsMinDelaySeconds: register.histogram({
name: "validator_monitor_prev_epoch_attestations_min_delay_seconds",
help: "The min delay between when the validator should send the attestation and when it was received",
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
}),
prevEpochAttestationAggregateInclusions: register.histogram({
name: "validator_monitor_prev_epoch_attestation_aggregate_inclusions",
help: "The count of times an attestation was seen inside an aggregate",
buckets: [0, 1, 2, 3, 5, 10],
}),
prevEpochAttestationBlockInclusions: register.histogram({
name: "validator_monitor_prev_epoch_attestation_block_inclusions",
help: "The count of times an attestation was seen inside a block",
buckets: [0, 1, 2, 3, 5],
}),
prevEpochAttestationBlockMinInclusionDistance: register.histogram({
name: "validator_monitor_prev_epoch_attestation_block_min_inclusion_distance",
help: "The minimum inclusion distance observed for the inclusion of an attestation in a block",
buckets: [1, 2, 3, 5, 10, 32],
}),
prevEpochBeaconBlocks: register.histogram({
name: "validator_monitor_prev_epoch_beacon_blocks",
help: "The number of beacon_blocks seen in the previous epoch",
buckets: [0, 1, 2, 3, 5, 10],
}),
prevEpochBeaconBlocksMinDelaySeconds: register.histogram({
name: "validator_monitor_prev_epoch_beacon_blocks_min_delay_seconds",
help: "The min delay between when the validator should send the block and when it was received",
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
}),
prevEpochAggregatesTotal: register.histogram({
name: "validator_monitor_prev_epoch_aggregates",
help: "The number of aggregates seen in the previous epoch",
buckets: [0, 1, 2, 3, 5, 10],
}),
prevEpochAggregatesMinDelaySeconds: register.histogram({
name: "validator_monitor_prev_epoch_aggregates_min_delay_seconds",
help: "The min delay between when the validator should send the aggregate and when it was received",
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
}),
prevEpochSyncCommitteeHits: register.gauge({
name: "validator_monitor_prev_epoch_sync_committee_hits",
help: "Count of times in prev epoch connected validators participated in imported block's syncAggregate",
}),
prevEpochSyncCommitteeMisses: register.gauge({
name: "validator_monitor_prev_epoch_sync_committee_misses",
help: "Count of times in prev epoch connected validators fail to participate in imported block's syncAggregate",
}),
prevEpochSyncSignatureAggregateInclusions: register.histogram({
name: "validator_monitor_prev_epoch_sync_signature_aggregate_inclusions",
help: "The count of times a sync signature was seen inside an aggregate",
buckets: [0, 1, 2, 3, 5, 10],
}),
prevEpochAttestationSummary: register.gauge({
name: "validator_monitor_prev_epoch_attestation_summary",
help: "Best guess of the node of the result of previous epoch validators attestation actions and causality",
labelNames: ["summary"],
}),
prevEpochBlockProposalSummary: register.gauge({
name: "validator_monitor_prev_epoch_block_proposal_summary",
help: "Best guess of the node of the result of previous epoch validators block proposal actions and causality",
labelNames: ["summary"],
}),
// Validator Monitor Metrics (real-time)
unaggregatedAttestationTotal: register.gauge({
name: "validator_monitor_unaggregated_attestation_total",
help: "Number of unaggregated attestations seen",
labelNames: ["src"],
}),
unaggregatedAttestationDelaySeconds: register.histogram({
name: "validator_monitor_unaggregated_attestation_delay_seconds",
help: "The delay between when the validator should send the attestation and when it was received",
labelNames: ["src"],
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
}),
unaggregatedAttestationSubmittedSentPeers: register.histogram({
name: "validator_monitor_unaggregated_attestation_submitted_sent_peers_count",
help: "Number of peers that an unaggregated attestation sent to",
// as of Apr 2022, most of the time we sent to >30 peers per attestations
// these bucket values just base on that fact to get equal range
// refine if we want more reasonable values
buckets: [0, 10, 20, 30],
}),
aggregatedAttestationTotal: register.gauge({
name: "validator_monitor_aggregated_attestation_total",
help: "Number of aggregated attestations seen",
labelNames: ["src"],
}),
aggregatedAttestationDelaySeconds: register.histogram({
name: "validator_monitor_aggregated_attestation_delay_seconds",
help: "The delay between then the validator should send the aggregate and when it was received",
labelNames: ["src"],
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
}),
attestationInAggregateTotal: register.gauge({
name: "validator_monitor_attestation_in_aggregate_total",
help: "Number of times an attestation has been seen in an aggregate",
labelNames: ["src"],
}),
attestationInAggregateDelaySeconds: register.histogram({
name: "validator_monitor_attestation_in_aggregate_delay_seconds",
help: "The delay between when the validator should send the aggregate and when it was received",
labelNames: ["src"],
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
}),
attestationInBlockTotal: register.gauge({
name: "validator_monitor_attestation_in_block_total",
help: "Number of times an attestation has been seen in a block",
}),
attestationInBlockDelaySlots: register.histogram({
name: "validator_monitor_attestation_in_block_delay_slots",
help: "The excess slots (beyond the minimum delay) between the attestation slot and the block slot",
buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
}),
attestationInBlockParticipants: register.histogram({
name: "validator_monitor_attestation_in_block_participants",
help: "The total participants in attestations of monitored validators included in blocks",
buckets: [1, 5, 20, 50, 100, 200],
}),
syncSignatureInAggregateTotal: register.gauge({
name: "validator_monitor_sync_signature_in_aggregate_total",
help: "Number of times a sync signature has been seen in an aggregate",
}),
beaconBlockTotal: register.gauge({