@lodestar/beacon-node/src/chain/validatorMonitor.ts

A Typescript implementation of the beacon chain

import {ChainForkConfig} from "@lodestar/config";
import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params";
import {
  IBeaconStateView,
  ParticipationFlags,
  computeEpochAtSlot,
  computeStartSlotAtEpoch,
  computeTimeAtSlot,
  getCurrentSlot,
  isStatePostAltair,
  parseAttesterFlags,
  parseParticipationFlags,
} from "@lodestar/state-transition";
import {
  BeaconBlock,
  Epoch,
  IndexedAttestation,
  RootHex,
  SignedAggregateAndProof,
  Slot,
  SubnetID,
  ValidatorIndex,
  altair,
  deneb,
  gloas,
} from "@lodestar/types";
import {LogData, LogHandler, LogLevel, Logger, MapDef, MapDefMax, prettyPrintIndices, toRootHex} from "@lodestar/utils";
import {GENESIS_SLOT} from "../constants/constants.js";
import {RegistryMetricCreator} from "../metrics/index.js";

/** The validator monitor collects per-epoch data about each monitored validator.
 * Historical data will be kept around for `HISTORIC_EPOCHS` before it is pruned.
 */
const MAX_CACHED_EPOCHS = 4;

const MAX_CACHED_DISTINCT_TARGETS = 4;

// TODO GLOAS: re-evaluate these timings
const LATE_ATTESTATION_SUBMISSION_BPS = 5000;
const LATE_BLOCK_SUBMISSION_BPS = 2500;

/** Number of epochs to retain registered validators after their last registration */
const RETAIN_REGISTERED_VALIDATORS_EPOCHS = 2;

type Seconds = number;
export enum OpSource {
  api = "api",
  gossip = "gossip",
}

export type ValidatorMonitor = {
  registerLocalValidator(index: number): void;
  registerLocalValidatorInSyncCommittee(index: number, untilEpoch: Epoch): void;
  registerValidatorStatuses(
    currentEpoch: Epoch,
    inclusionDelays: number[],
    flags: number[],
    isActiveCurrEpoch: boolean[],
    isActivePrevEpoch: boolean[],
    balances?: number[]
  ): void;
  registerBeaconBlock(src: OpSource, delaySec: Seconds, block: BeaconBlock): void;
  registerBlobSidecar(src: OpSource, seenTimestampSec: Seconds, blob: deneb.BlobSidecar): void;
  registerExecutionPayloadEnvelope(
    src: OpSource,
    delaySec: Seconds,
    envelope: gloas.SignedExecutionPayloadEnvelope
  ): void;
  registerImportedBlock(block: BeaconBlock, data: {proposerBalanceDelta: number}): void;
  onPoolSubmitUnaggregatedAttestation(
    seenTimestampSec: number,
    indexedAttestation: IndexedAttestation,
    subnet: SubnetID,
    sentPeers: number
  ): void;
  onPoolSubmitAggregatedAttestation(
    seenTimestampSec: number,
    indexedAttestation: IndexedAttestation,
    sentPeers: number
  ): void;
  registerGossipUnaggregatedAttestation(seenTimestampSec: Seconds, indexedAttestation: IndexedAttestation): void;
  registerGossipAggregatedAttestation(
    seenTimestampSec: Seconds,
    signedAggregateAndProof: SignedAggregateAndProof,
    indexedAttestation: IndexedAttestation
  ): void;
  registerAttestationInBlock(
    indexedAttestation: IndexedAttestation,
    parentSlot: Slot,
    correctHead: boolean,
    missedSlotVote: boolean,
    inclusionBlockRoot: RootHex,
    inclusionBlockSlot: Slot
  ): void;
  registerGossipSyncContributionAndProof(
    syncContributionAndProof: altair.ContributionAndProof,
    syncCommitteeParticipantIndices: ValidatorIndex[]
  ): void;
  registerSyncAggregateInBlock(
    epoch: Epoch,
    syncAggregate: altair.SyncAggregate,
    syncCommitteeIndices: Uint32Array
  ): void;
  onceEveryEndOfEpoch(state: IBeaconStateView): void;
  scrapeMetrics(slotClock: Slot): void;
  /** Returns the list of validator indices currently being monitored */
  getMonitoredValidatorIndices(): ValidatorIndex[];
};

export type ValidatorMonitorOpts = {
  /** Log validator monitor events as info */
  validatorMonitorLogs?: boolean;
};

export const defaultValidatorMonitorOpts: ValidatorMonitorOpts = {
  validatorMonitorLogs: false,
};

/** Information required to reward some validator during the current and previous epoch. */
type ValidatorStatus = {
  /** True if the validator has been slashed, ever. */
  isSlashed: boolean;
  /** True if the validator was active in the state's _current_ epoch. */
  isActiveInCurrentEpoch: boolean;
  /** True if the validator was active in the state's _previous_ epoch. */
  isActiveInPreviousEpoch: boolean;
  /** The validator's effective balance in the _current_ epoch. */
  currentEpochEffectiveBalance: number;

  /** True if the validator had an attestation included in the _previous_ epoch. */
  isPrevSourceAttester: boolean;
  /** True if the validator's beacon block root attestation for the first slot of the _previous_
      epoch matches the block root known to the state. */
  isPrevTargetAttester: boolean;
  /** True if the validator's beacon block root attestation in the _previous_ epoch at the
      attestation's slot (`attestation_data.slot`) matches the block root known to the state. */
  isPrevHeadAttester: boolean;

  /** True if the validator had an attestation included in the _current_ epoch. */
  isCurrSourceAttester: boolean;
  /** True if the validator's beacon block root attestation for the first slot of the _current_
      epoch matches the block root known to the state. */
  isCurrTargetAttester: boolean;
  /** True if the validator's beacon block root attestation in the _current_ epoch at the
      attestation's slot (`attestation_data.slot`) matches the block root known to the state. */
  isCurrHeadAttester: boolean;

  /** The distance between the attestation slot and the slot that attestation was included in a block. */
  inclusionDistance: number;
};

function statusToSummary(
  inclusionDelay: number,
  flag: number,
  isActiveInCurrentEpoch: boolean,
  isActiveInPreviousEpoch: boolean
): ValidatorStatus {
  const flags = parseAttesterFlags(flag);
  return {
    isSlashed: flags.unslashed,
    isActiveInCurrentEpoch,
    isActiveInPreviousEpoch,
    // TODO: Implement
    currentEpochEffectiveBalance: 0,

    isPrevSourceAttester: flags.prevSourceAttester,
    isPrevTargetAttester: flags.prevTargetAttester,
    isPrevHeadAttester: flags.prevHeadAttester,
    isCurrSourceAttester: flags.currSourceAttester,
    isCurrTargetAttester: flags.currTargetAttester,
    isCurrHeadAttester: flags.currHeadAttester,
    inclusionDistance: inclusionDelay,
  };
}

/** Contains data pertaining to one validator for one epoch. */
type EpochSummary = {
  // Attestations with a target in the current epoch.
  /** The number of attestations seen. */
  attestations: number;
  /** The delay between when the attestation should have been produced and when it was observed. */
  attestationMinDelay: Seconds | null;
  /** The number of times a validators attestation was seen in an aggregate. */
  attestationAggregateInclusions: number;
  /** The number of times a validators attestation was seen in a block. */
  attestationBlockInclusions: number;
  /** The minimum observed inclusion distance for an attestation for this epoch.. */
  attestationMinBlockInclusionDistance: Slot | null;
  /** The attestation contains the correct head or not */
  attestationCorrectHead: boolean | null;
  // Blocks with a slot in the current epoch.
  /** The number of blocks observed. */
  blocks: number;
  /** The delay between when the block should have been produced and when it was observed. */
  blockMinDelay: Seconds | null;
  // Aggregates with a target in the current epoch
  /** The number of signed aggregate and proofs observed. */
  aggregates: number;
  /** The delay between when the aggregate should have been produced and when it was observed. */
  aggregateMinDelay: Seconds | null;
  /** Count of times validator expected in sync aggregate participated */
  syncCommitteeHits: number;
  /** Count of times validator expected in sync aggregate failed to participate */
  syncCommitteeMisses: number;
  /** Number of times a validator's sync signature was seen in an aggregate */
  syncSignatureAggregateInclusions: number;
  /** Submitted proposals from this validator at this epoch */
  blockProposals: BlockProposals[];
};

type BlockProposals = {
  blockRoot: RootHex;
  blockSlot: Slot;
  poolSubmitDelaySec: number | null;
  successfullyImported: boolean;
};

function getEpochSummary(validator: MonitoredValidator, epoch: Epoch): EpochSummary {
  let summary = validator.summaries.get(epoch);
  if (!summary) {
    summary = {
      attestations: 0,
      attestationMinDelay: null,
      attestationAggregateInclusions: 0,
      attestationBlockInclusions: 0,
      attestationMinBlockInclusionDistance: null,
      blocks: 0,
      blockMinDelay: null,
      aggregates: 0,
      aggregateMinDelay: null,
      attestationCorrectHead: null,
      syncCommitteeHits: 0,
      syncCommitteeMisses: 0,
      syncSignatureAggregateInclusions: 0,
      blockProposals: [],
    };
    validator.summaries.set(epoch, summary);
  }

  // Prune
  const toPrune = validator.summaries.size - MAX_CACHED_EPOCHS;
  if (toPrune > 0) {
    let pruned = 0;
    for (const idx of validator.summaries.keys()) {
      validator.summaries.delete(idx);
      if (++pruned >= toPrune) break;
    }
  }

  return summary;
}

// To uniquely identify an attestation:
// `index=$validator_index target=$target_epoch:$target_root
type AttestationSummary = {
  poolSubmitDelayMinSec: number | null;
  poolSubmitSentPeers: number | null;
  aggregateInclusionDelaysSec: number[];
  blockInclusions: AttestationBlockInclusion[];
};

type AttestationBlockInclusion = {
  blockRoot: RootHex;
  blockSlot: Slot;
  votedCorrectHeadRoot: boolean;
  votedForMissedSlot: boolean;
  attestationSlot: Slot;
};

/** `$target_epoch:$target_root` */
type TargetRoot = string;

/// A validator that is being monitored by the `ValidatorMonitor`. */
type MonitoredValidator = {
  /// A history of the validator over time. */
  summaries: Map<Epoch, EpochSummary>;
  inSyncCommitteeUntilEpoch: number;
  // Unless the validator slashes itself, there MUST be one attestation per target checkpoint
  attestations: MapDefMax<Epoch, MapDefMax<TargetRoot, AttestationSummary>>;
  lastRegisteredTimeMs: number;
};

export function createValidatorMonitor(
  metricsRegister: RegistryMetricCreator | null,
  config: ChainForkConfig,
  genesisTime: number,
  logger: Logger,
  opts: ValidatorMonitorOpts
): ValidatorMonitor {
  const logLevel = opts.validatorMonitorLogs ? LogLevel.info : LogLevel.debug;
  const log: LogHandler = (message: string, context?: LogData) => {
    logger[logLevel](message, context);
  };

  // Calculate retain time dynamically based on slot duration (2 epochs)
  const retainRegisteredValidatorsMs = SLOTS_PER_EPOCH * config.SLOT_DURATION_MS * RETAIN_REGISTERED_VALIDATORS_EPOCHS;

  /** The validators that require additional monitoring. */
  const validators = new MapDef<ValidatorIndex, MonitoredValidator>(() => ({
    summaries: new Map<Epoch, EpochSummary>(),
    inSyncCommitteeUntilEpoch: -1,
    attestations: new MapDefMax(
      () =>
        new MapDefMax(
          () => ({
            poolSubmitDelayMinSec: null,
            poolSubmitSentPeers: null,
            aggregateInclusionDelaysSec: [],
            blockInclusions: [],
          }),
          MAX_CACHED_DISTINCT_TARGETS
        ),
      MAX_CACHED_EPOCHS
    ),
    lastRegisteredTimeMs: 0,
  }));

  let lastRegisteredStatusEpoch = -1;

  // Track validator additions/removals per epoch for logging
  const addedValidatorsInEpoch: Set<ValidatorIndex> = new Set();
  const removedValidatorsInEpoch: Set<ValidatorIndex> = new Set();

  const validatorMonitorMetrics = metricsRegister ? createValidatorMonitorMetrics(metricsRegister) : null;

  const validatorMonitor: ValidatorMonitor = {
    registerLocalValidator(index) {
      const isNewValidator = !validators.has(index);
      validators.getOrDefault(index).lastRegisteredTimeMs = Date.now();
      if (isNewValidator) {
        addedValidatorsInEpoch.add(index);
      }
    },

    registerLocalValidatorInSyncCommittee(index, untilEpoch) {
      const validator = validators.get(index);
      if (validator) {
        validator.inSyncCommitteeUntilEpoch = Math.max(untilEpoch, validator.inSyncCommitteeUntilEpoch ?? -1);
      }
    },

    registerValidatorStatuses(currentEpoch, inclusionDelays, flags, isActiveCurrEpoch, isActiveInPrevEpoch, balances) {
      // Prevent registering status for the same epoch twice. processEpoch() may be ran more than once for the same epoch.
      if (currentEpoch <= lastRegisteredStatusEpoch) {
        return;
      }
      lastRegisteredStatusEpoch = currentEpoch;
      const previousEpoch = currentEpoch - 1;

      // There won't be any validator activity in epoch -1
      if (previousEpoch === -1) {
        return;
      }

      // Track total balance instead of per-validator balance to reduce metric cardinality
      let totalBalance = 0;

      for (const [index, monitoredValidator] of validators.entries()) {
        // We subtract two from the state of the epoch that generated these summaries.
        //
        // - One to account for it being the previous epoch.
        // - One to account for the state advancing an epoch whilst generating the validator
        //     statuses.
        const summary = statusToSummary(
          inclusionDelays[index],
          flags[index],
          isActiveCurrEpoch[index],
          isActiveInPrevEpoch[index]
        );

        if (summary.isPrevSourceAttester) {
          validatorMonitorMetrics?.prevEpochOnChainSourceAttesterHit.inc();
        } else {
          validatorMonitorMetrics?.prevEpochOnChainSourceAttesterMiss.inc();
        }
        if (summary.isPrevHeadAttester) {
          validatorMonitorMetrics?.prevEpochOnChainHeadAttesterHit.inc();
        } else {
          validatorMonitorMetrics?.prevEpochOnChainHeadAttesterMiss.inc();
        }
        if (summary.isPrevTargetAttester) {
          validatorMonitorMetrics?.prevEpochOnChainTargetAttesterHit.inc();
        } else {
          validatorMonitorMetrics?.prevEpochOnChainTargetAttesterMiss.inc();
        }

        const prevEpochSummary = monitoredValidator.summaries.get(previousEpoch);
        const attestationCorrectHead = prevEpochSummary?.attestationCorrectHead;
        if (attestationCorrectHead !== null && attestationCorrectHead !== undefined) {
          if (attestationCorrectHead) {
            validatorMonitorMetrics?.prevOnChainAttesterCorrectHead.inc();
          } else {
            validatorMonitorMetrics?.prevOnChainAttesterIncorrectHead.inc();
          }
        }

        const attestationMinBlockInclusionDistance = prevEpochSummary?.attestationMinBlockInclusionDistance;
        const inclusionDistance =
          attestationMinBlockInclusionDistance != null && attestationMinBlockInclusionDistance > 0
            ? // altair, attestation is not missed
              attestationMinBlockInclusionDistance
            : summary.inclusionDistance
              ? // phase0, this is from the state transition
                summary.inclusionDistance
              : null;

        if (inclusionDistance !== null) {
          validatorMonitorMetrics?.prevEpochOnChainInclusionDistance.observe(inclusionDistance);
          validatorMonitorMetrics?.prevEpochOnChainAttesterHit.inc();
        } else {
          validatorMonitorMetrics?.prevEpochOnChainAttesterMiss.inc();
        }

        const balance = balances?.[index];
        if (balance !== undefined) {
          totalBalance += balance;
        }

        if (!summary.isPrevSourceAttester || !summary.isPrevTargetAttester || !summary.isPrevHeadAttester) {
          log("Failed attestation in previous epoch", {
            validator: index,
            prevEpoch: currentEpoch - 1,
            isPrevSourceAttester: summary.isPrevSourceAttester,
            isPrevHeadAttester: summary.isPrevHeadAttester,
            isPrevTargetAttester: summary.isPrevTargetAttester,
            // inclusionDistance is not available in summary since altair
            inclusionDistance,
          });
        }
      }

      if (balances !== undefined) {
        validatorMonitorMetrics?.prevEpochOnChainBalance.set(totalBalance);
      }
    },

    registerBeaconBlock(src, delaySec, block) {
      const validator = validators.get(block.proposerIndex);
      // Returns the delay between the start of `block.slot` and `seenTimestamp`.
      if (validator) {
        validatorMonitorMetrics?.beaconBlockTotal.inc({src});
        validatorMonitorMetrics?.beaconBlockDelaySeconds.observe({src}, delaySec);

        const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot));
        summary.blockProposals.push({
          blockRoot: toRootHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)),
          blockSlot: block.slot,
          poolSubmitDelaySec: delaySec,
          successfullyImported: false,
        });
      }
    },

    registerBlobSidecar(_src, _seenTimestampSec, _blob) {
      //TODO: freetheblobs
    },

    registerExecutionPayloadEnvelope(_src, _delaySec, _envelope) {
      // TODO GLOAS: implement execution payload envelope monitoring
    },

    registerImportedBlock(block, {proposerBalanceDelta}) {
      const validator = validators.get(block.proposerIndex);
      if (validator) {
        validatorMonitorMetrics?.proposerBalanceDeltaKnown.observe(proposerBalanceDelta);

        // There should be alredy a summary for the block. Could be missing when using one VC multiple BNs
        const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot));
        const proposal = summary.blockProposals.find((p) => p.blockSlot === block.slot);
        if (proposal) {
          proposal.successfullyImported = true;
        } else {
          summary.blockProposals.push({
            blockRoot: toRootHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)),
            blockSlot: block.slot,
            poolSubmitDelaySec: null,
            successfullyImported: true,
          });
        }
      }
    },

    onPoolSubmitUnaggregatedAttestation(seenTimestampSec, indexedAttestation, subnet, sentPeers) {
      const data = indexedAttestation.data;
      const fork = config.getForkName(data.slot);
      // Returns the duration between when the attestation `data` could be produced (ATTESTATION_DUE_BPS through the slot) and `seenTimestamp`.
      const delaySec =
        seenTimestampSec -
        (computeTimeAtSlot(config, data.slot, genesisTime) + config.getAttestationDueMs(fork) / 1000);
      for (const index of indexedAttestation.attestingIndices) {
        const validator = validators.get(index);
        if (validator) {
          validatorMonitorMetrics?.unaggregatedAttestationSubmittedSentPeers.observe(sentPeers);
          validatorMonitorMetrics?.unaggregatedAttestationDelaySeconds.observe({src: OpSource.api}, delaySec);
          log("Published unaggregated attestation", {
            validator: index,
            slot: data.slot,
            committeeIndex: data.index,
            subnet,
            sentPeers,
            delaySec: delaySec.toFixed(4),
          });

          const attestationSummary = validator.attestations
            .getOrDefault(indexedAttestation.data.target.epoch)
            .getOrDefault(toRootHex(indexedAttestation.data.target.root));
          if (
            attestationSummary.poolSubmitDelayMinSec === null ||
            attestationSummary.poolSubmitDelayMinSec > delaySec
          ) {
            attestationSummary.poolSubmitDelayMinSec = delaySec;
          }
        }
      }
    },

    registerGossipUnaggregatedAttestation(seenTimestampSec, indexedAttestation) {
      const src = OpSource.gossip;
      const data = indexedAttestation.data;
      const epoch = computeEpochAtSlot(data.slot);
      const fork = config.getForkName(data.slot);
      // Returns the duration between when the attestation `data` could be produced (ATTESTATION_DUE_BPS through the slot) and `seenTimestamp`.
      const delaySec =
        seenTimestampSec -
        (computeTimeAtSlot(config, data.slot, genesisTime) + config.getAttestationDueMs(fork) / 1000);

      for (const index of indexedAttestation.attestingIndices) {
        const validator = validators.get(index);
        if (validator) {
          validatorMonitorMetrics?.unaggregatedAttestationTotal.inc({src});
          validatorMonitorMetrics?.unaggregatedAttestationDelaySeconds.observe({src}, delaySec);
          const summary = getEpochSummary(validator, epoch);
          summary.attestations += 1;
          summary.attestationMinDelay = Math.min(delaySec, summary.attestationMinDelay ?? Infinity);
        }
      }
    },

    onPoolSubmitAggregatedAttestation(seenTimestampSec, indexedAttestation, sentPeers) {
      const data = indexedAttestation.data;
      const fork = config.getForkName(data.slot);
      // Returns the duration between when a `AggregateAndproof` with `data` could be produced (AGGREGATE_DUE_BPS through the slot) and `seenTimestamp`.
      const delaySec =
        seenTimestampSec - (computeTimeAtSlot(config, data.slot, genesisTime) + config.getAggregateDueMs(fork) / 1000);

      for (const index of indexedAttestation.attestingIndices) {
        const validator = validators.get(index);
        if (validator) {
          validatorMonitorMetrics?.aggregatedAttestationDelaySeconds.observe({src: OpSource.api}, delaySec);
          log("Published aggregated attestation", {
            validator: index,
            slot: data.slot,
            committeeIndex: data.index,
            sentPeers,
            delaySec: delaySec.toFixed(4),
          });

          validator.attestations
            .getOrDefault(indexedAttestation.data.target.epoch)
            .getOrDefault(toRootHex(indexedAttestation.data.target.root))
            .aggregateInclusionDelaysSec.push(delaySec);
        }
      }
    },

    registerGossipAggregatedAttestation(seenTimestampSec, signedAggregateAndProof, indexedAttestation) {
      const src = OpSource.gossip;
      const data = indexedAttestation.data;
      const epoch = computeEpochAtSlot(data.slot);
      const fork = config.getForkName(data.slot);
      // Returns the duration between when a `AggregateAndproof` with `data` could be produced (AGGREGATE_DUE_BPS through the slot) and `seenTimestamp`.
      const delaySec =
        seenTimestampSec - (computeTimeAtSlot(config, data.slot, genesisTime) + config.getAggregateDueMs(fork) / 1000);

      const aggregatorIndex = signedAggregateAndProof.message.aggregatorIndex;
      const validatorAggregator = validators.get(aggregatorIndex);
      if (validatorAggregator) {
        validatorMonitorMetrics?.aggregatedAttestationTotal.inc({src});
        validatorMonitorMetrics?.aggregatedAttestationDelaySeconds.observe({src}, delaySec);
        const summary = getEpochSummary(validatorAggregator, epoch);
        summary.aggregates += 1;
        summary.aggregateMinDelay = Math.min(delaySec, summary.aggregateMinDelay ?? Infinity);
      }

      for (const index of indexedAttestation.attestingIndices) {
        const validator = validators.get(index);
        if (validator) {
          validatorMonitorMetrics?.attestationInAggregateTotal.inc({src});
          validatorMonitorMetrics?.attestationInAggregateDelaySeconds.observe({src}, delaySec);
          const summary = getEpochSummary(validator, epoch);
          summary.attestationAggregateInclusions += 1;
          log("Attestation is included in aggregate", {
            validator: index,
            slot: data.slot,
            committeeIndex: data.index,
            aggregatorIndex,
          });

          validator.attestations
            .getOrDefault(indexedAttestation.data.target.epoch)
            .getOrDefault(toRootHex(indexedAttestation.data.target.root))
            .aggregateInclusionDelaysSec.push(delaySec);
        }
      }
    },

    // Register that the `indexed_attestation` was included in a *valid* `BeaconBlock`.
    registerAttestationInBlock(
      indexedAttestation,
      parentSlot,
      correctHead,
      missedSlotVote,
      inclusionBlockRoot,
      inclusionBlockSlot
    ): void {
      const data = indexedAttestation.data;
      // optimal inclusion distance, not to count skipped slots between data.slot and blockSlot
      const inclusionDistance = Math.max(parentSlot - data.slot, 0) + 1;
      const delay = inclusionDistance - MIN_ATTESTATION_INCLUSION_DELAY;
      const epoch = computeEpochAtSlot(data.slot);
      const participants = indexedAttestation.attestingIndices.length;

      for (const index of indexedAttestation.attestingIndices) {
        const validator = validators.get(index);
        if (validator) {
          validatorMonitorMetrics?.attestationInBlockTotal.inc();
          validatorMonitorMetrics?.attestationInBlockDelaySlots.observe(delay);
          validatorMonitorMetrics?.attestationInBlockParticipants.observe(participants);

          const summary = getEpochSummary(validator, epoch);
          summary.attestationBlockInclusions += 1;
          if (summary.attestationMinBlockInclusionDistance !== null) {
            summary.attestationMinBlockInclusionDistance = Math.min(
              summary.attestationMinBlockInclusionDistance,
              inclusionDistance
            );
          } else {
            summary.attestationMinBlockInclusionDistance = inclusionDistance;
          }

          summary.attestationCorrectHead = correctHead;

          validator.attestations
            .getOrDefault(indexedAttestation.data.target.epoch)
            .getOrDefault(toRootHex(indexedAttestation.data.target.root))
            .blockInclusions.push({
              blockRoot: inclusionBlockRoot,
              blockSlot: inclusionBlockSlot,
              votedCorrectHeadRoot: correctHead,
              votedForMissedSlot: missedSlotVote,
              attestationSlot: indexedAttestation.data.slot,
            });

          log("Attestation is included in block", {
            validator: index,
            slot: data.slot,
            committeeIndex: data.index,
            inclusionDistance,
            correctHead,
            participants,
          });
        }
      }
    },

    registerGossipSyncContributionAndProof(syncContributionAndProof, syncCommitteeParticipantIndices) {
      const epoch = computeEpochAtSlot(syncContributionAndProof.contribution.slot);

      for (const index of syncCommitteeParticipantIndices) {
        const validator = validators.get(index);
        if (validator) {
          validatorMonitorMetrics?.syncSignatureInAggregateTotal.inc();

          const summary = getEpochSummary(validator, epoch);
          summary.syncSignatureAggregateInclusions += 1;
        }
      }
    },

    registerSyncAggregateInBlock(epoch, syncAggregate, syncCommitteeIndices) {
      for (let i = 0; i < syncCommitteeIndices.length; i++) {
        const validator = validators.get(syncCommitteeIndices[i]);
        if (validator) {
          const summary = getEpochSummary(validator, epoch);
          if (syncAggregate.syncCommitteeBits.get(i)) {
            summary.syncCommitteeHits++;
          } else {
            summary.syncCommitteeMisses++;
          }
        }
      }
    },

    // Validator monitor tracks performance of validators in healthy network conditions.
    // It does not attempt to track correctly duties on forking conditions deeper than 1 epoch.
    // To guard against short re-orgs it will track the status of epoch N at the end of epoch N+1.
    // This function **SHOULD** be called at the last slot of an epoch to have max possible information.
    onceEveryEndOfEpoch(headState) {
      if (headState.slot <= GENESIS_SLOT) {
        // Before genesis, there won't be any validator activity
        return;
      }

      // Prune validators not seen in a while
      for (const [index, validator] of validators.entries()) {
        if (Date.now() - validator.lastRegisteredTimeMs > retainRegisteredValidatorsMs) {
          validators.delete(index);
          removedValidatorsInEpoch.add(index);
        }
      }

      // Log validator monitor status every epoch
      const allIndices = Array.from(validators.keys()).sort((a, b) => a - b);
      const addedIndices = Array.from(addedValidatorsInEpoch).sort((a, b) => a - b);
      const removedIndices = Array.from(removedValidatorsInEpoch).sort((a, b) => a - b);

      log("Validator monitor status", {
        epoch: computeEpochAtSlot(headState.slot),
        added: addedIndices.length > 0 ? prettyPrintIndices(addedIndices) : "none",
        removed: removedIndices.length > 0 ? prettyPrintIndices(removedIndices) : "none",
        total: validators.size,
        indices: prettyPrintIndices(allIndices),
      });

      // Clear tracking sets for next epoch
      addedValidatorsInEpoch.clear();
      removedValidatorsInEpoch.clear();

      // Compute summaries of previous epoch attestation performance
      const prevEpoch = computeEpochAtSlot(headState.slot) - 1;

      // During the end of first epoch, the prev epoch with be -1
      // Skip this as there is no attestation and block proposal summary in epoch -1
      if (prevEpoch === -1) {
        return;
      }

      if (validators.size === 0) {
        return;
      }

      const rootCache = new RootHexCache(headState);

      if (isStatePostAltair(headState)) {
        const prevEpochStartSlot = computeStartSlotAtEpoch(prevEpoch);
        const prevEpochTargetRoot = toRootHex(headState.getBlockRootAtSlot(prevEpochStartSlot));

        // Check attestation performance
        for (const [index, validator] of validators.entries()) {
          const flags = parseParticipationFlags(headState.getPreviousEpochParticipation(index));
          const attestationSummary = validator.attestations.get(prevEpoch)?.get(prevEpochTargetRoot);
          const summary = renderAttestationSummary(config, rootCache, attestationSummary, flags);
          validatorMonitorMetrics?.prevEpochAttestationSummary.inc({summary});
          log("Previous epoch attestation", {
            validator: index,
            epoch: prevEpoch,
            summary,
          });
        }
      }

      if (headState.previousProposers !== null) {
        // proposersPrevEpoch is null on the first epoch of `headState` being generated
        for (const [slotIndex, validatorIndex] of headState.previousProposers.entries()) {
          const validator = validators.get(validatorIndex);
          if (validator) {
            // If expected proposer is a tracked validator
            const epochSummary = validator.summaries.get(prevEpoch);
            const proposalSlot = SLOTS_PER_EPOCH * prevEpoch + slotIndex;
            const summary = renderBlockProposalSummary(config, rootCache, epochSummary, proposalSlot);
            validatorMonitorMetrics?.prevEpochBlockProposalSummary.inc({summary});
            log("Previous epoch block proposal", {
              validator: validatorIndex,
              slot: proposalSlot,
              epoch: prevEpoch,
              summary,
            });
          }
        }
      }
    },

    /**
     * Scrape `self` for metrics.
     * Should be called whenever Prometheus is scraping.
     */
    scrapeMetrics(slotClock) {
      validatorMonitorMetrics?.validatorsConnected.set(validators.size);

      // Update static metric with connected validator indices
      if (validatorMonitorMetrics?.validatorsConnectedIndices) {
        validatorMonitorMetrics.validatorsConnectedIndices.reset();
        const allIndices = Array.from(validators.keys()).sort((a, b) => a - b);
        validatorMonitorMetrics.validatorsConnectedIndices.set({indices: prettyPrintIndices(allIndices)}, 1);
      }

      const epoch = computeEpochAtSlot(slotClock);
      const slotInEpoch = slotClock % SLOTS_PER_EPOCH;

      // Only start to report on the current epoch once we've progressed past the point where
      // all attestation should be included in a block.
      //
      // This allows us to set alarms on Grafana to detect when an attestation has been
      // missed. If we didn't delay beyond the attestation inclusion period then we could
      // expect some occasional false-positives on attestation misses.
      //
      // I have chosen 3 as an arbitrary number where we *probably* shouldn't see that many
      // skip slots on mainnet.
      const previousEpoch = slotInEpoch > MIN_ATTESTATION_INCLUSION_DELAY + 3 ? epoch - 1 : epoch - 2;

      // reset() to mimic the behaviour of an aggregated .set({index})
      validatorMonitorMetrics?.prevEpochAttestations.reset();
      validatorMonitorMetrics?.prevEpochAttestationsMinDelaySeconds.reset();
      validatorMonitorMetrics?.prevEpochAttestationAggregateInclusions.reset();
      validatorMonitorMetrics?.prevEpochAttestationBlockInclusions.reset();
      validatorMonitorMetrics?.prevEpochAttestationBlockMinInclusionDistance.reset();
      validatorMonitorMetrics?.prevEpochSyncSignatureAggregateInclusions.reset();

      let validatorsInSyncCommittee = 0;
      let prevEpochSyncCommitteeHits = 0;
      let prevEpochSyncCommitteeMisses = 0;

      for (const validator of validators.values()) {
        // Participation in sync committee
        const validatorInSyncCommittee = validator.inSyncCommitteeUntilEpoch >= epoch;
        if (validatorInSyncCommittee) {
          validatorsInSyncCommittee++;
        }

        // Prev-epoch summary
        const summary = validator.summaries.get(previousEpoch);
        if (!summary) {
          continue;
        }

        // Attestations
        validatorMonitorMetrics?.prevEpochAttestations.observe(summary.attestations);
        if (summary.attestationMinDelay !== null)
          validatorMonitorMetrics?.prevEpochAttestationsMinDelaySeconds.observe(summary.attestationMinDelay);
        validatorMonitorMetrics?.prevEpochAttestationAggregateInclusions.observe(
          summary.attestationAggregateInclusions
        );
        validatorMonitorMetrics?.prevEpochAttestationBlockInclusions.observe(summary.attestationBlockInclusions);
        if (summary.attestationMinBlockInclusionDistance !== null) {
          validatorMonitorMetrics?.prevEpochAttestationBlockMinInclusionDistance.observe(
            summary.attestationMinBlockInclusionDistance
          );
        }

        // Blocks
        validatorMonitorMetrics?.prevEpochBeaconBlocks.observe(summary.blocks);
        if (summary.blockMinDelay !== null)
          validatorMonitorMetrics?.prevEpochBeaconBlocksMinDelaySeconds.observe(summary.blockMinDelay);

        // Aggregates
        validatorMonitorMetrics?.prevEpochAggregatesTotal.observe(summary.aggregates);
        if (summary.aggregateMinDelay !== null)
          validatorMonitorMetrics?.prevEpochAggregatesMinDelaySeconds.observe(summary.aggregateMinDelay);

        // Sync committee
        prevEpochSyncCommitteeHits += summary.syncCommitteeHits;
        prevEpochSyncCommitteeMisses += summary.syncCommitteeMisses;

        // Only observe if included in sync committee to prevent distorting metrics
        if (validatorInSyncCommittee) {
          validatorMonitorMetrics?.prevEpochSyncSignatureAggregateInclusions.observe(
            summary.syncSignatureAggregateInclusions
          );
        }
      }

      validatorMonitorMetrics?.validatorsInSyncCommittee.set(validatorsInSyncCommittee);
      validatorMonitorMetrics?.prevEpochSyncCommitteeHits.set(prevEpochSyncCommitteeHits);
      validatorMonitorMetrics?.prevEpochSyncCommitteeMisses.set(prevEpochSyncCommitteeMisses);
    },

    getMonitoredValidatorIndices() {
      return Array.from(validators.keys()).sort((a, b) => a - b);
    },
  };

  // Register a single collect() function to run all validatorMonitor metrics
  validatorMonitorMetrics?.validatorsConnected.addCollect(() => {
    const clockSlot = getCurrentSlot(config, genesisTime);
    validatorMonitor.scrapeMetrics(clockSlot);
  });

  return validatorMonitor;
}

/**
 * Best guess to automatically debug why validators do not achieve expected rewards.
 * Tries to answer common questions such as:
 * - Did the validator submit the attestation to this block?
 * - Was the attestation seen in an aggregate?
 * - Was the attestation seen in a block?
 */
function renderAttestationSummary(
  config: ChainForkConfig,
  rootCache: RootHexCache,
  summary: AttestationSummary | undefined,
  flags: ParticipationFlags
): string {
  // Reference https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices
  //
  // is_matching_source = data.source == justified_checkpoint
  // is_matching_target = is_matching_source and data.target.root == get_block_root(state, data.target.epoch)
  // is_matching_head = is_matching_target and data.beacon_block_root == get_block_root_at_slot(state, data.slot)
  //
  // is_matching_source MUST be true for the attestation to be included in a block
  //
  // timely_source = is_matching_source and inclusion_delay <= integer_squareroot(SLOTS_PER_EPOCH):
  // timely_target = is_matching_target and inclusion_delay <= SLOTS_PER_EPOCH:
  // timely_head = is_matching_head and inclusion_delay == MIN_ATTESTATION_INCLUSION_DELAY:

  if (flags.timelyHead) {
    // NOTE: If timelyHead everything else MUST be true also
    return "timely_head";
  }

  //
  if (flags.timelyTarget) {
    // timelyHead == false, means at least one is true
    // - attestation voted incorrect head
    // - attestation was included late

    // Note: the same attestation can be included in multiple blocks. For example, block with parent A at slot N can
    // include the attestation. Then block as slot N+1 re-orgs slot N setting as parent A and includes the attestations
    // from block at slot N.
    //
    // TODO: Track block inclusions, and then check which ones are canonical

    if (!summary) {
      // In normal conditions should never happen, validator is expected to submit an attestation to the tracking node.
      // If the validator is using multiple beacon nodes as fallback, this condition may be triggered.
      return "unexpected_timely_target_without_summary";
    }

    const canonicalBlockInclusion = summary.blockInclusions.find((block) => isCanonical(rootCache, block));
    if (!canonicalBlockInclusion) {
      // Should never happen, because for a state to exist that registers a validator's participation this specific
      // beacon node must have imported a block with the attestation that caused the change in participation.
      return "unexpected_timely_target_without_canonical_inclusion";
    }

    const {votedCorrectHeadRoot, blockSlot, attestationSlot} = canonicalBlockInclusion;
    const inclusionDistance = Math.max(blockSlot - attestationSlot - MIN_ATTESTATION_INCLUSION_DELAY, 0);

    if (votedCorrectHeadRoot && inclusionDistance === 0) {
      // Should never happen, in this case timelyHead must be true
      return "unexpected_timely_head_as_timely_target";
    }

    // Why is the distance > 0?
    // - Block that should have included the attestation was missed
    // - Attestation was not included in any aggregate
    // - Attestation was sent late

    // Why is the head vote wrong?
    // - We processed a block late and voted for the parent
    // - We voted for a block that latter was missed
    // - We voted for a block that was re-org for another chain

    let out = "timely_target";

    if (!votedCorrectHeadRoot) {
      out += "_" + whyIsHeadVoteWrong(rootCache, canonicalBlockInclusion);
    }

    if (inclusionDistance > 0) {
      out += "_" + whyIsDistanceNotOk(rootCache, canonicalBlockInclusion, summary);
    }

    return out;
  }

  //
  if (flags.timelySource) {
    // timelyTarget == false && timelySource == true means that
    // - attestation voted the wrong target but distance is <= integer_squareroot(SLOTS_PER_EPOCH)
    return "wrong_target_timely_source";
  }

  //
  // timelySource == false, either:
  // - attestation was not included in the block
  // - included in block with wrong target (very unlikely)
  // - included in block with distance > SLOTS_PER_EPOCH (very unlikely)

  // Validator failed to submit an attestation for this epoch, validator client is probably offline
  if (!summary || summary.poolSubmitDelayMinSec === null) {
    return "no_submission";
  }

  const canonicalBlockInclusion = summary.blockInclusions.find((block) => isCanonical(rootCache, block));
  if (canonicalBlockInclusion) {
    // Canonical block inclusion with no participation flags set means wrong target + late source
    return "wrong_target_late_source";
  }

  const submittedLate =
    summary.poolSubmitDelayMinSec > config.getSlotComponentDurationMs(LATE_ATTESTATION_SUBMISSION_BPS) / 1000;

  const aggregateInclusion = summary.aggregateInclusionDelaysSec.length > 0;

  if (submittedLate && aggregateInclusion) {
    return "late_submit";
  }
  if (submittedLate && !aggregateInclusion) {
    return "late_submit_no_aggregate_inclusion";
  }

  if (!submittedLate && aggregateInclusion) {
    // TODO: Why was it missed then?
    if (summary.blockInclusions.length) {
      return "block_inclusion_but_orphan";
    }
    return "aggregate_inclusion_but_missed";
    // } else if (!submittedLate && !aggregateInclusion) {
  }
  // Did the node had enough peers?
  if (summary.poolSubmitSentPeers === 0) {
    return "sent_to_zero_peers";
  }
  return "no_aggregate_inclusion";
}

function whyIsHeadVoteWrong(rootCache: RootHexCache, canonicalBlockInclusion: AttestationBlockInclusion): string {
  const {votedForMissedSlot, attestationSlot} = canonicalBlockInclusion;
  const canonicalAttestationSlotMissed = isMissedSlot(rootCache, attestationSlot);

  // __A_______C
  //    \_B1
  //      ^^ attestation slot
  //
  // We vote for B1, but the next proposer skips our voted block.
  // This scenario happens sometimes when blocks are published late

  // __A____________E
  //    \_B1__C__D
  //      ^^ attestation slot
  //
  // We vote for B1, and due to some issue a longer reorg happens orphaning our vote.
  // This scenario is considered in the above
  if (!votedForMissedSlot && canonicalAttestationSlotMissed) {
    // TODO: Did the block arrive late?
    return "vote_orphaned";
  }

  // __A__B1___C
  //    \_(A)
  //      ^^ attestation slot
  //
  // We vote for A assuming skip block, next proposer's view differs
  // This scenario happens sometimes when blocks are published late
  if (votedForMissedSlot && !canonicalAttestationSlotMissed) {
    // TODO: Did the block arrive late?
    return "wrong_skip_vote";
  }

  // __A__B2___C
  //    \_B1
  //      ^^ attestation slot
  //
  // We vote for B1, but the next proposer continues the chain on a competing block
  // This scenario is unlikely to happen in short re-orgs given no slashings, won't consider.
  //
  // __A____B_______C
  //    \    \_(B)
  //     \_(A)_(A)
  //
  // Vote for different heads on skipped slot
  return "wrong_head_vote";
}

function whyIsDistanceNotOk(
  rootCache: RootHexCache,
  canonicalBlockInclusion: AttestationBlockInclusion,
  summary: AttestationSummary
): string {
  // If the attestation is not included in any aggregate it's likely because it was sent late.
  if (summary.aggregateInclusionDelaysSec.length === 0) {
    return "no_aggregate_inclusion";
  }

  // If the next slot of an attestation is missed, distance will be > 0 even if everything else was timely
  if (isMissedSlot(rootCache, canonicalBlockInclusion.attestationSlot + 1)) {
    return "next_slot_missed";
  }

  //
  return "late_unknown";
}

/** Returns true if the state's root record includes `block` */
function isCanonical(rootCache: RootHexCache, block: AttestationBlockInclusion): boolean {
  return rootCache.getBlockRootAtSlot(block.blockSlot) === block.blockRoot;
}

/** Returns true if root at slot is the same at slot - 1 == there was no new block at slot */
function isMissedSlot(rootCache: RootHexCache, slot: Slot): boolean {
  return slot > 0 && rootCache.getBlockRootAtSlot(slot) === rootCache.getBlockRootAtSlot(slot - 1);
}

function renderBlockProposalSummary(
  config: ChainForkConfig,
  rootCache: RootHexCache,
  summary: EpochSummary | undefined,
  proposalSlot: Slot
): string {
  const proposal = summary?.blockProposals.find((proposal) => proposal.blockSlot === proposalSlot);
  if (!proposal) {
    return "not_submitted";
  }

  if (rootCache.getBlockRootAtSlot(proposalSlot) === proposal.blockRoot) {
    // Canonical state includes our block
    return "canonical";
  }

  let out = "orphaned";

  if (isMissedSlot(rootCache, proposalSlot)) {
    out += "_missed";
  }

  if (
    proposal.poolSubmitDelaySec !== null &&
    proposal.poolSubmitDelaySec > config.getSlotComponentDurationMs(LATE_BLOCK_SUBMISSION_BPS) / 1000
  ) {
    out += "_late";
  }

  return out;
}

/**
 * Cache to prevent accessing the state tree to fetch block roots repeteadly.
 * In normal network conditions the same root is read multiple times, specially the target.
 */
export class RootHexCache {
  private readonly blockRootSlotCache = new Map<Slot, RootHex>();

  constructor(private readonly state: IBeaconStateView) {}

  getBlockRootAtSlot(slot: Slot): RootHex {
    let root = this.blockRootSlotCache.get(slot);
    if (!root) {
      root = toRootHex(this.state.getBlockRootAtSlot(slot));
      this.blockRootSlotCache.set(slot, root);
    }
    return root;
  }
}

function createValidatorMonitorMetrics(register: RegistryMetricCreator) {
  return {
    validatorsConnected: register.gauge({
      name: "validator_monitor_validators",
      help: "Count of validators that are specifically monitored by this beacon node",
    }),

    validatorsConnectedIndices: register.gauge<{indices: string}>({
      name: "validator_monitor_indices",
      help: "Static metric with connected validator indices as label, value is always 1",
      labelNames: ["indices"],
    }),

    validatorsInSyncCommittee: register.gauge({
      name: "validator_monitor_validators_in_sync_committee",
      help: "Count of validators monitored by this beacon node that are part of sync committee",
    }),

    // Validator Monitor Metrics (per-epoch summaries)
    prevEpochOnChainBalance: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_balance",
      help: "Total balance of all monitored validators after an epoch",
    }),
    prevEpochOnChainAttesterHit: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_attester_hit_total",
      help: "Incremented if validator's submitted attestation is included in some blocks",
    }),
    prevEpochOnChainAttesterMiss: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_attester_miss_total",
      help: "Incremented if validator's submitted attestation is not included in any blocks",
    }),
    prevEpochOnChainSourceAttesterHit: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_source_attester_hit_total",
      help: "Incremented if the validator is flagged as a previous epoch source attester during per epoch processing",
    }),
    prevEpochOnChainSourceAttesterMiss: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_source_attester_miss_total",
      help: "Incremented if the validator is not flagged as a previous epoch source attester during per epoch processing",
    }),
    prevEpochOnChainHeadAttesterHit: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_head_attester_hit_total",
      help: "Incremented if the validator is flagged as a previous epoch head attester during per epoch processing",
    }),
    prevEpochOnChainHeadAttesterMiss: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_head_attester_miss_total",
      help: "Incremented if the validator is not flagged as a previous epoch head attester during per epoch processing",
    }),
    prevOnChainAttesterCorrectHead: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_attester_correct_head_total",
      help: "Total count of times a validator votes correct head",
    }),
    prevOnChainAttesterIncorrectHead: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_attester_incorrect_head_total",
      help: "Total count of times a validator votes incorrect head",
    }),
    prevEpochOnChainTargetAttesterHit: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_target_attester_hit_total",
      help: "Incremented if the validator is flagged as a previous epoch target attester during per epoch processing",
    }),
    prevEpochOnChainTargetAttesterMiss: register.gauge({
      name: "validator_monitor_prev_epoch_on_chain_target_attester_miss_total",
      help: "Incremented if the validator is not flagged as a previous epoch target attester during per epoch processing",
    }),
    prevEpochOnChainInclusionDistance: register.histogram({
      name: "validator_monitor_prev_epoch_on_chain_inclusion_distance",
      help: "The attestation inclusion distance calculated during per epoch processing",
      // min inclusion distance is 1, usual values are 1,2,3 max is 32 (1 epoch)
      buckets: [1, 2, 3, 5, 10, 32],
    }),
    prevEpochAttestations: register.histogram({
      name: "validator_monitor_prev_epoch_attestations",
      help: "The number of unagg. attestations seen in the previous epoch",
      buckets: [0, 1, 2, 3],
    }),
    prevEpochAttestationsMinDelaySeconds: register.histogram({
      name: "validator_monitor_prev_epoch_attestations_min_delay_seconds",
      help: "The min delay between when the validator should send the attestation and when it was received",
      buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
    }),
    prevEpochAttestationAggregateInclusions: register.histogram({
      name: "validator_monitor_prev_epoch_attestation_aggregate_inclusions",
      help: "The count of times an attestation was seen inside an aggregate",
      buckets: [0, 1, 2, 3, 5, 10],
    }),
    prevEpochAttestationBlockInclusions: register.histogram({
      name: "validator_monitor_prev_epoch_