@lodestar/beacon-node/src/sync/backfill/backfill.ts

A Typescript implementation of the beacon chain

import {EventEmitter} from "node:events";
import {StrictEventEmitter} from "strict-event-emitter-types";
import {BeaconConfig, ChainForkConfig} from "@lodestar/config";
import {SLOTS_PER_EPOCH} from "@lodestar/params";
import {IBeaconStateView, blockToHeader} from "@lodestar/state-transition";
import {Root, SignedBeaconBlock, Slot, phase0, ssz} from "@lodestar/types";
import {ErrorAborted, Logger, byteArrayEquals, sleep, toRootHex} from "@lodestar/utils";
import {IBeaconChain} from "../../chain/index.js";
import {GENESIS_SLOT, ZERO_HASH} from "../../constants/index.js";
import {IBeaconDb} from "../../db/index.js";
import {Metrics} from "../../metrics/metrics.js";
import {INetwork, NetworkEvent, NetworkEventData, PeerAction} from "../../network/index.js";
import {ItTrigger} from "../../util/itTrigger.js";
import {PeerIdStr} from "../../util/peerId.js";
import {shuffleOne} from "../../util/shuffle.js";
import {BackfillSyncError, BackfillSyncErrorCode} from "./errors.js";
import {BackfillBlock, BackfillBlockHeader, verifyBlockProposerSignature, verifyBlockSequence} from "./verify.js";

/**
 * Timeout in ms to take a break from reading a backfillBatchSize from db, as just yielding
 * to sync loop gives hardly any.
 */
const DB_READ_BREATHER_TIMEOUT = 1000;

export type BackfillSyncModules = {
  chain: IBeaconChain;
  db: IBeaconDb;
  network: INetwork;
  config: BeaconConfig;
  logger: Logger;
  metrics: Metrics | null;
  anchorState: IBeaconStateView;
  wsCheckpoint?: phase0.Checkpoint;
  signal: AbortSignal;
};

type BackfillModules = BackfillSyncModules & {
  syncAnchor: BackFillSyncAnchor;
  backfillStartFromSlot: Slot;
  prevFinalizedCheckpointBlock: BackfillBlockHeader;
  wsCheckpointHeader: BackfillBlockHeader | null;
  backfillRangeWrittenSlot: Slot | null;
};

export type BackfillSyncOpts = {
  backfillBatchSize: number;
};

export enum BackfillSyncEvent {
  completed = "BackfillSync-completed",
}

export enum BackfillSyncMethod {
  database = "database",
  backfilled_ranges = "backfilled_ranges",
  rangesync = "rangesync",
  blockbyroot = "blockbyroot",
}

export enum BackfillSyncStatus {
  pending = "pending",
  syncing = "syncing",
  completed = "completed",
  aborted = "aborted",
}

/** Map a SyncState to an integer for rendering in Grafana */
const syncStatus: {[K in BackfillSyncStatus]: number} = {
  [BackfillSyncStatus.aborted]: 0,
  [BackfillSyncStatus.pending]: 1,
  [BackfillSyncStatus.syncing]: 2,
  [BackfillSyncStatus.completed]: 3,
};

type BackfillSyncEvents = {
  [BackfillSyncEvent.completed]: (
    /** Oldest slot synced */
    oldestSlotSynced: Slot
  ) => void;
};

type BackfillSyncEmitter = StrictEventEmitter<EventEmitter, BackfillSyncEvents>;

/**
 * At any given point, we should have
 * 1. anchorBlock (with its root anchorBlockRoot at anchorSlot) for next round of sync
 *    which is the same as the lastBackSyncedBlock
 * 2. We know the anchorBlockRoot but don't have its anchorBlock and anchorSlot yet, and its
 *    parent of lastBackSyncedBlock we synced in a previous successfull round
 * 3. We just started with only anchorBlockRoot, but we know (and will validate) its anchorSlot
 */
type BackFillSyncAnchor =
  | {
      anchorBlock: SignedBeaconBlock;
      anchorBlockRoot: Root;
      anchorSlot: Slot;
      lastBackSyncedBlock: BackfillBlock;
    }
  | {anchorBlock: null; anchorBlockRoot: Root; anchorSlot: null; lastBackSyncedBlock: BackfillBlock}
  | {anchorBlock: null; anchorBlockRoot: Root; anchorSlot: Slot; lastBackSyncedBlock: null};

export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter}) {
  /** Lowest slot that we have backfilled to */
  syncAnchor: BackFillSyncAnchor;

  private readonly chain: IBeaconChain;
  private readonly network: INetwork;
  private readonly db: IBeaconDb;
  private readonly config: BeaconConfig;
  private readonly logger: Logger;
  private readonly metrics: Metrics | null;

  /**
   * Process in blocks of at max batchSize
   */
  private opts: BackfillSyncOpts;
  /**
   * If wsCheckpoint provided was in past then the (db) state from which beacon node started,
   * needs to be validated as per spec.
   *
   * 1. This could lie in between of the previous backfilled range, in which case it would be
   *    sufficient to check if its DB, once the linkage to that range has been verified.
   * 2. Else if it lies outside the backfilled range, the linkage to this checkpoint in
   *    backfill needs to be verified.
   */
  private wsCheckpointHeader: BackfillBlockHeader | null;
  private wsValidated = false;

  /**
   * From https://github.com/ethereum/consensus-specs/blob/v1.6.1/specs/phase0/weak-subjectivity.md
   *
   *
   * If
   *   1. The wsCheckpoint provided was ahead of the db's finalized checkpoint or
   *   2. There were gaps in the backfill - keys to backfillRanges are always (by construction)
   *     a) Finalized Checkpoint or b) previous wsCheckpoint
   *
   * the linkage to the previous finalized/wss checkpoint(s) needs to be verfied. If there is
   * no such checkpoint remaining, the linkage to genesis needs to be validated
   *
   * Initialize with the blockArchive's last block, and on verification update to the next
   * preceding backfillRange key's checkpoint.
   */
  private prevFinalizedCheckpointBlock: BackfillBlockHeader;
  /** Starting point that this specific backfill sync "session" started from */
  private backfillStartFromSlot: Slot;
  private backfillRangeWrittenSlot: Slot | null;

  private processor = new ItTrigger();
  private peers = new Set<PeerIdStr>();
  private status: BackfillSyncStatus = BackfillSyncStatus.pending;
  private signal: AbortSignal;

  constructor(opts: BackfillSyncOpts, modules: BackfillModules) {
    super();

    this.syncAnchor = modules.syncAnchor;
    this.backfillStartFromSlot = modules.backfillStartFromSlot;
    this.backfillRangeWrittenSlot = modules.backfillRangeWrittenSlot;
    this.prevFinalizedCheckpointBlock = modules.prevFinalizedCheckpointBlock;
    this.wsCheckpointHeader = modules.wsCheckpointHeader;

    this.chain = modules.chain;
    this.network = modules.network;
    this.db = modules.db;
    this.config = modules.config;
    this.logger = modules.logger;
    this.metrics = modules.metrics;

    this.opts = opts;
    this.network.events.on(NetworkEvent.peerConnected, this.addPeer);
    this.network.events.on(NetworkEvent.peerDisconnected, this.removePeer);
    this.signal = modules.signal;

    this.sync()
      .then((oldestSlotSynced) => {
        if (this.status !== BackfillSyncStatus.completed) {
          throw new ErrorAborted(`Invalid BackfillSyncStatus at the completion of sync loop status=${this.status}`);
        }
        this.emit(BackfillSyncEvent.completed, oldestSlotSynced);
        this.logger.info("BackfillSync completed", {oldestSlotSynced});
        // Sync completed, unsubscribe listeners and don't run the processor again.
        // Backfill is never necessary again until the node shuts down
        this.close();
      })
      .catch((e) => {
        this.logger.error("BackfillSync processor error", e);
        this.status = BackfillSyncStatus.aborted;
        this.close();
      });

    const metrics = this.metrics;
    if (metrics) {
      metrics.backfillSync.status.addCollect(() => metrics.backfillSync.status.set(syncStatus[this.status]));
      metrics.backfillSync.backfilledTillSlot.addCollect(() =>
        metrics.backfillSync.backfilledTillSlot.set(
          this.syncAnchor.lastBackSyncedBlock?.slot ?? this.backfillStartFromSlot
        )
      );
      metrics.backfillSync.prevFinOrWsSlot.addCollect(() =>
        metrics.backfillSync.prevFinOrWsSlot.set(Math.max(this.prevFinalizedCheckpointBlock.slot, GENESIS_SLOT))
      );
    }
  }

  /**
   * Use the root of the anchorState of the beacon node as the starting point of the
   * backfill sync with its expected slot to be anchorState.slot, which will be
   * validated once the block is resolved in the backfill sync.
   *
   * NOTE: init here is quite light involving couple of
   *
   *   1. db keys lookup in stateArchive/backfilledRanges
   *   2. computing root(s) for anchorBlockRoot and prevFinalizedCheckpointBlock
   *
   * The way we initialize beacon node, wsCheckpoint's slot is always <= anchorSlot
   * If:
   *   the root belonging to wsCheckpoint is in the DB, we need to verify linkage to it
   *   i.e. it becomes our first prevFinalizedCheckpointBlock
   * Else
   *   we initialize prevFinalizedCheckpointBlock from the last stored db finalized state
   *   for verification and when we go below its epoch we just check if a correct block
   *   corresponding to wsCheckpoint root was stored.
   *
   * and then we continue going back and verifying the next unconnected previous finalized
   * or wsCheckpoints identifiable as the keys of backfill sync.
   */
  static async init<T extends BackfillSync = BackfillSync>(
    opts: BackfillSyncOpts,
    modules: BackfillSyncModules
  ): Promise<T> {
    const {config, anchorState, db, wsCheckpoint, logger} = modules;

    const {checkpoint: anchorCp} = anchorState.computeAnchorCheckpoint();
    const anchorSlot = anchorState.latestBlockHeader.slot;
    const syncAnchor = {
      anchorBlock: null,
      anchorBlockRoot: anchorCp.root,
      anchorSlot,
      lastBackSyncedBlock: null,
    };

    // Load the previous written to slot for the key  backfillStartFromSlot
    // in backfilledRanges
    const backfillStartFromSlot = anchorSlot;
    const backfillRangeWrittenSlot = await db.backfilledRanges.get(backfillStartFromSlot);
    const previousBackfilledRanges = await db.backfilledRanges.entries({
      lte: backfillStartFromSlot,
    });
    modules.logger.info("Initializing from Checkpoint", {
      root: toRootHex(anchorCp.root),
      epoch: anchorCp.epoch,
      backfillStartFromSlot,
      previousBackfilledRanges: JSON.stringify(previousBackfilledRanges),
    });

    // wsCheckpointHeader is where the checkpoint can actually be validated
    const wsCheckpointHeader: BackfillBlockHeader | null = wsCheckpoint
      ? {root: wsCheckpoint.root, slot: wsCheckpoint.epoch * SLOTS_PER_EPOCH}
      : null;
    // Load a previous finalized or wsCheckpoint slot from DB below anchorSlot
    const prevFinalizedCheckpointBlock = await extractPreviousFinOrWsCheckpoint(config, db, anchorSlot, logger);

    return new BackfillSync(opts, {
      syncAnchor,
      backfillStartFromSlot,
      backfillRangeWrittenSlot,
      wsCheckpointHeader,
      prevFinalizedCheckpointBlock,
      ...modules,
    }) as T;
  }

  /** Throw / return all AsyncGenerators */
  close(): void {
    this.network.events.off(NetworkEvent.peerConnected, this.addPeer);
    this.network.events.off(NetworkEvent.peerDisconnected, this.removePeer);
    this.processor.end(new ErrorAborted("BackfillSync"));
  }

  /**
   * @returns Returns oldestSlotSynced
   */
  private async sync(): Promise<Slot> {
    this.processor.trigger();

    for await (const _ of this.processor) {
      if (this.status === BackfillSyncStatus.aborted) {
        /** Break out of sync loop and throw error */
        break;
      }
      this.status = BackfillSyncStatus.syncing;

      // 1. We should always have either anchorBlock or anchorBlockRoot, they are the
      //    anchor points for this round of the sync
      // 2. Check and validate if we have reached prevFinalizedCheckpointBlock
      //    On success Update prevFinalizedCheckpointBlock to check the *next* previous
      // 3. Validate Checkpoint as part of DB block tree if we have backfilled
      //    before the checkpoint
      // 4. Exit the sync if backfilled till genesis
      //
      // 5. Check if we can jump back from available backfill sequence, if found yield and
      //    recontinue from top making checks
      // 7. Check and read batchSize from DB, if found yield and recontinue from top
      // 8. If not in DB, and if peer available
      //    a) Either fetch blockByRoot if only anchorBlockRoot is set, which could be because
      //       i) its the unavailable root of the very first block to start off sync
      //       ii) its parent of lastBackSyncedBlock and there was an issue in establishing
      //           linear sequence in syncRange as there could be one or more
      //           skipped/orphaned slots
      //           between the parent we want to fetch and lastBackSyncedBlock
      //    b) read previous batchSize blocks from network assuming most likely those blocks
      //       form a linear anchored chain with anchorBlock. If not, try fetching the
      //       parent of
      //       the anchorBlock via strategy a) as it could be multiple skipped/orphaned slots
      //       behind
      if (this.syncAnchor.lastBackSyncedBlock != null) {
        // If after a previous sync round:
        //   lastBackSyncedBlock.slot < prevFinalizedCheckpointBlock.slot
        // then it means the prevFinalizedCheckpoint block has been missed because in each
        // round we backfill new blocks till (if the batchSize allows):
        // lastBackSyncedBlock.slot <= prevFinalizedCheckpointBlock.slot
        if (this.syncAnchor.lastBackSyncedBlock.slot < this.prevFinalizedCheckpointBlock.slot) {
          this.logger.error(
            `Backfilled till ${
              this.syncAnchor.lastBackSyncedBlock.slot
            } but not found previous saved finalized or wsCheckpoint with root=${toRootHex(
              this.prevFinalizedCheckpointBlock.root
            )}, slot=${this.prevFinalizedCheckpointBlock.slot}`
          );
          // Break sync loop and throw error
          break;
        }

        if (this.syncAnchor.lastBackSyncedBlock.slot === this.prevFinalizedCheckpointBlock.slot) {
          // Okay! we backfilled successfully till prevFinalizedCheckpointBlock
          if (!byteArrayEquals(this.syncAnchor.lastBackSyncedBlock.root, this.prevFinalizedCheckpointBlock.root)) {
            this.logger.error(
              `Invalid root synced at a previous finalized or wsCheckpoint, slot=${
                this.prevFinalizedCheckpointBlock.slot
              }: expected=${toRootHex(this.prevFinalizedCheckpointBlock.root)}, actual=${toRootHex(
                this.syncAnchor.lastBackSyncedBlock.root
              )}`
            );
            // Break sync loop and throw error
            break;
          }
          this.logger.verbose("Validated current prevFinalizedCheckpointBlock", {
            root: toRootHex(this.prevFinalizedCheckpointBlock.root),
            slot: this.prevFinalizedCheckpointBlock.slot,
          });

          // 1. If this is not a genesis block save this block in DB as this wasn't saved
          //    earlier pending validation. Genesis block will be saved with extra validation
          //    before returning from the sync.
          //
          // 2. Load another previous saved finalized or wsCheckpoint which has not
          //    been validated yet. These are the keys of backfill ranges as each
          //    range denotes
          //    a validated connected segment having the slots of previous wsCheckpoint
          //    or finalized as keys
          if (this.syncAnchor.lastBackSyncedBlock.slot !== GENESIS_SLOT) {
            await this.db.blockArchive.put(
              this.syncAnchor.lastBackSyncedBlock.slot,
              this.syncAnchor.lastBackSyncedBlock.block
            );
          }
          this.prevFinalizedCheckpointBlock = await extractPreviousFinOrWsCheckpoint(
            this.config,
            this.db,
            this.syncAnchor.lastBackSyncedBlock.slot,
            this.logger
          );
        }

        if (this.syncAnchor.lastBackSyncedBlock.slot === GENESIS_SLOT) {
          if (!byteArrayEquals(this.syncAnchor.lastBackSyncedBlock.block.message.parentRoot, ZERO_HASH)) {
            Error(
              `Invalid Gensis Block with non zero parentRoot=${toRootHex(
                this.syncAnchor.lastBackSyncedBlock.block.message.parentRoot
              )}`
            );
          }
          await this.db.blockArchive.put(GENESIS_SLOT, this.syncAnchor.lastBackSyncedBlock.block);
        }

        if (this.wsCheckpointHeader && !this.wsValidated) {
          await this.checkIfCheckpointSyncedAndValidate();
        }

        if (
          this.backfillRangeWrittenSlot === null ||
          this.syncAnchor.lastBackSyncedBlock.slot < this.backfillRangeWrittenSlot
        ) {
          this.backfillRangeWrittenSlot = this.syncAnchor.lastBackSyncedBlock.slot;
          await this.db.backfilledRanges.put(this.backfillStartFromSlot, this.backfillRangeWrittenSlot);
          this.logger.debug(
            `Updated the backfill range from=${this.backfillStartFromSlot} till=${this.backfillRangeWrittenSlot}`
          );
        }

        if (this.syncAnchor.lastBackSyncedBlock.slot === GENESIS_SLOT) {
          this.logger.verbose("Successfully synced to genesis.");
          this.status = BackfillSyncStatus.completed;
          return GENESIS_SLOT;
        }

        const foundValidSeq = await this.checkUpdateFromBackfillSequences();
        if (foundValidSeq) {
          // Go back to top and do checks till
          this.processor.trigger();
          continue;
        }
      }

      try {
        const foundBlocks = await this.fastBackfillDb();
        if (foundBlocks) {
          this.processor.trigger();
          continue;
        }
      } catch (e) {
        this.logger.error("Error while reading from DB", {}, e as Error);
        // Break sync loop and throw error
        break;
      }

      // Try the network if nothing found in DB
      const peer = shuffleOne(Array.from(this.peers.values()));
      if (!peer) {
        this.status = BackfillSyncStatus.pending;
        this.logger.debug("No peers yet");
        continue;
      }

      try {
        if (!this.syncAnchor.anchorBlock) {
          await this.syncBlockByRoot(peer, this.syncAnchor.anchorBlockRoot);

          // Go back and make the checks in case this block could be at or
          // behind prevFinalizedCheckpointBlock
        } else {
          await this.syncRange(peer);

          // Go back and make the checks in case the lastbackSyncedBlock could be at or
          // behind prevFinalizedCheckpointBlock
        }
      } catch (e) {
        this.metrics?.backfillSync.errors.inc();
        this.logger.error("Sync error", {}, e as Error);

        if (e instanceof BackfillSyncError) {
          switch (e.type.code) {
            case BackfillSyncErrorCode.INTERNAL_ERROR:
              // Break it out of the loop and throw error
              this.status = BackfillSyncStatus.aborted;
              break;
            case BackfillSyncErrorCode.NOT_ANCHORED:
            // biome-ignore lint/suspicious/noFallthroughSwitchClause: We need fall-through behavior here
            case BackfillSyncErrorCode.NOT_LINEAR:
              // Lets try to jump directly to the parent of this anchorBlock as previous
              // (segment) of blocks could be orphaned/missed
              if (this.syncAnchor.anchorBlock) {
                this.syncAnchor = {
                  anchorBlock: null,
                  anchorBlockRoot: this.syncAnchor.anchorBlock.message.parentRoot,
                  anchorSlot: null,
                  lastBackSyncedBlock: this.syncAnchor.lastBackSyncedBlock,
                };
              }

            // falls through
            case BackfillSyncErrorCode.INVALID_SIGNATURE:
              this.network.reportPeer(peer, PeerAction.LowToleranceError, "BadSyncBlocks");
          }
        }
      } finally {
        if (this.status !== BackfillSyncStatus.aborted) this.processor.trigger();
      }
    }

    throw new ErrorAborted("BackfillSync");
  }

  private addPeer = (data: NetworkEventData[NetworkEvent.peerConnected]): void => {
    const requiredSlot = this.syncAnchor.lastBackSyncedBlock?.slot ?? this.backfillStartFromSlot;
    this.logger.debug("Add peer", {peerhead: data.status.headSlot, requiredSlot});
    if (data.status.headSlot >= requiredSlot) {
      this.peers.add(data.peer);
      this.processor.trigger();
    }
  };

  private removePeer = (data: NetworkEventData[NetworkEvent.peerDisconnected]): void => {
    this.peers.delete(data.peer);
  };

  /**
   * Ensure that any weak subjectivity checkpoint provided in past with respect
   * the initialization point is the same block tree as the DB once backfill
   */
  private async checkIfCheckpointSyncedAndValidate(): Promise<void> {
    if (this.syncAnchor.lastBackSyncedBlock == null) {
      throw Error("Invalid lastBackSyncedBlock for checkpoint validation");
    }
    if (this.wsCheckpointHeader == null) {
      throw Error("Invalid null checkpoint for validation");
    }
    if (this.wsValidated) return;

    if (this.wsCheckpointHeader.slot >= this.syncAnchor.lastBackSyncedBlock.slot) {
      // Checkpoint root should be in db now , in case there are string of orphaned/missed
      // slots before/leading up to checkpoint, the block just backsynced before the
      // wsCheckpointHeader.slot will have the checkpoint root
      const wsDbCheckpointBlock = await this.db.blockArchive.getByRoot(this.wsCheckpointHeader.root);
      if (
        !wsDbCheckpointBlock ||
        // The only validation we can do here is that wsDbCheckpointBlock is found at/before
        // wsCheckpoint's epoch as there could be orphaned/missed slots all the way
        // from wsDbCheckpointBlock's slot to the wsCheckpoint's epoch

        // TODO: one can verify the child of wsDbCheckpointBlock is at
        // slot > wsCheckpointHeader
        // Note: next epoch is at wsCheckpointHeader.slot + SLOTS_PER_EPOCH
        wsDbCheckpointBlock.message.slot >= this.wsCheckpointHeader.slot + SLOTS_PER_EPOCH
      )
        // TODO: explode and stop the entire node
        throw new Error(
          `InvalidWsCheckpoint root=${toRootHex(this.wsCheckpointHeader.root)}, epoch=${
            this.wsCheckpointHeader.slot / SLOTS_PER_EPOCH
          }, ${
            wsDbCheckpointBlock
              ? "found at epoch=" + Math.floor(wsDbCheckpointBlock?.message.slot / SLOTS_PER_EPOCH)
              : "not found"
          }`
        );
      this.logger.info("wsCheckpoint validated!", {
        root: toRootHex(this.wsCheckpointHeader.root),
        epoch: this.wsCheckpointHeader.slot / SLOTS_PER_EPOCH,
      });
      this.wsValidated = true;
    }
  }

  private async checkUpdateFromBackfillSequences(): Promise<boolean> {
    if (this.syncAnchor.lastBackSyncedBlock === null) {
      throw Error("Backfill ranges can only be used once we have a valid lastBackSyncedBlock as a pivot point");
    }

    let validSequence = false;
    if (this.syncAnchor.lastBackSyncedBlock.slot === null) return validSequence;
    const lastBackSyncedSlot = this.syncAnchor.lastBackSyncedBlock.slot;

    const filteredSeqs = await this.db.backfilledRanges.entries({
      gte: lastBackSyncedSlot,
    });

    if (filteredSeqs.length > 0) {
      const jumpBackTo = Math.min(...filteredSeqs.map(({value: justToSlot}) => justToSlot));

      if (jumpBackTo < lastBackSyncedSlot) {
        validSequence = true;
        const anchorBlock = await this.db.blockArchive.get(jumpBackTo);
        if (!anchorBlock) {
          validSequence = false;
          this.logger.warn(
            `Invalid backfill sequence: expected a block at ${jumpBackTo} in blockArchive, ignoring the sequence`
          );
        }
        if (anchorBlock && validSequence && this.prevFinalizedCheckpointBlock.slot >= jumpBackTo) {
          this.logger.debug(
            `Found a sequence going back to ${jumpBackTo} before the previous finalized or wsCheckpoint`,
            {slot: this.prevFinalizedCheckpointBlock.slot}
          );

          // Everything saved in db between a backfilled range is a connected sequence
          // we only need to check if prevFinalizedCheckpointBlock is in db
          const prevBackfillCpBlock = await this.db.blockArchive.getByRoot(this.prevFinalizedCheckpointBlock.root);
          if (
            prevBackfillCpBlock != null &&
            this.prevFinalizedCheckpointBlock.slot === prevBackfillCpBlock.message.slot
          ) {
            this.logger.verbose("Validated current prevFinalizedCheckpointBlock", {
              root: toRootHex(this.prevFinalizedCheckpointBlock.root),
              slot: prevBackfillCpBlock.message.slot,
            });
          } else {
            validSequence = false;
            this.logger.warn(
              `Invalid backfill sequence: previous finalized or checkpoint block root=${toRootHex(
                this.prevFinalizedCheckpointBlock.root
              )}, slot=${this.prevFinalizedCheckpointBlock.slot} ${
                prevBackfillCpBlock ? "found at slot=" + prevBackfillCpBlock.message.slot : "not found"
              }, ignoring the sequence`
            );
          }
        }

        if (anchorBlock && validSequence) {
          // Update the current sequence in DB as we will be cleaning up previous sequences
          await this.db.backfilledRanges.put(this.backfillStartFromSlot, jumpBackTo);
          this.backfillRangeWrittenSlot = jumpBackTo;
          this.logger.verbose(
            `Jumped and updated the backfilled range ${this.backfillStartFromSlot}, ${this.backfillRangeWrittenSlot}`,
            {jumpBackTo}
          );

          const anchorBlockHeader = blockToHeader(this.config, anchorBlock.message);
          const anchorBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(anchorBlockHeader);

          this.syncAnchor = {
            anchorBlock,
            anchorBlockRoot,
            anchorSlot: jumpBackTo,
            lastBackSyncedBlock: {root: anchorBlockRoot, slot: jumpBackTo, block: anchorBlock},
          };
          if (this.prevFinalizedCheckpointBlock.slot >= jumpBackTo) {
            // prevFinalizedCheckpointBlock must have been validated, update to a
            // new unverified
            // finalized or wsCheckpoint behind the new lastBackSyncedBlock
            this.prevFinalizedCheckpointBlock = await extractPreviousFinOrWsCheckpoint(
              this.config,
              this.db,
              jumpBackTo,
              this.logger
            );
          }

          this.metrics?.backfillSync.totalBlocks.inc(
            {method: BackfillSyncMethod.backfilled_ranges},
            lastBackSyncedSlot - jumpBackTo
          );
        }
      }
    }

    // Only delete < backfillStartFromSlot, the keys greater than this would be cleaned
    // up by the archival process of forward sync
    const cleanupSeqs = filteredSeqs.filter((entry) => entry.key < this.backfillStartFromSlot);
    if (cleanupSeqs.length > 0) {
      await this.db.backfilledRanges.batchDelete(cleanupSeqs.map((entry) => entry.key));
      this.logger.debug(
        `Cleaned up the old sequences between ${this.backfillStartFromSlot},${toRootHex(
          this.syncAnchor.lastBackSyncedBlock.root
        )}`,
        {cleanupSeqs: JSON.stringify(cleanupSeqs)}
      );
    }

    return validSequence;
  }

  private async fastBackfillDb(): Promise<boolean> {
    // Block of this anchorBlockRoot can't be behind the prevFinalizedCheckpointBlock
    // as prevFinalizedCheckpointBlock can't be skipped
    let anchorBlockRoot: Root;
    let expectedSlot: Slot | null = null;
    if (this.syncAnchor.anchorBlock) {
      anchorBlockRoot = this.syncAnchor.anchorBlock.message.parentRoot;
    } else {
      anchorBlockRoot = this.syncAnchor.anchorBlockRoot;
      expectedSlot = this.syncAnchor.anchorSlot;
    }
    let anchorBlock = await this.db.blockArchive.getByRoot(anchorBlockRoot);
    if (!anchorBlock) return false;

    if (expectedSlot !== null && anchorBlock.message.slot !== expectedSlot)
      throw Error(
        `Invalid slot of anchorBlock read from DB with root=${toRootHex(
          anchorBlockRoot
        )}, expected=${expectedSlot}, actual=${anchorBlock.message.slot}`
      );

    // If possible, read back till anchorBlock > this.prevFinalizedCheckpointBlock
    let parentBlock: SignedBeaconBlock | null,
      backCount = 1;

    let isPrevFinWsConfirmedAnchorParent = false;
    while (
      backCount !== this.opts.backfillBatchSize &&
      // biome-ignore lint/suspicious/noAssignInExpressions: May be refactored later
      (parentBlock = await this.db.blockArchive.getByRoot(anchorBlock.message.parentRoot))
    ) {
      // Before moving anchorBlock back, we need check for prevFinalizedCheckpointBlock
      if (anchorBlock.message.slot < this.prevFinalizedCheckpointBlock.slot) {
        throw Error(
          `Skipped a prevFinalizedCheckpointBlock with slot=${toRootHex(
            this.prevFinalizedCheckpointBlock.root
          )}, root=${toRootHex(this.prevFinalizedCheckpointBlock.root)}`
        );
      }
      if (anchorBlock.message.slot === this.prevFinalizedCheckpointBlock.slot) {
        if (
          !isPrevFinWsConfirmedAnchorParent &&
          !byteArrayEquals(anchorBlockRoot, this.prevFinalizedCheckpointBlock.root)
        ) {
          throw Error(
            `Invalid root for prevFinalizedCheckpointBlock at slot=${
              this.prevFinalizedCheckpointBlock.slot
            }, expected=${toRootHex(this.prevFinalizedCheckpointBlock.root)}, found=${toRootHex(anchorBlockRoot)}`
          );
        }

        // If the new parentBlock is just one slot back, we can safely assign
        // prevFinalizedCheckpointBlock with the parentBlock and skip root
        // validation in next iteration. Else we need to extract
        // prevFinalizedCheckpointBlock
        if (parentBlock.message.slot === anchorBlock.message.slot - 1) {
          this.prevFinalizedCheckpointBlock = {root: anchorBlock.message.parentRoot, slot: parentBlock.message.slot};
          isPrevFinWsConfirmedAnchorParent = true;
        } else {
          // Extract new prevFinalizedCheckpointBlock below anchorBlock
          this.prevFinalizedCheckpointBlock = await extractPreviousFinOrWsCheckpoint(
            this.config,
            this.db,
            anchorBlock.message.slot,
            this.logger
          );
          isPrevFinWsConfirmedAnchorParent = false;
        }
      }
      anchorBlockRoot = anchorBlock.message.parentRoot;
      anchorBlock = parentBlock;
      backCount++;
    }

    this.syncAnchor = {
      anchorBlock,
      anchorBlockRoot,
      anchorSlot: anchorBlock.message.slot,
      lastBackSyncedBlock: {root: anchorBlockRoot, slot: anchorBlock.message.slot, block: anchorBlock},
    };
    this.metrics?.backfillSync.totalBlocks.inc({method: BackfillSyncMethod.database}, backCount);
    this.logger.verbose(`Read ${backCount} blocks from DB till `, {
      slot: anchorBlock.message.slot,
    });
    if (backCount >= this.opts.backfillBatchSize) {
      // We should sleep as there seems to be more that can be read from db but yielding to
      // the sync loop hardly gives any breather to the beacon node
      await sleep(DB_READ_BREATHER_TIMEOUT, this.signal);
    }
    return true;
  }

  private async syncBlockByRoot(peer: PeerIdStr, anchorBlockRoot: Root): Promise<void> {
    const res = await this.network.sendBeaconBlocksByRoot(peer, [anchorBlockRoot]);
    if (res.length < 1) throw new Error("InvalidBlockSyncedFromPeer");
    const anchorBlock = res[0];

    // GENESIS_SLOT doesn't has valid signature
    if (anchorBlock.message.slot === GENESIS_SLOT) return;
    await verifyBlockProposerSignature(this.chain.config, this.chain.bls, [anchorBlock]);

    // We can write to the disk if this is ahead of prevFinalizedCheckpointBlock otherwise
    // we will need to go make checks on the top of sync loop before writing as it might
    // override prevFinalizedCheckpointBlock
    if (this.prevFinalizedCheckpointBlock.slot < anchorBlock.message.slot) {
      const serialized = this.chain.serializedCache.get(anchorBlock);
      if (serialized) {
        await this.db.blockArchive.putBinary(anchorBlock.message.slot, serialized);
      } else {
        await this.db.blockArchive.put(anchorBlock.message.slot, anchorBlock);
      }
    }

    this.syncAnchor = {
      anchorBlock: anchorBlock,
      anchorBlockRoot,
      anchorSlot: anchorBlock.message.slot,
      lastBackSyncedBlock: {root: anchorBlockRoot, slot: anchorBlock.message.slot, block: anchorBlock},
    };

    this.metrics?.backfillSync.totalBlocks.inc({method: BackfillSyncMethod.blockbyroot});

    this.logger.verbose("Fetched new anchorBlock", {
      root: toRootHex(anchorBlockRoot),
      slot: anchorBlock.message.slot,
    });

    return;
  }

  private async syncRange(peer: PeerIdStr): Promise<void> {
    if (!this.syncAnchor.anchorBlock) {
      throw Error("Invalid anchorBlock null for syncRange");
    }

    const toSlot = this.syncAnchor.anchorBlock.message.slot;
    const fromSlot = Math.max(
      toSlot - this.opts.backfillBatchSize,
      this.prevFinalizedCheckpointBlock.slot,
      GENESIS_SLOT
    );
    const blocks = await this.network.sendBeaconBlocksByRange(peer, {
      startSlot: fromSlot,
      count: toSlot - fromSlot,
      step: 1,
    });

    const anchorParentRoot = this.syncAnchor.anchorBlock.message.parentRoot;

    if (blocks.length === 0) {
      // Lets just directly try to jump to anchorParentRoot
      this.syncAnchor = {
        anchorBlock: null,
        anchorBlockRoot: anchorParentRoot,
        anchorSlot: null,
        lastBackSyncedBlock: this.syncAnchor.lastBackSyncedBlock,
      };
      return;
    }

    const {nextAnchor, verifiedBlocks, error} = verifyBlockSequence(this.config, blocks, anchorParentRoot);

    // If any of the block's proposer signature fail, we can't trust this peer at all
    if (verifiedBlocks.length > 0) {
      await verifyBlockProposerSignature(this.chain.config, this.chain.bls, verifiedBlocks);

      // This is bad, like super bad. Abort the backfill
      if (!nextAnchor)
        throw new BackfillSyncError({
          code: BackfillSyncErrorCode.INTERNAL_ERROR,
          reason: "Invalid verifyBlockSequence result",
        });

      // Verified blocks are in reverse order with the nextAnchor being the smallest slot
      // if nextAnchor is on the same slot as prevFinalizedCheckpointBlock, we can't save
      // it before returning to top of sync loop for validation
      const blocksToPut =
        nextAnchor.slot > this.prevFinalizedCheckpointBlock.slot
          ? verifiedBlocks
          : verifiedBlocks.slice(0, verifiedBlocks.length - 1);

      const binaryPuts = [];
      const nonBinaryPuts = [];

      for (const block of blocksToPut) {
        const serialized = this.chain.serializedCache.get(block);
        const item = {
          key: block.message.slot,
          slot: block.message.slot,
          blockRoot: this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message),
          parentRoot: block.message.parentRoot,
        };

        if (serialized) {
          binaryPuts.push({...item, value: serialized});
        } else {
          nonBinaryPuts.push({...item, value: block});
        }
      }

      if (binaryPuts.length > 0) {
        await this.db.blockArchive.batchPutBinary(binaryPuts);
      }
      if (nonBinaryPuts.length > 0) {
        await this.db.blockArchive.batchPut(nonBinaryPuts);
      }

      this.metrics?.backfillSync.totalBlocks.inc({method: BackfillSyncMethod.rangesync}, verifiedBlocks.length);
    }

    // If nextAnchor provided, found some linear anchored blocks
    if (nextAnchor !== null) {
      this.syncAnchor = {
        anchorBlock: nextAnchor.block,
        anchorBlockRoot: nextAnchor.root,
        anchorSlot: nextAnchor.slot,
        lastBackSyncedBlock: nextAnchor,
      };
      this.logger.verbose(`syncRange discovered ${verifiedBlocks.length} valid blocks`, {
        backfilled: this.syncAnchor.lastBackSyncedBlock.slot,
      });
    }
    if (error != null) throw new BackfillSyncError({code: error});
  }
}

async function extractPreviousFinOrWsCheckpoint(
  config: ChainForkConfig,
  db: IBeaconDb,
  belowSlot: Slot,
  logger?: Logger
): Promise<BackfillBlockHeader> {
  // Anything below genesis block is just zero hash
  if (belowSlot <= GENESIS_SLOT) return {root: ZERO_HASH, slot: belowSlot - 1};

  // To extract the next prevFinalizedCheckpointBlock, we just need to look back in DB
  // Any saved previous finalized or ws checkpoint, will also have a corresponding block
  // saved in DB, as we make sure of that
  //   1. When we archive new finalized state and blocks
  //   2. When we backfill from a wsCheckpoint
  const nextPrevFinOrWsBlock = (
    await db.blockArchive.values({
      lt: belowSlot,
      reverse: true,
      limit: 1,
    })
  )[0];

  let prevFinalizedCheckpointBlock: BackfillBlockHeader;
  if (nextPrevFinOrWsBlock != null) {
    const header = blockToHeader(config, nextPrevFinOrWsBlock.message);
    const root = ssz.phase0.BeaconBlockHeader.hashTreeRoot(header);
    prevFinalizedCheckpointBlock = {root, slot: nextPrevFinOrWsBlock.message.slot};
    logger?.debug("Extracted new prevFinalizedCheckpointBlock as potential previous finalized or wsCheckpoint", {
      root: toRootHex(prevFinalizedCheckpointBlock.root),
      slot: prevFinalizedCheckpointBlock.slot,
    });
  } else {
    // GENESIS_SLOT -1 is the placeholder for parentHash of the genesis block
    // which should always be ZERO_HASH.
    prevFinalizedCheckpointBlock = {root: ZERO_HASH, slot: GENESIS_SLOT - 1};
  }
  return prevFinalizedCheckpointBlock;
}