@hashgraph/solo/src/commands/block-node.ts

An opinionated CLI tool to deploy and manage private Hedera Networks.

// SPDX-License-Identifier: Apache-2.0

import {Listr} from 'listr2';
import {SoloError} from '../core/errors/solo-error.js';
import * as helpers from '../core/helpers.js';
import {
  checkDockerImageExists,
  createAndCopyBlockNodeJsonFileForConsensusNode,
  showVersionBanner,
  sleep,
} from '../core/helpers.js';
import * as constants from '../core/constants.js';
import {BaseCommand} from './base.js';
import {Flags as flags} from './flags.js';
import {type AnyListrContext, type ArgvStruct, type NodeAlias} from '../types/aliases.js';
import {ListrLock} from '../core/lock/listr-lock.js';
import {
  type ClusterReferenceName,
  type ComponentId,
  type Context,
  type DeploymentName,
  type Optional,
  type SoloListr,
  type SoloListrTask,
  type SoloListrTaskWrapper,
} from '../types/index.js';
import * as versions from '../../version.js';
import {MINIMUM_HIERO_BLOCK_NODE_VERSION_FOR_NEW_LIVENESS_CHECK_PORT} from '../../version.js';
import {type CommandFlag, type CommandFlags} from '../types/flag-types.js';
import {type Lock} from '../core/lock/lock.js';
import {NamespaceName} from '../types/namespace/namespace-name.js';
import {ContainerReference} from '../integration/kube/resources/container/container-reference.js';
import {Duration} from '../core/time/duration.js';
import {type PodReference} from '../integration/kube/resources/pod/pod-reference.js';
import chalk from 'chalk';
import {type Pod} from '../integration/kube/resources/pod/pod.js';
import {type BlockNodeStateSchema} from '../data/schema/model/remote/state/block-node-state-schema.js';
import {ComponentTypes} from '../core/config/remote/enumerations/component-types.js';
import {injectable} from 'tsyringe-neo';
import {Templates} from '../core/templates.js';
import {SemanticVersion} from '../business/utils/semantic-version.js';
import {assertUpgradeVersionNotOlder} from '../core/upgrade-version-guard.js';
import {PvcReference} from '../integration/kube/resources/pvc/pvc-reference.js';
import {PvcName} from '../integration/kube/resources/pvc/pvc-name.js';
import {LedgerPhase} from '../data/schema/model/remote/ledger-phase.js';
import {DeploymentStateSchema} from '../data/schema/model/remote/deployment-state-schema.js';
import {ConsensusNode} from '../core/model/consensus-node.js';
import {type ClusterSchema} from '../data/schema/model/common/cluster-schema.js';
import {ExternalBlockNodeStateSchema} from '../data/schema/model/remote/state/external-block-node-state-schema.js';
import {DeploymentPhase} from '../data/schema/model/remote/deployment-phase.js';
import {
  ComponentUpgradeMigrationRules,
  type ComponentUpgradeMigrationStep,
} from './migrations/component-upgrade-rules.js';
import {optionFromFlag} from './command-helpers.js';

interface BlockNodeDeployConfigClass {
  chartVersion: string;
  chartDirectory: string;
  blockNodeChartDirectory: string;
  blockNodeTssOverlay: boolean;
  clusterRef: ClusterReferenceName;
  deployment: DeploymentName;
  devMode: boolean;
  domainName: Optional<string>;
  enableIngress: boolean;
  quiet: boolean;
  valuesFile: Optional<string>;
  releaseTag: string;
  imageTag: Optional<string>;
  namespace: NamespaceName;
  context: string;
  valuesArg: string;
  newBlockNodeComponent: BlockNodeStateSchema;
  releaseName: string;
  livenessCheckPort: number;
  priorityMapping: Record<NodeAlias, number>;
}

interface BlockNodeDeployContext {
  config: BlockNodeDeployConfigClass;
}

interface BlockNodeDestroyConfigClass {
  chartDirectory: string;
  clusterRef: ClusterReferenceName;
  deployment: DeploymentName;
  devMode: boolean;
  quiet: boolean;
  namespace: NamespaceName;
  context: string;
  isChartInstalled: boolean;
  valuesArg: string;
  releaseName: string;
  id: number;
  isLegacyChartInstalled: boolean;
}

interface BlockNodeDestroyContext {
  config: BlockNodeDestroyConfigClass;
}

interface BlockNodeUpgradeConfigClass {
  chartDirectory: string;
  blockNodeChartDirectory: string;
  blockNodeTssOverlay: boolean;
  clusterRef: ClusterReferenceName;
  deployment: DeploymentName;
  devMode: boolean;
  quiet: boolean;
  valuesFile: Optional<string>;
  namespace: NamespaceName;
  context: string;
  releaseName: string;
  upgradeVersion: string;
  currentVersion: string;
  migrationPlan: ComponentUpgradeMigrationStep[];
  valuesArg: string;
  id: number;
  isLegacyChartInstalled: boolean;
  /** Set by recreateBlockNodeChart; used by the readiness check to ignore the terminating predecessor pod. */
  recreateInstallTime?: Date;
}

interface BlockNodeUpgradeContext {
  config: BlockNodeUpgradeConfigClass;
}

interface BlockNodeAddExternalConfigClass {
  clusterRef: ClusterReferenceName;
  deployment: DeploymentName;
  devMode: boolean;
  quiet: boolean;
  context: string;
  externalBlockNodeAddress: string;
  newExternalBlockNodeComponent: ExternalBlockNodeStateSchema;
  namespace: NamespaceName;
  priorityMapping: Record<NodeAlias, number>;
}

interface BlockNodeAddExternalContext {
  config: BlockNodeAddExternalConfigClass;
}

interface BlockNodeDeleteExternalConfigClass {
  clusterRef: ClusterReferenceName;
  deployment: DeploymentName;
  devMode: boolean;
  quiet: boolean;
  namespace: NamespaceName;
  context: string;
  id: number;
}

interface BlockNodeDeleteExternalContext {
  config: BlockNodeDeleteExternalConfigClass;
}

interface InferredData {
  id: ComponentId;
  releaseName: string;
  isChartInstalled: boolean;
  isLegacyChartInstalled: boolean;
}

@injectable()
export class BlockNodeCommand extends BaseCommand {
  public constructor() {
    super();
  }

  private static readonly ADD_CONFIGS_NAME: string = 'addConfigs';

  private static readonly DESTROY_CONFIGS_NAME: string = 'destroyConfigs';

  private static readonly UPGRADE_CONFIGS_NAME: string = 'upgradeConfigs';

  private static readonly ADD_EXTERNAL_CONFIGS_NAME: string = 'addExternalConfigs';

  private static readonly DELETE_CONFIGS_NAME: string = 'deleteExternalConfigs';
  private static readonly MIGRATION_COMPONENT_KEY: string = 'block-node';

  public static readonly ADD_FLAGS_LIST: CommandFlags = {
    required: [flags.deployment],
    optional: [
      flags.blockNodeChartVersion,
      flags.blockNodeChartDirectory,
      flags.blockNodeTssOverlay,
      flags.chartDirectory,
      flags.clusterRef,
      flags.devMode,
      flags.domainName,
      flags.enableIngress,
      flags.quiet,
      flags.valuesFile,
      flags.releaseTag,
      flags.imageTag,
      flags.priorityMapping,
    ],
  };

  public static readonly ADD_EXTERNAL_FLAGS_LIST: CommandFlags = {
    required: [flags.deployment, flags.externalBlockNodeAddress],
    optional: [flags.clusterRef, flags.devMode, flags.quiet, flags.priorityMapping],
  };

  public static readonly DELETE_EXTERNAL_FLAGS_LIST: CommandFlags = {
    required: [flags.deployment],
    optional: [flags.clusterRef, flags.devMode, flags.force, flags.quiet, flags.id],
  };

  public static readonly DESTROY_FLAGS_LIST: CommandFlags = {
    required: [flags.deployment],
    optional: [flags.chartDirectory, flags.clusterRef, flags.devMode, flags.force, flags.quiet, flags.id],
  };

  public static readonly UPGRADE_FLAGS_LIST: CommandFlags = {
    required: [flags.deployment],
    optional: [
      flags.chartDirectory,
      flags.blockNodeChartDirectory,
      flags.clusterRef,
      flags.devMode,
      flags.force,
      flags.quiet,
      flags.valuesFile,
      flags.upgradeVersion,
      flags.id,
    ],
  };

  private async prepareValuesArgForBlockNode(
    config: BlockNodeDeployConfigClass | BlockNodeUpgradeConfigClass,
  ): Promise<string> {
    let valuesArgument: string = '';

    valuesArgument += helpers.prepareValuesFiles(constants.BLOCK_NODE_VALUES_FILE);

    // Block node can be deployed before consensus deploy persists tssEnabled into remote config.
    // The explicit CLI switch allows users to opt into TSS sizing and message limits in that order-of-operations.
    if (this.remoteConfig.configuration.state.tssEnabled || config.blockNodeTssOverlay) {
      valuesArgument += helpers.prepareValuesFiles(constants.BLOCK_NODE_TSS_VALUES_FILE);
    }

    if (config.valuesFile) {
      valuesArgument += helpers.prepareValuesFiles(config.valuesFile);
    }

    valuesArgument += helpers.populateHelmArguments({nameOverride: config.releaseName});

    // Only handle domainName and imageTag for deploy config (not upgrade config)
    if ('domainName' in config && config.domainName) {
      valuesArgument += helpers.populateHelmArguments({
        'ingress.enabled': true,
        'ingress.hosts[0].host': config.domainName,
        'ingress.hosts[0].paths[0].path': '/',
        'ingress.hosts[0].paths[0].pathType': 'ImplementationSpecific',
      });
    }

    if ('imageTag' in config && config.imageTag) {
      config.imageTag = SemanticVersion.getValidSemanticVersion(config.imageTag, false, 'Block node image tag');
      if (!checkDockerImageExists(constants.BLOCK_NODE_IMAGE_NAME, config.imageTag)) {
        throw new SoloError(`Local block node image with tag "${config.imageTag}" does not exist.`);
      }
      // use local image from docker engine
      valuesArgument += helpers.populateHelmArguments({
        'image.repository': constants.BLOCK_NODE_IMAGE_NAME,
        'image.tag': config.imageTag,
        'image.pullPolicy': 'Never',
      });
    }

    const {state, clusters} = this.remoteConfig.configuration;

    for (const [index, blockNode] of state.blockNodes.entries()) {
      const cluster: ClusterSchema = clusters.find(({name}): boolean => name === blockNode.metadata.cluster);

      const fqdn: string = Templates.renderSvcFullyQualifiedDomainName(
        `block-node-${blockNode.metadata.id}`,
        config.namespace.name,
        cluster.dnsBaseDomain,
      );

      valuesArgument += helpers.populateHelmArguments({
        [`blockNode.sources[${index}].address`]: fqdn,
        [`blockNode.sources[${index}].port`]: constants.BLOCK_NODE_PORT,
        [`blockNode.sources[${index}].priority`]: 1,
      });
    }

    return valuesArgument;
  }

  private static appendExtraCommandArgs(baseArgument: string, extraCommandArguments: string[]): string {
    if (extraCommandArguments.length === 0) {
      return baseArgument;
    }
    return `${baseArgument} ${extraCommandArguments.join(' ')}`.trim();
  }

  private getReleaseName(): string {
    return this.renderReleaseName(
      this.remoteConfig.configuration.components.getNewComponentId(ComponentTypes.BlockNode),
    );
  }

  private renderReleaseName(id: ComponentId): string {
    if (typeof id !== 'number') {
      throw new SoloError(`Invalid component id: ${id}, type: ${typeof id}`);
    }
    return `${constants.BLOCK_NODE_RELEASE_NAME}-${id}`;
  }

  private updateConsensusNodesInRemoteConfig(): SoloListrTask<BlockNodeDeployContext> {
    return {
      title: 'Update consensus nodes in remote config',
      task: async ({config: {newBlockNodeComponent, priorityMapping}}): Promise<void> => {
        const state: DeploymentStateSchema = this.remoteConfig.configuration.state;
        const nodeAliases: string[] = Object.keys(priorityMapping);

        for (const node of state.consensusNodes.filter((node): boolean =>
          nodeAliases.includes(Templates.renderNodeAliasFromNumber(node.metadata.id)),
        )) {
          const priority: number = priorityMapping[Templates.renderNodeAliasFromNumber(node.metadata.id)];

          node.blockNodeMap.push([newBlockNodeComponent.metadata.id, priority]);
        }

        await this.remoteConfig.persist();
      },
    };
  }

  private updateConsensusNodesPostGenesis(): SoloListrTask<BlockNodeDeployContext> {
    return {
      title: 'Copy block-nodes.json to consensus nodes',
      task: async ({config: {priorityMapping}}): Promise<void> => {
        const nodeAliases: string[] = Object.keys(priorityMapping);

        const filteredConsensusNodes: ConsensusNode[] = this.remoteConfig
          .getConsensusNodes()
          .filter((node): boolean => nodeAliases.includes(node.name));

        for (const node of filteredConsensusNodes) {
          await createAndCopyBlockNodeJsonFileForConsensusNode(node, this.logger, this.k8Factory);
        }
      },
    };
  }

  private updateConsensusNodesPostGenesisForExternal(): SoloListrTask<BlockNodeAddExternalContext> {
    return {
      title: 'Copy block-nodes.json to consensus nodes',
      task: async ({config: {priorityMapping}}): Promise<void> => {
        const nodeAliases: string[] = Object.keys(priorityMapping);

        const filteredConsensusNodes: ConsensusNode[] = this.remoteConfig
          .getConsensusNodes()
          .filter((node): boolean => nodeAliases.includes(node.name));

        for (const node of filteredConsensusNodes) {
          await createAndCopyBlockNodeJsonFileForConsensusNode(node, this.logger, this.k8Factory);
        }
      },
    };
  }

  private handleConsensusNodeUpdating(): SoloListrTask<BlockNodeDeployContext> {
    return {
      title: 'Update consensus nodes',
      task: (_, task): SoloListr<BlockNodeDeployContext> => {
        const subTasks: SoloListrTask<BlockNodeDeployContext>[] = [this.updateConsensusNodesInRemoteConfig()];

        if (this.remoteConfig.configuration.state.ledgerPhase !== LedgerPhase.UNINITIALIZED) {
          subTasks.push(this.updateConsensusNodesPostGenesis());
        }

        return task.newListr(subTasks, constants.LISTR_DEFAULT_OPTIONS.DEFAULT);
      },
    };
  }

  private updateConsensusNodesInRemoteConfigForExternalBlockNode(): SoloListrTask<BlockNodeAddExternalContext> {
    return {
      title: 'Update consensus nodes in remote config',
      task: async ({config: {newExternalBlockNodeComponent, priorityMapping}}): Promise<void> => {
        const state: DeploymentStateSchema = this.remoteConfig.configuration.state;
        const nodeAliases: string[] = Object.keys(priorityMapping);

        for (const node of state.consensusNodes.filter((node): boolean =>
          nodeAliases.includes(Templates.renderNodeAliasFromNumber(node.metadata.id)),
        )) {
          const priority: number = priorityMapping[Templates.renderNodeAliasFromNumber(node.metadata.id)];

          node.externalBlockNodeMap.push([newExternalBlockNodeComponent.id, priority]);
        }

        this.remoteConfig.configuration.state.consensusNodes = state.consensusNodes;

        await this.remoteConfig.persist();
      },
    };
  }

  private handleConsensusNodeUpdatingForExternalBlockNode(): SoloListrTask<BlockNodeAddExternalContext> {
    return {
      title: 'Update consensus nodes',
      task: (_, task): SoloListr<BlockNodeAddExternalContext> => {
        const subTasks: SoloListrTask<BlockNodeAddExternalContext>[] = [
          this.updateConsensusNodesInRemoteConfigForExternalBlockNode(),
        ];

        if (this.remoteConfig.configuration.state.ledgerPhase !== LedgerPhase.UNINITIALIZED) {
          subTasks.push(this.updateConsensusNodesPostGenesisForExternal());
        }

        return task.newListr(subTasks, constants.LISTR_DEFAULT_OPTIONS.DEFAULT);
      },
    };
  }

  public async add(argv: ArgvStruct): Promise<boolean> {
    let lease: Lock;

    const tasks: SoloListr<BlockNodeDeployContext> = this.taskList.newTaskList<BlockNodeDeployContext>(
      [
        {
          title: 'Initialize',
          task: async (context_, task): Promise<Listr<AnyListrContext>> => {
            await this.localConfig.load();
            await this.loadRemoteConfigOrWarn(argv);
            if (!this.oneShotState.isActive()) {
              lease = await this.leaseManager.create();
            }

            this.configManager.update(argv);

            flags.disablePrompts(BlockNodeCommand.ADD_FLAGS_LIST.optional);

            const allFlags: CommandFlag[] = [
              ...BlockNodeCommand.ADD_FLAGS_LIST.required,
              ...BlockNodeCommand.ADD_FLAGS_LIST.optional,
            ];

            await this.configManager.executePrompt(task, allFlags);

            const config: BlockNodeDeployConfigClass = this.configManager.getConfig(
              BlockNodeCommand.ADD_CONFIGS_NAME,
              allFlags,
            ) as BlockNodeDeployConfigClass;

            context_.config = config;

            // check if block node version compatible with current hedera platform version
            let consensusNodeVersion: string = this.remoteConfig.configuration.versions.consensusNode.toString();
            if (consensusNodeVersion === '0.0.0') {
              // if is possible block node deployed before consensus node, then use release tag as fallback
              consensusNodeVersion = config.releaseTag;
            }

            const currentVersion: SemanticVersion<string> = new SemanticVersion(consensusNodeVersion);
            const minimumVersion: SemanticVersion<string> = new SemanticVersion(
              versions.MINIMUM_HIERO_PLATFORM_VERSION_FOR_BLOCK_NODE,
            );

            if (currentVersion.lessThan(minimumVersion)) {
              throw new SoloError(
                `Current version is ${consensusNodeVersion}, Hedera platform versions less than ${versions.MINIMUM_HIERO_PLATFORM_VERSION_FOR_BLOCK_NODE_LEGACY_RELEASE} are not supported`,
              );
            }

            config.namespace = await this.getNamespace(task);
            config.clusterRef = this.getClusterReference();
            config.context = this.getClusterContext(config.clusterRef);

            config.priorityMapping = Templates.parseBlockNodePriorityMapping(
              config.priorityMapping as unknown as string,
              this.remoteConfig.getConsensusNodes(),
            );

            const currentBlockNodeVersion: SemanticVersion<string> = new SemanticVersion(config.chartVersion);
            const consensusNodeSemanticVersion: SemanticVersion<string> = new SemanticVersion(consensusNodeVersion);
            if (
              consensusNodeSemanticVersion.lessThan(
                new SemanticVersion(versions.MINIMUM_HIERO_PLATFORM_VERSION_FOR_BLOCK_NODE),
              ) &&
              currentBlockNodeVersion.greaterThanOrEqual(MINIMUM_HIERO_BLOCK_NODE_VERSION_FOR_NEW_LIVENESS_CHECK_PORT)
            ) {
              throw new SoloError(
                `Current platform version is ${consensusNodeVersion}, Hedera platform version less than ${versions.MINIMUM_HIERO_PLATFORM_VERSION_FOR_BLOCK_NODE} ` +
                  `are not supported for block node version ${MINIMUM_HIERO_BLOCK_NODE_VERSION_FOR_NEW_LIVENESS_CHECK_PORT.toString()}`,
              );
            }

            config.chartVersion = SemanticVersion.getValidSemanticVersion(
              config.chartVersion,
              false,
              'Block node chart version',
            );

            config.livenessCheckPort = this.getLivenessCheckPortNumber(config.chartVersion, config.imageTag);

            if (!this.oneShotState.isActive()) {
              return ListrLock.newAcquireLockTask(lease, task);
            }
            return ListrLock.newSkippedLockTask(task);
          },
        },
        {
          title: 'Prepare release name and block node name',
          task: async ({config}): Promise<void> => {
            config.releaseName = this.getReleaseName();

            config.newBlockNodeComponent = this.componentFactory.createNewBlockNodeComponent(
              config.clusterRef,
              config.namespace,
            );

            config.newBlockNodeComponent.metadata.phase = DeploymentPhase.REQUESTED;
          },
        },
        this.addBlockNodeComponent(),
        {
          title: 'Prepare chart values',
          task: async ({config}): Promise<void> => {
            config.valuesArg = await this.prepareValuesArgForBlockNode(config);
          },
        },
        {
          title: 'Deploy block node',
          task: async ({config}, task): Promise<void> => {
            const {
              context,
              namespace,
              releaseName,
              chartVersion,
              valuesArg,
              clusterRef,
              imageTag,
              blockNodeChartDirectory,
              newBlockNodeComponent,
            } = config;

            await this.chartManager.install(
              namespace,
              releaseName,
              constants.BLOCK_NODE_CHART,
              blockNodeChartDirectory || constants.BLOCK_NODE_CHART_URL,
              chartVersion,
              valuesArg,
              context,
            );

            this.remoteConfig.configuration.components.changeComponentPhase(
              newBlockNodeComponent.metadata.id,
              ComponentTypes.BlockNode,
              DeploymentPhase.DEPLOYED,
            );

            await this.remoteConfig.persist();

            if (imageTag) {
              // update config map with new VERSION info since
              // it will be used as a critical environment variable by block node
              const blockNodeStateSchema: BlockNodeStateSchema = this.componentFactory.createNewBlockNodeComponent(
                clusterRef,
                namespace,
              );
              const blockNodeId: ComponentId = blockNodeStateSchema.metadata.id;

              const name: string = `block-node-${blockNodeId}-config`;
              const data: Record<string, string> = {VERSION: imageTag};

              await this.k8Factory.getK8(context).configMaps().update(namespace, name, data);

              task.title += ` with local built image (${imageTag})`;
            }

            showVersionBanner(this.logger, releaseName, chartVersion);

            await this.updateBlockNodeVersionInRemoteConfig(config);
          },
        },
        {
          title: 'Check block node pod is running',
          task: async ({config}): Promise<void> => {
            await this.k8Factory
              .getK8(config.context)
              .pods()
              .waitForRunningPhase(
                config.namespace,
                Templates.renderBlockNodeLabels(config.newBlockNodeComponent.metadata.id),
                constants.BLOCK_NODE_PODS_RUNNING_MAX_ATTEMPTS,
                constants.BLOCK_NODE_PODS_RUNNING_DELAY,
              );
          },
        },
        {
          title: 'Check software',
          task: async ({config: {newBlockNodeComponent, context, namespace}}): Promise<void> => {
            const labels: string[] = Templates.renderBlockNodeLabels(newBlockNodeComponent.metadata.id);

            const blockNodePods: Pod[] = await this.k8Factory.getK8(context).pods().list(namespace, labels);

            if (blockNodePods.length === 0) {
              throw new SoloError('Failed to list block node pod');
            }
          },
        },
        {
          title: 'Check block node pod is ready',
          task: async ({config}): Promise<void> => {
            try {
              await this.k8Factory
                .getK8(config.context)
                .pods()
                .waitForReadyStatus(
                  config.namespace,
                  Templates.renderBlockNodeLabels(config.newBlockNodeComponent.metadata.id),
                  constants.BLOCK_NODE_PODS_RUNNING_MAX_ATTEMPTS,
                  constants.BLOCK_NODE_PODS_RUNNING_DELAY,
                );
            } catch (error) {
              throw new SoloError(`Block node ${config.releaseName} is not ready: ${error.message}`, error);
            }
          },
        },
        this.checkBlockNodeReadiness(),
        this.handleConsensusNodeUpdating(),
      ],
      constants.LISTR_DEFAULT_OPTIONS.DEFAULT,
      undefined,
      'block node add',
    );

    if (tasks.isRoot()) {
      try {
        await tasks.run();
      } catch (error) {
        throw new SoloError(`Error deploying block node: ${error.message}`, error);
      } finally {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      }
    } else {
      this.taskList.registerCloseFunction(async (): Promise<void> => {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      });
    }

    return true;
  }

  public async destroy(argv: ArgvStruct): Promise<boolean> {
    let lease: Lock;

    const tasks: SoloListr<BlockNodeDestroyContext> = this.taskList.newTaskList<BlockNodeDestroyContext>(
      [
        {
          title: 'Initialize',
          task: async (context_, task): Promise<Listr<AnyListrContext>> => {
            await this.localConfig.load();
            await this.remoteConfig.loadAndValidate(argv);
            if (!this.oneShotState.isActive()) {
              lease = await this.leaseManager.create();
            }

            this.configManager.update(argv);

            flags.disablePrompts(BlockNodeCommand.DESTROY_FLAGS_LIST.optional);

            const allFlags: CommandFlag[] = [
              ...BlockNodeCommand.DESTROY_FLAGS_LIST.required,
              ...BlockNodeCommand.DESTROY_FLAGS_LIST.optional,
            ];

            await this.configManager.executePrompt(task, allFlags);

            const config: BlockNodeDestroyConfigClass = this.configManager.getConfig(
              BlockNodeCommand.DESTROY_CONFIGS_NAME,
              allFlags,
            ) as BlockNodeDestroyConfigClass;

            context_.config = config;

            config.namespace = await this.getNamespace(task);
            config.clusterRef = this.getClusterReference();
            config.context = this.getClusterContext(config.clusterRef);

            const {id, releaseName, isChartInstalled, isLegacyChartInstalled} = await this.inferDestroyData(
              config.id,
              config.namespace,
              config.context,
            );

            config.id = id;
            config.releaseName = releaseName;
            config.isChartInstalled = isChartInstalled;
            config.isLegacyChartInstalled = isLegacyChartInstalled;

            await this.throwIfNamespaceIsMissing(config.context, config.namespace);

            if (!this.oneShotState.isActive()) {
              return ListrLock.newAcquireLockTask(lease, task);
            }
            return ListrLock.newSkippedLockTask(task);
          },
        },
        {
          title: 'Destroy block node',
          task: async ({config: {namespace, releaseName, context}}): Promise<void> => {
            await this.chartManager.uninstall(namespace, releaseName, context);

            const podReferences: PodReference[] = await this.k8Factory
              .getK8(context)
              .pvcs()
              .list(namespace, [`app.kubernetes.io/instance=${releaseName}`])
              .then((pvcs): PvcName[] => pvcs.map((pvc): PvcName => PvcName.of(pvc)))
              .then((names): PodReference[] => names.map((pvc): PodReference => PvcReference.of(namespace, pvc)));

            for (const podReference of podReferences) {
              await this.k8Factory.getK8(context).pvcs().delete(podReference);
            }
          },
          skip: ({config}): boolean => !config.isChartInstalled,
        },
        this.removeBlockNodeComponentFromRemoteConfig(),
        this.rebuildBlockNodesJsonForConsensusNodes(),
      ],
      constants.LISTR_DEFAULT_OPTIONS.DEFAULT,
      undefined,
      'block node destroy',
    );

    if (tasks.isRoot()) {
      try {
        await tasks.run();
      } catch (error) {
        throw new SoloError(`Error destroying block node: ${error.message}`, error);
      } finally {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      }
    } else {
      this.taskList.registerCloseFunction(async (): Promise<void> => {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      });
    }

    return true;
  }

  public async upgrade(argv: ArgvStruct): Promise<boolean> {
    let lease: Lock;

    const tasks: SoloListr<BlockNodeUpgradeContext> = this.taskList.newTaskList<BlockNodeUpgradeContext>(
      [
        {
          title: 'Initialize',
          task: async (context_, task): Promise<Listr<AnyListrContext>> => {
            await this.localConfig.load();
            await this.remoteConfig.loadAndValidate(argv);
            if (!this.oneShotState.isActive()) {
              lease = await this.leaseManager.create();
            }

            this.configManager.update(argv);

            flags.disablePrompts(BlockNodeCommand.UPGRADE_FLAGS_LIST.optional);

            const allFlags: CommandFlag[] = [
              ...BlockNodeCommand.UPGRADE_FLAGS_LIST.required,
              ...BlockNodeCommand.UPGRADE_FLAGS_LIST.optional,
            ];

            await this.configManager.executePrompt(task, allFlags);

            const config: BlockNodeUpgradeConfigClass = this.configManager.getConfig(
              BlockNodeCommand.UPGRADE_CONFIGS_NAME,
              allFlags,
            ) as BlockNodeUpgradeConfigClass;

            context_.config = config;

            config.namespace = await this.getNamespace(task);
            config.clusterRef = this.getClusterReference();
            config.context = this.getClusterContext(config.clusterRef);

            config.id = this.inferBlockNodeId(config.id);

            config.isLegacyChartInstalled = await this.checkIfLegacyChartIsInstalled(
              config.id,
              config.namespace,
              config.context,
            );

            config.releaseName = config.isLegacyChartInstalled
              ? `${constants.BLOCK_NODE_RELEASE_NAME}-0`
              : this.renderReleaseName(config.id);

            config.context = this.remoteConfig.getClusterRefs()[config.clusterRef];
            config.upgradeVersion ||= versions.BLOCK_NODE_VERSION;
            config.currentVersion =
              this.remoteConfig.getComponentVersion(ComponentTypes.BlockNode)?.toString() ?? '0.0.0';

            assertUpgradeVersionNotOlder(
              'Block node',
              config.upgradeVersion,
              this.remoteConfig.getComponentVersion(ComponentTypes.BlockNode),
              optionFromFlag(flags.upgradeVersion),
            );

            if (!this.oneShotState.isActive()) {
              return ListrLock.newAcquireLockTask(lease, task);
            }
            return ListrLock.newSkippedLockTask(task);
          },
        },
        {
          title: 'Look-up block node',
          task: async ({config}): Promise<void> => {
            try {
              this.remoteConfig.configuration.components.getComponent<BlockNodeStateSchema>(
                ComponentTypes.BlockNode,
                config.id,
              );
            } catch (error) {
              throw new SoloError(`Block node ${config.releaseName} was not found`, error);
            }
          },
        },
        {
          title: 'Prepare chart values',
          task: async ({config}): Promise<void> => {
            config.valuesArg = await this.prepareValuesArgForBlockNode(config);
          },
        },
        {
          title: 'Plan block node upgrade migration',
          task: async ({config}, task): Promise<void> => {
            config.migrationPlan = this.buildBlockNodeUpgradeMigrationPlan(
              config.currentVersion,
              config.upgradeVersion,
            );

            const renderedPlan: string = config.migrationPlan
              .map((step): string => `${step.fromVersion} -> ${step.toVersion} [${step.strategy}] (${step.reason})`)
              .join(' | ');
            task.title = `${task.title}: ${renderedPlan}`;
          },
        },
        {
          title: 'Update block node chart',
          task: async ({config}): Promise<void> => {
            const {namespace, releaseName, context} = config;

            for (const step of config.migrationPlan) {
              const stepTargetVersion: string = SemanticVersion.getValidSemanticVersion(
                step.toVersion,
                false,
                'Block node chart version',
              );
              const stepValuesArgument: string = BlockNodeCommand.appendExtraCommandArgs(
                config.valuesArg,
                step.extraCommandArgs,
              );

              if (step.strategy === 'recreate') {
                this.logger.showUser(
                  `Applying block node recreate migration for ${releaseName} (${step.fromVersion} -> ${stepTargetVersion}): ${step.reason}`,
                );
                await this.recreateBlockNodeChart(config, stepTargetVersion, step);
              } else {
                try {
                  await this.chartManager.upgrade(
                    namespace,
                    releaseName,
                    constants.BLOCK_NODE_CHART,
                    config.blockNodeChartDirectory || constants.BLOCK_NODE_CHART_URL,
                    stepTargetVersion,
                    stepValuesArgument,
                    context,
                  );
                } catch (error) {
                  if (this.isImmutableStatefulSetError(error)) {
                    this.logger.showUser(
                      `Detected immutable StatefulSet upgrade for ${releaseName}; retrying with recreate migration`,
                    );
                    await this.recreateBlockNodeChart(config, stepTargetVersion, step);
                  } else {
                    throw error;
                  }
                }
              }

              // Persist the applied step version so remote config reflects the last
              // successfully applied step even if a later step fails.
              this.remoteConfig.updateComponentVersion(
                ComponentTypes.BlockNode,
                new SemanticVersion<string>(stepTargetVersion),
              );
              await this.remoteConfig.persist();
            }

            showVersionBanner(this.logger, constants.BLOCK_NODE_CHART, config.upgradeVersion);

            await this.updateBlockNodeVersionInRemoteConfig(config);
          },
        },
        {
          title: 'Check block node pod is ready',
          task: async ({config}): Promise<void> => {
            try {
              await this.k8Factory
                .getK8(config.context)
                .pods()
                .waitForReadyStatus(
                  config.namespace,
                  Templates.renderBlockNodeLabels(config.id),
                  constants.BLOCK_NODE_PODS_RUNNING_MAX_ATTEMPTS,
                  constants.BLOCK_NODE_PODS_RUNNING_DELAY,
                  config.recreateInstallTime,
                );
            } catch (error) {
              throw new SoloError(
                `Block node ${config.releaseName} is not ready after upgrade: ${error.message}`,
                error,
              );
            }
          },
        },
      ],
      constants.LISTR_DEFAULT_OPTIONS.DEFAULT,
      undefined,
      'block node upgrade',
    );

    if (tasks.isRoot()) {
      try {
        await tasks.run();
      } catch (error) {
        throw new SoloError(`Error upgrading block node: ${error.message}`, error);
      } finally {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      }
    } else {
      this.taskList.registerCloseFunction(async (): Promise<void> => {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      });
    }

    return true;
  }

  public async addExternal(argv: ArgvStruct): Promise<boolean> {
    let lease: Lock;

    const tasks: SoloListr<BlockNodeAddExternalContext> = this.taskList.newTaskList<BlockNodeAddExternalContext>(
      [
        {
          title: 'Initialize',
          task: async (context_, task): Promise<Listr<AnyListrContext>> => {
            await this.localConfig.load();
            await this.remoteConfig.loadAndValidate(argv);
            if (!this.oneShotState.isActive()) {
              lease = await this.leaseManager.create();
            }

            this.configManager.update(argv);

            flags.disablePrompts(BlockNodeCommand.ADD_EXTERNAL_FLAGS_LIST.optional);

            const allFlags: CommandFlag[] = [
              ...BlockNodeCommand.ADD_EXTERNAL_FLAGS_LIST.required,
              ...BlockNodeCommand.ADD_EXTERNAL_FLAGS_LIST.optional,
            ];

            await this.configManager.executePrompt(task, allFlags);

            const config: BlockNodeAddExternalConfigClass = this.configManager.getConfig(
              BlockNodeCommand.ADD_EXTERNAL_CONFIGS_NAME,
              allFlags,
            ) as BlockNodeAddExternalConfigClass;

            context_.config = config;

            config.clusterRef = this.getClusterReference();
            config.context = this.getClusterContext(config.clusterRef);
            config.namespace = await this.getNamespace(task);

            config.priorityMapping = Templates.parseBlockNodePriorityMapping(
              config.priorityMapping as unknown as string,
              this.remoteConfig.getConsensusNodes(),
            );

            const id: ComponentId = this.remoteConfig.configuration.state.externalBlockNodes.length + 1;

            const [address, port] = Templates.parseExternalBlockAddress(config.externalBlockNodeAddress);
            config.newExternalBlockNodeComponent = new ExternalBlockNodeStateSchema(id, address, port);

            this.logger.showUser(
              'Configuring external block node, ' +
                `${chalk.grey('ID')} ${chalk.cyan(`[${id}]`)}, ` +
                `${chalk.grey('address')} ${chalk.cyan(`[${address}:${port}]`)} `,
            );

            if (!this.oneShotState.isActive()) {
              return ListrLock.newAcquireLockTask(lease, task);
            }
            return ListrLock.newSkippedLockTask(task);
          },
        },
        this.addExternalBlockNodeComponent(),
        this.handleConsensusNodeUpdatingForExternalBlockNode(),
      ],
      constants.LISTR_DEFAULT_OPTIONS.DEFAULT,
      undefined,
      'block node add-external',
    );

    if (tasks.isRoot()) {
      try {
        await tasks.run();
      } catch (error) {
        throw new SoloError(`Error adding external block node: ${error.message}`, error);
      } finally {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      }
    } else {
      this.taskList.registerCloseFunction(async (): Promise<void> => {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      });
    }

    return true;
  }

  public async deleteExternal(argv: ArgvStruct): Promise<boolean> {
    let lease: Lock;

    const tasks: SoloListr<BlockNodeDeleteExternalContext> = this.taskList.newTaskList<BlockNodeDeleteExternalContext>(
      [
        {
          title: 'Initialize',
          task: async (context_, task): Promise<Listr<AnyListrContext>> => {
            await this.localConfig.load();
            await this.remoteConfig.loadAndValidate(argv);
            if (!this.oneShotState.isActive()) {
              lease = await this.leaseManager.create();
            }

            this.configManager.update(argv);

            flags.disablePrompts(BlockNodeCommand.DELETE_EXTERNAL_FLAGS_LIST.optional);

            const allFlags: CommandFlag[] = [
              ...BlockNodeCommand.DELETE_EXTERNAL_FLAGS_LIST.required,
              ...BlockNodeCommand.DELETE_EXTERNAL_FLAGS_LIST.optional,
            ];

            await this.configManager.executePrompt(task, allFlags);

            const config: BlockNodeDeleteExternalConfigClass = this.configManager.getConfig(
              BlockNodeCommand.DELETE_CONFIGS_NAME,
              allFlags,
            ) as BlockNodeDeleteExternalConfigClass;

            context_.config = config;

            config.namespace = await this.getNamespace(task);
            config.clusterRef = this.getClusterReference();
            config.context = this.getClusterContext(config.clusterRef);
            config.id = this.inferExternalBlockNodeId(config.id);

            await this.throwIfNamespaceIsMissing(config.context, config.namespace);

            if (!this.oneShotState.isActive()) {
              return ListrLock.newAcquireLockTask(lease, task);
            }
            return ListrLock.newSkippedLockTask(task);
          },
        },
        this.removeExternalBlockNodeComponent(),
        this.rebuildBlockNodesJsonForConsensusNodes(),
      ],
      constants.LISTR_DEFAULT_OPTIONS.DEFAULT,
      undefined,
      'block node delete-external',
    );

    if (tasks.isRoot()) {
      try {
        await tasks.run();
      } catch (error) {
        throw new SoloError(`Error removing external block node: ${error.message}`, error);
      } finally {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      }
    } else {
      this.taskList.registerCloseFunction(async (): Promise<void> => {
        if (!this.oneShotState.isActive()) {
          await lease?.release();
        }
      });
    }

    return true;
  }

  private rebuildBlockNodesJsonForConsensusNodes(): SoloListrTask<AnyListrContext> {
    return {
      title: "Rebuild 'block.nodes.json' for consensus nodes",
      skip: (): boolean => this.remoteConfig.configuration.state.ledgerPhase === LedgerPhase.UNINITIALIZED,
      task: async (): Promise<void> => {
        for (const node of this.remoteConfig.getConsensusNodes()) {
          await createAndCopyBlockNodeJsonFileForConsensusNode(node, this.logger, this.k8Factory);
        }
      },
    };
  }

  /**
   * Gives the port used for liveness check based on the chart version and image tag (if set)
   */
  private getLivenessCheckPortNumber(
    chartVersion: string | SemanticVersion<string>,
    imageTag: Optional<string | SemanticVersion<string>>,
  ): number {
    let useLegacyPort: boolean = false;

    chartVersion = typeof chartVersion === 'string' ? new SemanticVersion<string>(chartVersion) : chartVersion;
    imageTag = typeof imageTag === 'string' && imageTag ? new SemanticVersion<string>(imageTag) : undefined;

    if (chartVersion.lessThan(versions.MINIMUM_HIERO_BLOCK_NODE_VERSION_FOR_NEW_LIVENESS_CHECK_PORT)) {
      useLegacyPort = true;
    } else if (imageTag && imageTag.lessThan(versions.MINIMUM_HIERO_BLOCK_NODE_VERSION_FOR_NEW_LIVENESS_CHECK_PORT)) {
      useLegacyPort = true;
    }

    return useLegacyPort ? constants.BLOCK_NODE_PORT_LEGACY : constants.BLOCK_NODE_PORT;
  }

  private async updateBlockNodeVersionInRemoteConfig(
    config: BlockNodeDeployConfigClass | BlockNodeUpgradeConfigClass,
  ): Promise<void> {
    let blockNodeVersion: SemanticVersion<string>;
    let imageTag: SemanticVersion<string> | undefined;

    if (config.hasOwnProperty('upgradeVersion') && (config as BlockNodeUpgradeConfigClass).upgradeVersion) {
      const version: string = (config as BlockNodeUpgradeConfigClass).upgradeVersion;
      blockNodeVersion = typeof version === 'string' ? new SemanticVersion<string>(version) : version;
    }

    if (config.hasOwnProperty('chartVersion') && (config as BlockNodeDeployConfigClass).chartVersion) {
      const version: string = (config as BlockNodeDeployConfigClass).chartVersion;
      blockNodeVersion = typeof version === 'string' ? new SemanticVersion<string>(version) : version;
    }

    if (config.hasOwnProperty('imageTag') && (config as BlockNodeDeployConfigClass).imageTag) {
      const tag: string = (config as BlockNodeDeployConfigClass).imageTag;
      imageTag = typeof tag === 'string' ? new SemanticVersion<string>(tag) : tag;
    }

    const finalVersion: SemanticVersion<string> =
      imageTag && blockNodeVersion.lessThan(imageTag) ? imageTag : blockNodeVersion;
    this.remoteConfig.updateComponentVersion(ComponentTypes.BlockNode, finalVersion);

    await this.remoteConfig.persist();
  }

  private buildBlockNodeUpgradeMigrationPlan(
    currentVersion: string,
    targetVersion: string,
  ): ComponentUpgradeMigrationStep[] {
    const normalizedCurrentVersion: string = SemanticVersion.getValidSemanticVersion(
      currentVersion || '0.0.0',
      false,
      'Current block node chart version',
    );
    const normalizedTargetVersion: string = SemanticVersion.getValidSemanticVersion(
      targetVersion || versions.BLOCK_NODE_VERSION,
      false,
      'Target block node chart version',
    );

    return ComponentUpgradeMigrationRules.planUpgradeMigrationPath(
      BlockNodeCommand.MIGRATION_COMPONENT_KEY,
      normalizedCurrentVersion,
      normalizedTargetVersion,
    );
  }

  private isImmutableStatefulSetError(error: unknown): boolean {
    const message: string = error instanceof Error ? error.message : String(error);
    return message.includes('StatefulSet.apps') && message.includes('spec: Forbidden');
  }

  private async recreateBlockNodeChart(
    config: BlockNodeUpgradeConfigClass,
    validatedUpgradeVersion: string,
    step: ComponentUpgradeMigrationStep,
  ): Promise<void> {
    const valuesArgument: string = BlockNodeCommand.appendExtraCommandArgs(config.valuesArg, step.extraCommandArgs);
    await this.chartManager.uninstall(config.namespace, config.releaseName, config.context);

    // Wait for the old pod to be fully terminated before creating the new StatefulSet.
    // helm uninstall returns immediately (no --wait), but the pod has a graceful shutdown period.
    // The new StatefulSet will not create a replacement pod until the old pod with the same
    // ordinal name is completely gone (StatefulSet at-most-one semantics), and the PVC cannot
    // be reattached while the old pod still holds a ReadWriteOnce volume mount.
    await this.waitForBlockNodePodsDeleted(config.namespace, config.id, config.context);

    // Record the install time so the readiness check can ignore any stale pod references.
    config.recreateInstallTime = new Date();

    await this.chartManager.install(
      config.namespace,
      config.releaseName,
      constants.BLOCK_NODE_CHART,
      config.blockNodeChartDirectory || constants.BLOCK_NODE_CHART_URL,
      validatedUpgradeVersion,
      valuesArgument,
      config.context,
    );
  }

  /**
   * Polls until no pods with the block-node label exist in the namespace.
   * Used before re-installing the chart so the new StatefulSet pod is not blocked
   * by a terminating predecessor.
   */
  private async waitForBlockNodePodsDeleted(namespace: NamespaceName, id: ComponentId, context: string): Promise<void> {
    const labels: string[] = Templates.renderBlockNodeLabels(id);
    const maxAttempts: number = constants.BLOCK_NODE_PODS_RUNNING_MAX_ATTEMPTS;
    const delay: number = constants.BLOCK_NODE_PODS_RUNNING_DELAY;

    for (let attempt: number = 0; attempt < maxAttempts; attempt++) {
      const pods: Pod[] = await this.k8Factory.getK8(context).pods().list(namespace, labels);
      if (pods.length === 0) {
        return;
      }
      await new Promise<void>((resolve): ReturnType<typeof setTimeout> => setTimeout(resolve, delay));
    }

    this.logger.warn(
      `Block node pods with labels ${labels.join(',')} did not terminate within ${maxAttempts} attempts; proceeding with install`,
    );
  }

  /** Adds the block node component to remote config. */
  private addBlockNodeComponent(): SoloListrTask<BlockNodeDeployContext> {
    return {
      title: 'Add block node component in remote config',
      skip: (): boolean => !this.remoteConfig.isLoaded() || this.oneShotState.isActive(),
      task: async ({config}): Promise<void> => {
        this.remoteConfig.configuration.components.addNewComponent(
          config.newBlockNodeComponent,
          ComponentTypes.BlockNode,
        );

        await this.remoteConfig.persist();
      },
    };
  }

  /** Adds the block node component to remote config. */
  private addExternalBlockNodeComponent(): SoloListrTask<BlockNodeAddExternalContext> {
    return {
      title: 'Add external block node component in remote config',
      skip: (): boolean => !this.remoteConfig.isLoaded(),
      task: async ({config: {newExternalBlockNodeComponent}}): Promise<void> => {
        this.remoteConfig.configuration.state.externalBlockNodes.push(newExternalBlockNodeComponent);

        await this.remoteConfig.persist();
      },
    };
  }

  /** Adds the block node component to remote config. */
  private removeBlockNodeComponentFromRemoteConfig(): SoloListrTask<BlockNodeDestroyContext> {
    return {
      title: 'Disable block node component in remote config',
      skip: (): boolean => !this.remoteConfig.isLoaded(),
      task: async ({config}): Promise<void> => {
        this.remoteConfig.configuration.components.removeComponent(config.id, ComponentTypes.BlockNode);

        for (const node of this.remoteConfig.configuration.state.consensusNodes) {
          node.blockNodeMap = node.blockNodeMap.filter(([id]): boolean => id !== config.id);
        }

        await this.remoteConfig.persist();
      },
    };
  }

  /** Adds the block node component to remote config. */
  private removeExternalBlockNodeComponent(): SoloListrTask<BlockNodeDestroyContext> {
    return {
      title: 'Remove block node component from remote config',
      skip: (): boolean => !this.remoteConfig.isLoaded(),
      task: async ({config}): Promise<void> => {
        this.remoteConfig.configuration.state.externalBlockNodes =
          this.remoteConfig.configuration.state.externalBlockNodes.filter(
            (component): boolean => component.id !== config.id,
          );

        for (const node of this.remoteConfig.configuration.state.consensusNodes) {
          node.externalBlockNodeMap = node.externalBlockNodeMap.filter(([id]): boolean => id !== config.id);
        }

        await this.remoteConfig.persist();
      },
    };
  }

  private displayHealthCheckData(
    task: SoloListrTaskWrapper<BlockNodeDeployContext>,
  ): (attempt: number, maxAttempt: number, color?: 'yellow' | 'green' | 'red', additionalData?: string) =>