@hashgraph/solo/src/commands/mirror-node.ts

An opinionated CLI tool to deploy and manage private Hedera Networks.

// SPDX-License-Identifier: Apache-2.0

import {Listr} from 'listr2';
import {ListrInquirerPromptAdapter} from '@listr2/prompt-adapter-inquirer';
import {confirm as confirmPrompt} from '@inquirer/prompts';
import {IllegalArgumentError} from '../core/errors/illegal-argument-error.js';
import {SoloError} from '../core/errors/solo-error.js';
import {UserBreak} from '../core/errors/user-break.js';
import * as constants from '../core/constants.js';
import {type AccountManager} from '../core/account-manager.js';
import {BaseCommand} from './base.js';
import {Flags as flags} from './flags.js';
import {resolveNamespaceFromDeployment} from '../core/resolvers.js';
import * as helpers from '../core/helpers.js';
import {prepareValuesFiles, showVersionBanner} from '../core/helpers.js';
import {type AnyListrContext, type ArgvStruct} from '../types/aliases.js';
import {type Rbacs} from '../integration/kube/resources/rbac/rbacs.js';
import {ListrLock} from '../core/lock/listr-lock.js';
import * as fs from 'node:fs';
import {
  type ClusterReferenceName,
  type ComponentId,
  type Context,
  type DeploymentName,
  type NamespaceNameAsString,
  type Optional,
  type Realm,
  type Shard,
  type SoloListr,
  type SoloListrTask,
} from '../types/index.js';
import * as versions from '../../version.js';
import {INGRESS_CONTROLLER_VERSION} from '../../version.js';
import {type NamespaceName} from '../types/namespace/namespace-name.js';
import {PodReference} from '../integration/kube/resources/pod/pod-reference.js';
import {Pod} from '../integration/kube/resources/pod/pod.js';
import {type Pods} from '../integration/kube/resources/pod/pods.js';
import chalk from 'chalk';
import {type CommandFlag, type CommandFlags} from '../types/flag-types.js';
import {PvcReference} from '../integration/kube/resources/pvc/pvc-reference.js';
import {PvcName} from '../integration/kube/resources/pvc/pvc-name.js';
import {KeyManager} from '../core/key-manager.js';
import {PathEx} from '../business/utils/path-ex.js';
import {inject, injectable} from 'tsyringe-neo';
import {InjectTokens} from '../core/dependency-injection/inject-tokens.js';
import {patchInject} from '../core/dependency-injection/container-helper.js';
import {ComponentTypes} from '../core/config/remote/enumerations/component-types.js';
import {MirrorNodeStateSchema} from '../data/schema/model/remote/state/mirror-node-state-schema.js';
import {Lock} from '../core/lock/lock.js';
import {Base64} from 'js-base64';
import {SemanticVersion} from '../business/utils/semantic-version.js';
import {assertUpgradeVersionNotOlder} from '../core/upgrade-version-guard.js';
import {IngressClass} from '../integration/kube/resources/ingress-class/ingress-class.js';
import {Secret} from '../integration/kube/resources/secret/secret.js';
import {BlockNodeStateSchema} from '../data/schema/model/remote/state/block-node-state-schema.js';
import {PostgresStateSchema} from '../data/schema/model/remote/state/postgres-state-schema.js';
import {RedisStateSchema} from '../data/schema/model/remote/state/redis-state-schema.js';
import {Templates} from '../core/templates.js';
import {RemoteConfig} from '../business/runtime-state/config/remote/remote-config.js';
import {ClusterSchema} from '../data/schema/model/common/cluster-schema.js';
import yaml from 'yaml';
import {DeploymentPhase} from '../data/schema/model/remote/deployment-phase.js';
import {PostgresSharedResource} from '../core/shared-resources/postgres.js';
import {SharedResourceManager} from '../core/shared-resources/shared-resource-manager.js';
import {MirrorNodeDeployedEvent} from '../core/events/event-types/mirror-node-deployed-event.js';
import {type SoloEventBus} from '../core/events/solo-event-bus.js';
import {optionFromFlag} from './command-helpers.js';
import {ImageReference, type ParsedImageReference} from '../business/utils/image-reference.js';
// Port forwarding is now a method on the components object

interface MirrorNodeDeployConfigClass {
  isChartInstalled: boolean;
  cacheDir: string;
  chartDirectory: string;
  mirrorNodeChartDirectory: string;
  clusterContext: string;
  clusterReference: ClusterReferenceName;
  namespace: NamespaceName;
  enableIngress: boolean;
  ingressControllerValueFile: string;
  mirrorStaticIp: string;
  valuesFile: string;
  valuesArg: string;
  quiet: boolean;
  mirrorNodeVersion: string;
  componentImage: string;
  pinger: boolean;
  operatorId: string;
  operatorKey: string;
  useExternalDatabase: boolean;
  storageType: constants.StorageType;
  storageReadAccessKey: string;
  storageReadSecrets: string;
  storageEndpoint: string;
  storageBucket: string;
  storageBucketPrefix: string;
  storageBucketRegion: string;
  externalDatabaseHost: Optional<string>;
  externalDatabaseOwnerUsername: Optional<string>;
  externalDatabaseOwnerPassword: Optional<string>;
  externalDatabaseReadonlyUsername: Optional<string>;
  externalDatabaseReadonlyPassword: Optional<string>;
  domainName: Optional<string>;
  forcePortForward: Optional<boolean>;
  releaseName: string;
  ingressReleaseName: string;
  newMirrorNodeComponent: MirrorNodeStateSchema;
  isLegacyChartInstalled: boolean;
  id: number;
  soloChartVersion: string;
  deployment: DeploymentName;
  forceBlockNodeIntegration: boolean; // Used to bypass version requirements for block node integration
  installSharedResources: boolean;
  parallelDeploy: boolean;
}

interface MirrorNodeDeployContext {
  config: MirrorNodeDeployConfigClass;
  addressBook: string;
}

interface MirrorNodeUpgradeConfigClass {
  isChartInstalled: boolean;
  cacheDir: string;
  chartDirectory: string;
  mirrorNodeChartDirectory: string;
  clusterContext: string;
  clusterReference: ClusterReferenceName;
  namespace: NamespaceName;
  enableIngress: boolean;
  ingressControllerValueFile: string;
  mirrorStaticIp: string;
  valuesFile: string;
  valuesArg: string;
  quiet: boolean;
  mirrorNodeVersion: string;
  componentImage: string;
  pinger: boolean;
  operatorId: string;
  operatorKey: string;
  useExternalDatabase: boolean;
  storageType: constants.StorageType;
  storageReadAccessKey: string;
  storageReadSecrets: string;
  storageEndpoint: string;
  storageBucket: string;
  storageBucketPrefix: string;
  storageBucketRegion: string;
  externalDatabaseHost: Optional<string>;
  externalDatabaseOwnerUsername: Optional<string>;
  externalDatabaseOwnerPassword: Optional<string>;
  externalDatabaseReadonlyUsername: Optional<string>;
  externalDatabaseReadonlyPassword: Optional<string>;
  domainName: Optional<string>;
  forcePortForward: Optional<boolean>;
  releaseName: string;
  ingressReleaseName: string;
  isLegacyChartInstalled: boolean;
  id: number;
  soloChartVersion: string;
  installSharedResources: boolean;
  forceBlockNodeIntegration: boolean; // Used to bypass version requirements for block node integration
  deployment: DeploymentName;
}

interface MirrorNodeUpgradeContext {
  config: MirrorNodeUpgradeConfigClass;
  addressBook: string;
}

interface MirrorNodeDestroyConfigClass {
  namespace: NamespaceName;
  clusterContext: string;
  isChartInstalled: boolean;
  clusterReference: ClusterReferenceName;
  id: ComponentId;
  releaseName: string;
  ingressReleaseName: string;
  isLegacyChartInstalled: boolean;
  isIngressControllerChartInstalled: boolean;
}

interface MirrorNodeDestroyContext {
  config: MirrorNodeDestroyConfigClass;
}

interface InferredData {
  id: ComponentId;
  releaseName: string;
  isChartInstalled: boolean;
  ingressReleaseName: string;
  isLegacyChartInstalled: boolean;
}

enum MirrorNodeCommandType {
  ADD = 'add',
  UPGRADE = 'upgrade',
  DESTROY = 'destroy',
}

@injectable()
export class MirrorNodeCommand extends BaseCommand {
  private static readonly MIRROR_ENVIRONMENT_VARIABLE_PREFIX: string = 'HIERO';
  private static readonly MIRROR_CHART_NAMESPACE: string = 'hiero';
  public constructor(
    @inject(InjectTokens.PostgresSharedResource) private readonly postgresSharedResource: PostgresSharedResource,
    @inject(InjectTokens.SharedResourceManager) private readonly sharedResourceManager: SharedResourceManager,
    @inject(InjectTokens.AccountManager) private readonly accountManager?: AccountManager,
    @inject(InjectTokens.SoloEventBus) private readonly eventBus?: SoloEventBus,
  ) {
    super();

    this.accountManager = patchInject(accountManager, InjectTokens.AccountManager, this.constructor.name);
    this.postgresSharedResource = patchInject(
      postgresSharedResource,
      InjectTokens.PostgresSharedResource,
      this.constructor.name,
    );
    this.sharedResourceManager = patchInject(
      sharedResourceManager,
      InjectTokens.SharedResourceManager,
      this.constructor.name,
    );
  }

  private static readonly DEPLOY_CONFIGS_NAME: string = 'deployConfigs';

  private static readonly UPGRADE_CONFIGS_NAME: string = 'upgradeConfigs';

  public static readonly DEPLOY_FLAGS_LIST: CommandFlags = {
    required: [flags.deployment],
    optional: [
      flags.cacheDir,
      flags.chartDirectory,
      flags.mirrorNodeChartDirectory,
      flags.clusterRef,
      flags.enableIngress,
      flags.ingressControllerValueFile,
      flags.mirrorStaticIp,
      flags.quiet,
      flags.valuesFile,
      flags.mirrorNodeVersion,
      flags.componentImage,
      flags.pinger,
      flags.useExternalDatabase,
      flags.operatorId,
      flags.operatorKey,
      flags.storageType,
      flags.storageReadAccessKey,
      flags.storageReadSecrets,
      flags.storageEndpoint,
      flags.storageBucket,
      flags.storageBucketPrefix,
      flags.storageBucketRegion,
      flags.externalDatabaseHost,
      flags.externalDatabaseOwnerUsername,
      flags.externalDatabaseOwnerPassword,
      flags.externalDatabaseReadonlyUsername,
      flags.externalDatabaseReadonlyPassword,
      flags.domainName,
      flags.forcePortForward,
      flags.externalAddress,
      flags.soloChartVersion,
      flags.forceBlockNodeIntegration, // Used to bypass version requirements for block node integration
      flags.parallelDeploy,
    ],
  };

  public static readonly UPGRADE_FLAGS_LIST: CommandFlags = {
    required: [flags.deployment],
    optional: [
      flags.clusterRef,
      flags.cacheDir,
      flags.chartDirectory,
      flags.mirrorNodeChartDirectory,
      flags.enableIngress,
      flags.ingressControllerValueFile,
      flags.mirrorStaticIp,
      flags.quiet,
      flags.valuesFile,
      flags.mirrorNodeVersion,
      flags.componentImage,
      flags.pinger,
      flags.useExternalDatabase,
      flags.operatorId,
      flags.operatorKey,
      flags.storageType,
      flags.storageReadAccessKey,
      flags.storageReadSecrets,
      flags.storageEndpoint,
      flags.storageBucket,
      flags.storageBucketPrefix,
      flags.storageBucketRegion,
      flags.externalDatabaseHost,
      flags.externalDatabaseOwnerUsername,
      flags.externalDatabaseOwnerPassword,
      flags.externalDatabaseReadonlyUsername,
      flags.externalDatabaseReadonlyPassword,
      flags.domainName,
      flags.forcePortForward,
      flags.externalAddress,
      flags.id,
      flags.soloChartVersion,
      flags.forceBlockNodeIntegration, // Used to bypass version requirements for block node integration
    ],
  };

  public static readonly DESTROY_FLAGS_LIST: CommandFlags = {
    required: [flags.deployment],
    optional: [flags.chartDirectory, flags.clusterRef, flags.force, flags.quiet, flags.devMode, flags.id],
  };

  private prepareBlockNodeIntegrationValues(
    config: MirrorNodeUpgradeConfigClass | MirrorNodeDeployConfigClass,
  ): string {
    const configuration: RemoteConfig = this.remoteConfig.configuration;
    const blockNodeSchemas: ReadonlyArray<Readonly<BlockNodeStateSchema>> = configuration.components.state.blockNodes;
    const sameClusterBlockNodeSchemas: ReadonlyArray<Readonly<BlockNodeStateSchema>> = blockNodeSchemas.filter(
      (blockNode): boolean => blockNode.metadata.cluster === config.clusterReference,
    );

    if (blockNodeSchemas.length === 0) {
      this.logger.debug('No block nodes found in remote config configuration');
      return '';
    }

    if (sameClusterBlockNodeSchemas.length === 0) {
      this.logger.info(
        `Skipping block node integration for mirror node cluster ${config.clusterReference}; no block node in the same cluster`,
      );
      return '';
    }

    let shouldConfigureMirrorNodeToPullFromBlockNode: boolean;

    if (config.forceBlockNodeIntegration) {
      // Bypass following checks
      this.logger.warn('Force flag enabled, bypassing version checks for block node integration');
      shouldConfigureMirrorNodeToPullFromBlockNode = true;
    } else {
      const isConsensusNodeVersionSupported: boolean =
        this.remoteConfig.configuration.versions.consensusNode.greaterThanOrEqual(
          versions.MINIMUM_HIERO_PLATFORM_VERSION_FOR_TSS,
        );

      const isBlockNodeChartVersionSupported: boolean =
        this.remoteConfig.configuration.versions.blockNodeChart.greaterThanOrEqual(
          versions.MINIMUM_BLOCK_NODE_CHART_VERSION_FOR_MIRROR_NODE_INTEGRATION,
        );

      const isMirrorNodeVersionSupported: boolean = new SemanticVersion<string>(
        config.mirrorNodeVersion,
      ).greaterThanOrEqual(versions.MINIMUM_MIRROR_NODE_CHART_VERSION_FOR_MIRROR_NODE_INTEGRATION);

      shouldConfigureMirrorNodeToPullFromBlockNode =
        isConsensusNodeVersionSupported && isBlockNodeChartVersionSupported && isMirrorNodeVersionSupported;
    }

    if (!shouldConfigureMirrorNodeToPullFromBlockNode) {
      this.logger.info(
        'Mirror node will remain configured to pull from consensus node because version requirements were not met',
      );
      return '';
    }

    const clusterSchemas: ReadonlyArray<Readonly<ClusterSchema>> = configuration.clusters;

    this.logger.debug('Preparing mirror node values args overrides for block nodes integration');

    const blockNodeFqdnList: {host: string; port: number}[] = [];

    for (const blockNode of sameClusterBlockNodeSchemas) {
      const id: ComponentId = blockNode.metadata.id;
      const clusterReference: ClusterReferenceName = blockNode.metadata.cluster;

      const cluster: Readonly<ClusterSchema> = clusterSchemas.find(
        (cluster): boolean => cluster.name === clusterReference,
      );

      if (!cluster) {
        throw new SoloError(`Cluster ${clusterReference} not found in remote config`);
      }

      const serviceName: string = Templates.renderBlockNodeName(id);
      const namespace: NamespaceNameAsString = blockNode.metadata.namespace;
      const dnsBaseDomain: string = cluster.dnsBaseDomain;

      const fqdn: string = Templates.renderSvcFullyQualifiedDomainName(serviceName, namespace, dnsBaseDomain);

      blockNodeFqdnList.push({
        host: fqdn,
        port: constants.BLOCK_NODE_PORT,
      });
    }

    const data: {SPRING_PROFILES_ACTIVE: string} & Record<string, string | number> = {
      SPRING_PROFILES_ACTIVE: 'blocknode',
    };

    for (const [index, node] of blockNodeFqdnList.entries()) {
      data[`HIERO_MIRROR_IMPORTER_BLOCK_NODES_${index}_HOST`] = node.host;
      if (node.port !== constants.BLOCK_NODE_PORT) {
        data[`HIERO_MIRROR_IMPORTER_BLOCK_NODES_${index}_PORT`] = node.port;
      }
    }

    const mirrorNodeBlockNodeValues: {
      importer: {
        env: {SPRING_PROFILES_ACTIVE: string} & Record<string, string | number>;
      };
    } = {
      importer: {
        env: data,
      },
    };

    const mirrorNodeBlockNodeValuesYaml: string = yaml.stringify(mirrorNodeBlockNodeValues);

    const valuesFilePath: string = PathEx.join(config.cacheDir, 'mirror-bn-values.yaml');

    fs.writeFileSync(valuesFilePath, mirrorNodeBlockNodeValuesYaml);

    return ` --values ${valuesFilePath}`;
  }

  private async prepareValuesArg(config: MirrorNodeDeployConfigClass | MirrorNodeUpgradeConfigClass): Promise<string> {
    let valuesArgument: string = '';

    valuesArgument += ' --install';
    if (config.valuesFile) {
      valuesArgument += helpers.prepareValuesFiles(config.valuesFile);
    }

    config.mirrorNodeVersion = SemanticVersion.getValidSemanticVersion(
      config.mirrorNodeVersion,
      true,
      'Mirror node version',
    );

    const chartNamespace: string = MirrorNodeCommand.MIRROR_CHART_NAMESPACE;
    const environmentVariablePrefix: string = MirrorNodeCommand.MIRROR_ENVIRONMENT_VARIABLE_PREFIX;

    if (config.componentImage) {
      const parsedImageReference: ParsedImageReference = ImageReference.parseImageReference(config.componentImage);
      valuesArgument += helpers.populateHelmArguments({
        'importer.image.registry': parsedImageReference.registry,
        'grpc.image.registry': parsedImageReference.registry,
        'rest.image.registry': parsedImageReference.registry,
        'restjava.image.registry': parsedImageReference.registry,
        'web3.image.registry': parsedImageReference.registry,
        'monitor.image.registry': parsedImageReference.registry,
        'importer.image.repository': parsedImageReference.repository,
        'grpc.image.repository': parsedImageReference.repository,
        'rest.image.repository': parsedImageReference.repository,
        'restjava.image.repository': parsedImageReference.repository,
        'web3.image.repository': parsedImageReference.repository,
        'monitor.image.repository': parsedImageReference.repository,
        'importer.image.tag': parsedImageReference.tag,
        'grpc.image.tag': parsedImageReference.tag,
        'rest.image.tag': parsedImageReference.tag,
        'restjava.image.tag': parsedImageReference.tag,
        'web3.image.tag': parsedImageReference.tag,
        'monitor.image.tag': parsedImageReference.tag,
      });
    }

    if (config.storageBucket) {
      valuesArgument += ` --set importer.config.${chartNamespace}.mirror.importer.downloader.bucketName=${config.storageBucket}`;
    }
    if (config.storageBucketPrefix) {
      this.logger.info(`Setting storage bucket prefix to ${config.storageBucketPrefix}`);
      valuesArgument += ` --set importer.config.${chartNamespace}.mirror.importer.downloader.pathPrefix=${config.storageBucketPrefix}`;
    }

    let storageType: string = '';
    if (
      config.storageType !== constants.StorageType.MINIO_ONLY &&
      config.storageReadAccessKey &&
      config.storageReadSecrets &&
      config.storageEndpoint
    ) {
      if (
        config.storageType === constants.StorageType.GCS_ONLY ||
        config.storageType === constants.StorageType.AWS_AND_GCS
      ) {
        storageType = 'gcp';
      } else if (config.storageType === constants.StorageType.AWS_ONLY) {
        storageType = 's3';
      } else {
        throw new IllegalArgumentError(`Invalid cloud storage type: ${config.storageType}`);
      }

      const mapping: Record<string, string | boolean | number> = {
        [`importer.env.${environmentVariablePrefix}_MIRROR_IMPORTER_DOWNLOADER_CLOUDPROVIDER`]: storageType,
        [`importer.env.${environmentVariablePrefix}_MIRROR_IMPORTER_DOWNLOADER_ENDPOINTOVERRIDE`]:
          config.storageEndpoint,
        [`importer.env.${environmentVariablePrefix}_MIRROR_IMPORTER_DOWNLOADER_ACCESSKEY`]: config.storageReadAccessKey,
        [`importer.env.${environmentVariablePrefix}_MIRROR_IMPORTER_DOWNLOADER_SECRETKEY`]: config.storageReadSecrets,
      };
      valuesArgument += helpers.populateHelmArguments(mapping);
    }

    if (config.storageBucketRegion) {
      valuesArgument += ` --set importer.env.${environmentVariablePrefix}_MIRROR_IMPORTER_DOWNLOADER_REGION=${config.storageBucketRegion}`;
    }

    if (config.domainName) {
      valuesArgument += helpers.populateHelmArguments({
        'ingress.enabled': true,
        'ingress.tls.enabled': false,
        'ingress.hosts[0].host': config.domainName,
      });
    }

    // if the useExternalDatabase populate all the required values before installing the chart
    let host: string, ownerPassword: string, ownerUsername: string, readonlyPassword: string, readonlyUsername: string;
    valuesArgument += helpers.populateHelmArguments({
      // Disable default database deployment
      'stackgres.enabled': false,
      'postgresql.enabled': false,
      'db.name': 'mirror_node',
    });

    if (config.useExternalDatabase) {
      host = config.externalDatabaseHost;
      ownerPassword = config.externalDatabaseOwnerPassword;
      ownerUsername = config.externalDatabaseOwnerUsername;
      readonlyUsername = config.externalDatabaseReadonlyUsername;
      readonlyPassword = config.externalDatabaseReadonlyPassword;

      valuesArgument += helpers.populateHelmArguments({
        // Set the host and name
        'db.host': host,

        // set the usernames
        'db.owner.username': ownerUsername,
        'importer.db.username': ownerUsername,

        'grpc.db.username': readonlyUsername,
        'restjava.db.username': readonlyUsername,
        'web3.db.username': readonlyUsername,

        // TODO: Fixes a problem where importer's V1.0__Init.sql migration fails
        // 'rest.db.username': readonlyUsername,

        // set the passwords
        'db.owner.password': ownerPassword,
        'importer.db.password': ownerPassword,

        'grpc.db.password': readonlyPassword,
        'restjava.db.password': readonlyPassword,
        'web3.db.password': readonlyPassword,
        'rest.db.password': readonlyPassword,
      });
    } else {
      valuesArgument += helpers.populateHelmArguments({
        'db.host': `solo-shared-resources-postgres.${config.namespace.name}.svc.cluster.local`,
      });
    }

    valuesArgument += this.prepareBlockNodeIntegrationValues(config);

    return valuesArgument;
  }

  private async deployMirrorNode(
    {config}: MirrorNodeDeployContext | MirrorNodeUpgradeContext,
    commandType: MirrorNodeCommandType,
  ): Promise<void> {
    // Determine if we should reuse values based on the currently deployed version from remote config
    // If upgrading from a version <= MIRROR_NODE_VERSION_BOUNDARY, we need to skip reuseValues
    // to avoid RegularExpression rules from old version causing relay node request failures
    const currentVersion: SemanticVersion<string> | null = this.remoteConfig.getComponentVersion(
      ComponentTypes.MirrorNode,
    );
    let shouldReuseValues: boolean = currentVersion
      ? currentVersion.greaterThan(constants.MIRROR_NODE_VERSION_BOUNDARY)
      : false; // If no current version (first install), don't reuse values

    // Don't reuse values when crossing the shared-resources/memory-improvements boundary
    // (upgrading from < v0.152.0 → >= v0.152.0).  Versions before this boundary used an
    // embedded chart-managed Redis with sentinel nodes pointed at "<release>-redis".
    // Reusing those old values would leak the stale "SPRING_DATA_REDIS_SENTINEL_NODES"
    // configuration into the upgraded pods even though redis.enabled is now set to false,
    // because --reuse-values merges ALL old chart values (including sentinel node addresses)
    // and we only explicitly override redis.enabled / redis.host / redis.port — not every
    // sentinel sub-key.  Forcing a clean value set here prevents pods from failing to
    // resolve the no-longer-existent "<release>-redis" hostname.
    if (
      shouldReuseValues &&
      currentVersion !== null &&
      currentVersion.lessThan(versions.MEMORY_ENHANCEMENTS_MIRROR_NODE_VERSION) &&
      new SemanticVersion<string>(config.mirrorNodeVersion).greaterThanOrEqual(
        versions.MEMORY_ENHANCEMENTS_MIRROR_NODE_VERSION,
      )
    ) {
      shouldReuseValues = false;
    }

    if (commandType === MirrorNodeCommandType.ADD) {
      shouldReuseValues = false;
    }

    await this.chartManager.upgrade(
      config.namespace,
      config.releaseName,
      constants.MIRROR_NODE_CHART,
      config.mirrorNodeChartDirectory || constants.MIRROR_NODE_RELEASE_NAME,
      config.mirrorNodeVersion,
      config.valuesArg,
      config.clusterContext,
      shouldReuseValues,
    );

    this.eventBus.emit(new MirrorNodeDeployedEvent(config.deployment));

    showVersionBanner(this.logger, constants.MIRROR_NODE_RELEASE_NAME, config.mirrorNodeVersion);

    if (commandType === MirrorNodeCommandType.ADD) {
      this.remoteConfig.configuration.components.changeComponentPhase(
        (config as MirrorNodeDeployConfigClass).newMirrorNodeComponent.metadata.id,
        ComponentTypes.MirrorNode,
        DeploymentPhase.DEPLOYED,
      );

      await this.remoteConfig.persist();
    } else if (commandType === MirrorNodeCommandType.UPGRADE) {
      // update mirror node version in remote config after successful upgrade
      this.remoteConfig.updateComponentVersion(
        ComponentTypes.MirrorNode,
        new SemanticVersion<string>(config.mirrorNodeVersion),
      );

      await this.remoteConfig.persist();
    }

    if (config.enableIngress) {
      const existingIngressClasses: IngressClass[] = await this.k8Factory
        .getK8(config.clusterContext)
        .ingressClasses()
        .list();
      for (const ingressClass of existingIngressClasses) {
        this.logger.debug(`Found existing IngressClass [${ingressClass.name}]`);
        if (ingressClass.name === constants.MIRROR_INGRESS_CLASS_NAME) {
          this.logger.showUser(`${constants.MIRROR_INGRESS_CLASS_NAME} already found, skipping`);
          return;
        }
      }

      await KeyManager.createTlsSecret(
        this.k8Factory,
        config.namespace,
        config.domainName,
        config.cacheDir,
        constants.MIRROR_INGRESS_TLS_SECRET_NAME,
      );
      // patch ingressClassName of mirror ingress, so it can be recognized by haproxy ingress controller
      const updated: object = {
        metadata: {
          annotations: {
            'haproxy-ingress.github.io/path-type': 'regex',
          },
        },
        spec: {
          ingressClassName: `${constants.MIRROR_INGRESS_CLASS_NAME}`,
          tls: [
            {
              hosts: [config.domainName || 'localhost'],
              secretName: constants.MIRROR_INGRESS_TLS_SECRET_NAME,
            },
          ],
        },
      };
      await this.k8Factory
        .getK8(config.clusterContext)
        .ingresses()
        .update(config.namespace, constants.MIRROR_NODE_RELEASE_NAME, updated);

      await this.k8Factory
        .getK8(config.clusterContext)
        .ingressClasses()
        .create(
          constants.MIRROR_INGRESS_CLASS_NAME,
          constants.INGRESS_CONTROLLER_PREFIX + constants.MIRROR_INGRESS_CONTROLLER,
        );
    }
  }

  private getReleaseName(): string {
    return this.renderReleaseName(
      this.remoteConfig.configuration.components.getNewComponentId(ComponentTypes.MirrorNode),
    );
  }

  private getIngressReleaseName(): string {
    return this.renderIngressReleaseName(
      this.remoteConfig.configuration.components.getNewComponentId(ComponentTypes.MirrorNode),
    );
  }

  private renderReleaseName(id: ComponentId): string {
    if (typeof id !== 'number') {
      throw new SoloError(`Invalid component id: ${id}, type: ${typeof id}`);
    }
    return `${constants.MIRROR_NODE_RELEASE_NAME}-${id}`;
  }

  private renderIngressReleaseName(id: ComponentId): string {
    if (typeof id !== 'number') {
      throw new SoloError(`Invalid component id: ${id}, type: ${typeof id}`);
    }
    return `${constants.INGRESS_CONTROLLER_RELEASE_NAME}-${id}`;
  }

  private enableSharedResourcesTask(): SoloListrTask<AnyListrContext> {
    return {
      title: 'Enable shared resources',
      task: async (_, task): Promise<SoloListr<AnyListrContext>> => {
        const subTasks: SoloListrTask<AnyListrContext>[] = [
          {
            title: 'Install Shared Resources chart',
            task: async (context_): Promise<void> => {
              if (!context_.config.useExternalDatabase) {
                this.sharedResourceManager.enablePostgres();
              }

              this.sharedResourceManager.enableRedis();
              context_.config.installSharedResources = await this.sharedResourceManager.installChart(
                context_.config.namespace,
                context_.config.chartDirectory,
                context_.config.soloChartVersion,
                context_.config.clusterContext,
                {
                  'redis.image.registry': constants.REDIS_IMAGE_REGISTRY,
                  'redis.image.repository': constants.REDIS_IMAGE_REPOSITORY,
                  'redis.image.tag': versions.REDIS_IMAGE_VERSION,
                  'redis.sentinel.image.registry': constants.REDIS_SENTINEL_IMAGE_REGISTRY,
                  'redis.sentinel.image.repository': constants.REDIS_SENTINEL_IMAGE_REPOSITORY,
                  'redis.sentinel.image.tag': versions.REDIS_SENTINEL_IMAGE_VERSION,
                  'redis.sentinel.masterSet': constants.REDIS_SENTINEL_MASTER_SET,
                },
              );
            },
          },
          {
            title: 'Load redis credentials',
            task: async (context_): Promise<void> => {
              const secrets: Secret[] = await this.k8Factory
                .getK8(context_.config.clusterContext)
                .secrets()
                .list(context_.config.namespace, ['app.kubernetes.io/instance=solo-shared-resources']);
              const secret: Secret = secrets.find(
                (secret: Secret): boolean => secret.name === 'solo-shared-resources-redis',
              );

              // Update values
              context_.config.valuesArg += helpers.populateHelmArguments({
                'redis.enabled': false,
                'redis.auth.password': Base64.decode(secret.data['SPRING_DATA_REDIS_PASSWORD']),
                'redis.host': Base64.decode(secret.data['SPRING_DATA_REDIS_HOST']),
                'redis.port': Base64.decode(secret.data['SPRING_DATA_REDIS_PORT']),
              });
            },
          },
          {
            title: 'Initialize Postgres pod',
            task: (_context_, task): SoloListr<MirrorNodeDeployContext> => {
              const subTasks: SoloListrTask<MirrorNodeDeployContext>[] = [
                {
                  title: 'Wait for Postgres pod to be ready',
                  task: async (context_): Promise<void> => {
                    await this.postgresSharedResource.waitForPodReady(
                      context_.config.namespace,
                      context_.config.clusterContext,
                    );
                  },
                },
              ];

              // set up the sub-tasks
              return task.newListr(subTasks, {
                concurrent: false, // no need to run concurrently since if one node is up, the rest should be up by then
                rendererOptions: {
                  collapseSubtasks: false,
                },
              });
            },
            skip: (context_): boolean => context_.config.useExternalDatabase,
          },
          {
            title: 'Add shared resource components to remote config',
            skip: (context_): boolean => !context_.config.installSharedResources || !this.remoteConfig.isLoaded(),
            task: async (context_): Promise<void> => {
              if (!context_.config.useExternalDatabase) {
                const postgresComponent: PostgresStateSchema = this.componentFactory.createNewPostgresComponent(
                  context_.config.clusterReference,
                  context_.config.namespace,
                );
                this.remoteConfig.configuration.components.addNewComponent(postgresComponent, ComponentTypes.Postgres);
              }
              const redisComponent: RedisStateSchema = this.componentFactory.createNewRedisComponent(
                context_.config.clusterReference,
                context_.config.namespace,
              );
              this.remoteConfig.configuration.components.addNewComponent(redisComponent, ComponentTypes.Redis);
              await this.remoteConfig.persist();
            },
          },
        ];

        // set up the sub-tasks
        return task.newListr(subTasks, {
          concurrent: false, // no need to run concurrently since if one node is up, the rest should be up by then
          rendererOptions: {
            collapseSubtasks: false,
          },
        });
      },
    };
  }

  private initializeSharedPostgresDatabaseTask(): SoloListrTask<AnyListrContext> {
    return {
      title: 'Run database initialization script',
      task: async (context_): Promise<void> => {
        await this.postgresSharedResource.initializeMirrorNode(
          context_.config.namespace,
          context_.config.clusterContext,
          MirrorNodeCommand.MIRROR_ENVIRONMENT_VARIABLE_PREFIX,
        );
      },
      skip: ({config}: MirrorNodeDeployContext): boolean =>
        config.useExternalDatabase || !config.installSharedResources,
    };
  }

  /**
   * Installs the mirror chart with all application components disabled in order to create the
   * `mirror-passwords` secret.  The init script (run by {@link initializeSharedPostgresDatabaseTask})
   * reads that secret to obtain the DB user passwords, so the secret must exist before init runs.
   * The importer must not be running during init (it would hold a session that blocks DROP DATABASE),
   * so we use this lightweight prime install instead of a full chart install.
   *
   * Skipped when the secret already exists (upgrade path) or when using an external database.
   */
  /**
   * Deletes the `<release>-redis` secret so that the subsequent mirror chart install/upgrade
   * re-creates it cleanly.  This is necessary because Kubernetes strategic-merge-patch does not
   * remove keys — stale `SPRING_DATA_REDIS_SENTINEL_NODES` values written by a previous install
   * (using the internal chart-managed Redis) would otherwise persist and cause pods to try to
   * resolve a non-existent hostname.
   */
  private deleteStaleRedisSecretTask(): SoloListrTask<AnyListrContext> {
    return {
      title: 'Delete stale mirror redis secret',
      task: async (context_): Promise<void> => {
        // secrets().delete() returns true for NotFound, so no try/catch needed.
        await this.k8Factory
          .getK8(context_.config.clusterContext)
          .secrets()
          .delete(context_.config.namespace, `${context_.config.releaseName}-redis`);
      },
    };
  }

  private primePostgresSecretTask(): SoloListrTask<AnyListrContext> {
    return {
      title: 'Prime mirror-node postgres secret',
      task: async (context_): Promise<void> => {
        // Skip if the secret was already created by a previous install.
        const secretExists: boolean = await this.k8Factory
          .getK8(context_.config.clusterContext)
          .secrets()
          .exists(context_.config.namespace, 'mirror-passwords');
        if (secretExists) {
          return;
        }

        // Install the mirror chart with every application component disabled.  This is enough for
        // Helm to render and apply the `mirror-passwords` Secret template without starting any pods
        // that could connect to Postgres before the init script runs.
        //
        // redis.enabled must be false here: when true the chart writes SPRING_DATA_REDIS_SENTINEL_NODES
        // into the <release>-redis secret using the chart default host ({{ .Release.Name }}-redis).
        // Kubernetes strategic-merge-patch does not remove keys, so those stale sentinel values would
        // persist through the full upgrade (which sets redis.enabled=false and skips the sentinel block).
        // Setting redis.enabled=false in the prime install prevents the stale keys from ever being written.
        const primeValuesArgument: string =
          ' --install' +
          helpers.populateHelmArguments({
            'stackgres.enabled': false,
            'postgresql.enabled': false,
            'redis.enabled': false,
            'db.host': `solo-shared-resources-postgres.${context_.config.namespace.name}.svc.cluster.local`,
            'db.name': 'mirror_node',
            'importer.enabled': false,
            'grpc.enabled': false,
            'rest.enabled': false,
            'restjava.enabled': false,
            'web3.enabled': false,
            'rosetta.enabled': false,
            'graphql.enabled': false,
            'monitor.enabled': false,
          });

        await this.chartManager.upgrade(
          context_.config.namespace,
          context_.config.releaseName,
          constants.MIRROR_NODE_CHART,
          context_.config.mirrorNodeChartDirectory || constants.MIRROR_NODE_RELEASE_NAME,
          context_.config.mirrorNodeVersion,
          primeValuesArgument,
          context_.config.clusterContext,
          false,
        );
      },
      skip: ({config}: MirrorNodeDeployContext): boolean =>
        config.useExternalDatabase || !config.installSharedResources,
    };
  }

  private enableMirrorNodeTask(commandType: MirrorNodeCommandType): SoloListrTask<AnyListrContext> {
    return {
      title: 'Enable mirror-node',
      task: (_, parentTask): SoloListr<AnyListrContext> =>
        parentTask.newListr<MirrorNodeDeployContext>(
          [
            {
              title: 'Prepare address book',
              task: async (context_): Promise<void> => {
                if (this.oneShotState.isActive()) {
                  context_.addressBook = await this.accountManager.buildAddressBookBase64(
                    PathEx.join(context_.config.cacheDir, 'keys'),
                    context_.config.deployment,
                  );

                  context_.config.valuesArg += ` --set "importer.addressBook=${context_.addressBook}"`;
                } else {
                  const deployment: DeploymentName = this.configManager.getFlag(flags.deployment);
                  const portForward: boolean = this.configManager.getFlag(flags.forcePortForward);
                  context_.addressBook = await this.accountManager.prepareAddressBookBase64(
                    context_.config.namespace,
                    this.remoteConfig.getClusterRefs(),
                    deployment,
                    this.configManager.getFlag(flags.operatorId),
                    this.configManager.getFlag(flags.operatorKey),
                    portForward,
                  );
                  context_.config.valuesArg += ` --set "importer.addressBook=${context_.addressBook}"`;
                }
              },
            },
            {
              title: 'Install mirror ingress controller',
              task: async (context_): Promise<void> => {
                const config: MirrorNodeDeployConfigClass = context_.config;

                let mirrorIngressControllerValuesArgument: string = ' --install ';
                mirrorIngressControllerValuesArgument += helpers.prepareValuesFiles(
                  constants.INGRESS_CONTROLLER_VALUES_FILE,
                );
                if (config.mirrorStaticIp !== '') {
                  mirrorIngressControllerValuesArgument += ` --set controller.service.loadBalancerIP=${context_.config.mirrorStaticIp}`;
                }
                mirrorIngressControllerValuesArgument += ` --set fullnameOverride=${constants.MIRROR_INGRESS_CONTROLLER}-${config.namespace.name}`;
                mirrorIngressControllerValuesArgument += ` --set controller.ingressClass=${constants.MIRROR_INGRESS_CLASS_NAME}`;
                mirrorIngressControllerValuesArgument += ` --set controller.extraArgs.controller-class=${constants.MIRROR_INGRESS_CONTROLLER}`;

                mirrorIngressControllerValuesArgument += prepareValuesFiles(config.ingressControllerValueFile);

                await this.chartManager.upgrade(
                  config.namespace,
                  config.ingressReleaseName,
                  constants.INGRESS_CONTROLLER_RELEASE_NAME,
                  constants.INGRESS_CONTROLLER_RELEASE_NAME,
                  INGRESS_CONTROLLER_VERSION,
                  mirrorIngressControllerValuesArgument,
                  context_.config.clusterContext,
                );
                await this.adoptMirrorIngressControllerRbacOwnership(config);
                showVersionBanner(this.logger, config.ingressReleaseName, INGRESS_CONTROLLER_VERSION);
              },
              skip: (context_): boolean => !context_.config.enableIngress,
            },
            {
              title: 'Deploy mirror-node',
              task: async (context_): Promise<void> => {
                await this.deployMirrorNode(context_, commandType);
              },
            },
          ],
          constants.LISTR_DEFAULT_OPTIONS.DEFAULT,
        ),
    };
  }

  private checkPodsAreReadyNodeTask(): SoloListrTask<AnyListrContext> {
    return {
      title: 'Check pods are ready',
      task: async (context_, task): Promise<SoloListr<MirrorNodeDeployContext | MirrorNodeUpgradeContext>> => {
        const instanceCandidates: string[] = [
          this.renderReleaseName(context_.config.id), // e.g. mirror-1
          context_.config.releaseName,
        ];
        if (context_.config.id === 1) {
          instanceCandidates.push(constants.MIRROR_NODE_RELEASE_NAME); // legacy release name
        }

        const podsInAllNamespaces: Pod[] = [];
        for (const instanceName of new Set(instanceCandidates)) {
          const candidatePods: Pod[] = await this.k8Factory
            .getK8(context_.config.clusterContext)
            .pods()
            .listForAllNamespaces([`app.kubernetes.io/instance=${instanceName}`]);
          podsInAllNamespaces.push(...candidatePods);
        }

        const podsClient: Pods = this.k8Factory.getK8(context_.config.clusterContext).pods();
        const namespacePodReferences: PodReference[] = [
          ...new Map(
            podsInAllNamespaces
              .filter((pod): boolean => pod.podReference?.namespace?.name === context_.config.namespace.name)
              .map((pod): [string, PodReference] => [
                `${pod.podReference.namespace.name}/${pod.podReference.name.name}`,
                pod.podReference,
              ]),
          ).values(),
        ];
        const namespacePods: Pod[] = await Promise.all(
          namespacePodReferences.map(
            async (podReference: PodReference): Promise<Pod> => await podsClient.read(podReference),
          ),
        );

        const deployedPods: Pod[] = namespacePods.filter(
          (pod): boolean => !!pod.labels?.['app.kubernetes.io/component'] && !!pod.labels?.['app.kubernetes.io/name'],
        );

        if (deployedPods.length === 0) {
          throw new SoloError(
            `No deployed mirror-node pods found for release ${context_.config.releaseName} in namespace ${context_.config.namespace.name}`,
          );
        }

        const checksBySelector: Map<string, {title: string; labels: string[]}> = new Map();
        for (const pod of deployedPods) {
          const component: string = pod.labels?.['app.kubernetes.io/component'];
          const name: string = pod.labels?.['app.kubernetes.io/name'];
          const key: string = `${component}|${name}`;
          if (!checksBySelector.has(key)) {
            const titleName: string = component
              .split('-')
              .map((word: string): string => word.charAt(0).toUpperCase() + word.slice(1))
              .join(' ');
            checksBySelector.set(key, {
              title: `Check ${titleName}`,
              labels: [
                `app.kubernetes.io/component=${component}`,
                `app.kubernetes.io/name=${name}`,
                `app.kubernetes.io/instance=${pod.labels?.['app.kubernetes.io/instance']}`,
              ],
            });
          }
        }

        const subTasks: SoloListrTask<MirrorNodeDeployContext | MirrorNodeUpgradeContext>[] = [
          ...checksBySelector.values(),
        ].map(
          ({
            title,
            labels,
          }: {
            title: string;
            labels: string[];
          }): SoloListrTask<MirrorNodeDeployContext | MirrorNodeUpgradeContext> => ({
            title,
            task: async (): Promise<Pod[]> =>
              await this.k8Factory
                .getK8(context_.config.clusterContext)
                .pods()
                .waitForReadyStatus(
                  context_.config.namespace,
                  labels,
                  constants.PODS_READY_MAX_ATTEMPTS,
                  constants.PODS_READY_DELAY,
                ),
          }),
        );

        return task.newListr(subTasks, constants.LISTR_DEFAULT_OPTIONS.WITH_CONCURRENCY);
      },
    };
  }

  private enablePortForwardingTask(): SoloListrTask<AnyListrContext> {
    return {
      title: 'Enable port forwarding for mirror ingress controller',
      skip: ({config}: MirrorNodeDeployContext): boolean => !config.forcePortForward || !config.enableIngress,
      task: async ({config}: MirrorNodeDeployContext): Promise<void> => {
        const externalAddress: string = this.configManager.getFlag<string>(flags.externalAddress);
        const pods: Pod[] = await this.k8Factory
          .getK8(config.clusterContext)
          .pods()
          .list(config.namespace, [`app.kubernetes.io/instance=${config.ingressReleaseName}`]);
        if (pods.length === 0) {
          throw new SoloError('No mirror ingress controller pod found');
        }
        let podReference: PodReference;
        for (const pod of pods) {
          if (pod?.podReference?.name?.name?.startsWith('mirror-ingress')) {
            podReference = pod.podReference;
            break;
          }
        }

        await this.remoteConfig.configuration.components.managePortForward(
          config.clusterReference,
          podReference,
          80, // Pod port
          constants.MIRROR_NODE_PORT, // Local port
          this.k8Factory.getK8(config.clusterContext),
          this.logger,
          ComponentTypes.MirrorNode,
          'Mirror ingress controller',
          config.isChartInstalled, // Reuse existing port if chart is already installed
          undefined,
          true, // persist: auto-restart on failure using persist-port-forward.js
          externalAddress,
        );
        await this.remoteConfig.persist();
      },
    };
  }

  public async add(argv: ArgvStruct): Promise<boolean> {
    let lease: Lock;

    const tasks: SoloListr<MirrorNodeDeployContext> = this.taskList.newTaskList<MirrorNodeDeployContext>(
      [
        {
          title: 'Initialize',
          task: async (context_, task): Promise<Listr<AnyListrContext>> => {
            await this.localConfig.load();
            await this.loadRemoteConfigOrWarn(argv);
            if (!this.oneShotState.isActive()) {
              lease = await this.leaseManager.create();
            }
            this.configManager.update(argv);

            flags.disablePrompts(MirrorNodeCommand.DEPLOY_FLAGS_LIST.optional);

            const allFlags: CommandFlag[] = [
              ...MirrorNodeCommand.DEPLOY_FLAGS_LIST.required,
              ...MirrorNodeCommand.DEPLOY_FLAGS_LIST.optional,
            ];

            await this.configManager.executePrompt(task, allFlags);

            const config: MirrorNodeDeployConfigClass = this.configManager.getConfig(
              MirrorNodeCommand.DEPLOY_CONFIGS_NAME,
              allFlags,
              [],
            ) as MirrorNodeDeployConfigClass;

            context_.config = config;

            const hasMirrorNodeMemoryImprovements: boolean = new SemanticVersion<string>(
              config.mirrorNodeVersion,
            ).greaterThanOrEqual(versions.MEMORY_ENHANCEMENTS_MIRROR_NODE_VERSION);

            config.namespace = await this.getNamespace(task);
            config.clusterReference = this.getClusterReference();
            config.clusterContext = this.getClusterContext(config.clusterReference);

            config.newMirrorNodeComponent = this.componentFactory.createNewMirrorNodeComponent(
              config.clusterReference,
              config.namespace,
            );

            config.newMirrorNodeComponent.metadata.phase = DeploymentPhase.REQUESTED;

            config.id = config.newMirrorNodeComponent.metadata.id;
            config.installSharedResources = false;

            const useMirrorNodeLegacyReleaseName: boolean = process.env.USE_MIRROR_NODE_LEGACY_RELEASE_NAME === 'true';
            if (useMirrorNodeLegacyReleaseName) {
              config.releaseName = constants.MIRROR_NODE_RELEASE_NAME;
              config.ingressReleaseName = `${constants.INGRESS_CONTROLLER_RELEASE_NAME}-${config.namespace.name}`;
            } else {
              config.releaseName = this.getReleaseName();
              config.ingressReleaseName = this.getIngressReleaseName();
            }

            config.isChartInstalled = await this.chartManager.isChartInstalled(
              config.namespace,
              config.releaseName,
              config.clusterContext,
            );

            context_.config.soloChartVersion = SemanticVersion.getValidSemanticVersion(
              context_.config.soloChartVersion,
              false,
              'Solo chart version',
            );

            // predefined values first
            config.valuesArg = helpers.prepareValuesFiles(constants.MIRROR_NODE_VALUES_FILE);

            // user defined values later to override predefined values
            config.valuesArg += await this.prepareValuesArg(config);

            config.deployment = this.configManager.getFlag(flags.deployment);

            const realm: Realm = this.localConfig.configuration.realmForDeployment(config.deployment);
            const shard: Shard = this.localConfig.configuration.shardForDeployment(config.deployment);
            const chartNamespace: string = MirrorNodeCommand.MIRROR_CHART_