@hashgraph/solo/src/commands/node/tasks.ts

An opinionated CLI tool to deploy and manage private Hedera Networks.

// SPDX-License-Identifier: Apache-2.0

import {type AccountManager} from '../../core/account-manager.js';
import {type ConfigManager} from '../../core/config-manager.js';
import {type OneShotState} from '../../core/one-shot-state.js';
import {type KeyManager} from '../../core/key-manager.js';
import {type ProfileManager} from '../../core/profile-manager.js';
import {type PlatformInstaller} from '../../core/platform-installer.js';
import {type K8Factory} from '../../integration/kube/k8-factory.js';
import {type ChartManager} from '../../core/chart-manager.js';
import {type CertificateManager} from '../../core/certificate-manager.js';
import {type HelmClient} from '../../integration/helm/helm-client.js';
import {ReleaseItem} from '../../integration/helm/model/release/release-item.js';
import {Zippy} from '../../core/zippy.js';
import * as constants from '../../core/constants.js';
import {
  CHECK_WRAPS_DIRECTORY_BACKOFF_MS,
  CHECK_WRAPS_DIRECTORY_MAX_ATTEMPTS,
  DEFAULT_NETWORK_NODE_NAME,
  HEDERA_HAPI_PATH,
  HEDERA_NODE_DEFAULT_STAKE_AMOUNT,
} from '../../core/constants.js';

const localBuildPathFilter: (path: string | string[]) => boolean = (path: string | string[]): boolean => {
  return !(path.includes('data/keys') || path.includes('data/config'));
};
import {Templates} from '../../core/templates.js';
import {
  AccountBalance,
  AccountBalanceQuery,
  AccountId,
  AccountUpdateTransaction,
  type Client,
  FileAppendTransaction,
  FileId,
  FileUpdateTransaction,
  FreezeTransaction,
  FreezeType,
  Long,
  NodeCreateTransaction,
  NodeDeleteTransaction,
  NodeUpdateTransaction,
  PrivateKey,
  ServiceEndpoint,
  Status,
  Timestamp,
  TransactionReceipt,
  TransactionResponse,
} from '@hiero-ledger/sdk';
import {SoloError} from '../../core/errors/solo-error.js';
import {MissingArgumentError} from '../../core/errors/missing-argument-error.js';
import fs from 'node:fs';
import crypto from 'node:crypto';
import {execSync} from 'node:child_process';
import find from 'find-process';
import type FindConfig from 'find-process';
import type ProcessInfo from 'find-process';
import * as helpers from '../../core/helpers.js';
import {
  addRootImageValues,
  createAndCopyBlockNodeJsonFileForConsensusNode,
  entityId,
  extractContextFromConsensusNodes,
  prepareEndpoints,
  prepareValuesFilesMap,
  prepareValuesFilesMapMultipleCluster,
  renameAndCopyFile,
  showVersionBanner,
  sleep,
  splitFlagInput,
} from '../../core/helpers.js';
import chalk from 'chalk';
import {Flags as flags} from '../flags.js';
import * as versions from '../../../version.js';
import {
  HEDERA_PLATFORM_VERSION,
  MINIMUM_HIERO_PLATFORM_VERSION_FOR_GRPC_WEB_ENDPOINTS,
  needsConfigTxtForConsensusVersion,
} from '../../../version.js';
import {ListrInquirerPromptAdapter} from '@listr2/prompt-adapter-inquirer';
import {confirm as confirmPrompt} from '@inquirer/prompts';
import {type SoloLogger} from '../../core/logging/solo-logger.js';
import {
  type AnyListrContext,
  type AnyObject,
  type ArgvStruct,
  type ConfigBuilder,
  type IP,
  type NodeAlias,
  type NodeAliases,
  type NodeId,
  type SkipCheck,
} from '../../types/aliases.js';
import {PodName} from '../../integration/kube/resources/pod/pod-name.js';
import {NodeStatusCodes, NodeStatusEnums, NodeSubcommandType} from '../../core/enumerations.js';
import {type Lock} from '../../core/lock/lock.js';
import {ListrLock} from '../../core/lock/listr-lock.js';
import {Duration} from '../../core/time/duration.js';
import {type NodeAddConfigClass} from './config-interfaces/node-add-config-class.js';
import {GenesisNetworkDataConstructor} from '../../core/genesis-network-models/genesis-network-data-constructor.js';
import {NodeOverridesModel} from '../../core/node-overrides-model.js';
import {NamespaceName} from '../../types/namespace/namespace-name.js';
import {PodReference} from '../../integration/kube/resources/pod/pod-reference.js';
import {ContainerReference} from '../../integration/kube/resources/container/container-reference.js';
import {NetworkNodes} from '../../core/network-nodes.js';
import {container, inject, injectable} from 'tsyringe-neo';
import {
  type AccountIdWithKeyPairObject,
  type ClusterReferenceName,
  type ClusterReferences,
  type ComponentData,
  type ComponentDisplayName,
  type ComponentId,
  type Context,
  type DeploymentName,
  type NodeAliasToAddressMapping,
  type Optional,
  type PriorityMapping,
  type PrivateKeyAndCertificateObject,
  type Realm,
  type Shard,
  type SoloListr,
  type SoloListrTask,
  type SoloListrTaskWrapper,
} from '../../types/index.js';
import {patchInject} from '../../core/dependency-injection/container-helper.js';
import {ConsensusNode} from '../../core/model/consensus-node.js';
import {type K8} from '../../integration/kube/k8.js';
import {Base64} from 'js-base64';
import {SecretType} from '../../integration/kube/resources/secret/secret-type.js';
import {InjectTokens} from '../../core/dependency-injection/inject-tokens.js';
import {PathEx} from '../../business/utils/path-ex.js';
import {helmValuesHelper} from '../../core/helm-values-helper.js';
import {type GitClient} from '../../integration/git/git-client.js';
import {type NodeDestroyConfigClass} from './config-interfaces/node-destroy-config-class.js';
import {type NodeRefreshConfigClass} from './config-interfaces/node-refresh-config-class.js';
import {type NodeUpdateConfigClass} from './config-interfaces/node-update-config-class.js';
import {type NodeAddContext} from './config-interfaces/node-add-context.js';
import {type NodeDestroyContext} from './config-interfaces/node-destroy-context.js';
import {type NodeUpdateContext} from './config-interfaces/node-update-context.js';
import {type NodeStatesContext} from './config-interfaces/node-states-context.js';
import {type NodeUpgradeContext} from './config-interfaces/node-upgrade-context.js';
import {type NodeRefreshContext} from './config-interfaces/node-refresh-context.js';
import {type NodeStopContext} from './config-interfaces/node-stop-context.js';
import {type NodeFreezeContext} from './config-interfaces/node-freeze-context.js';
import {type NodeStartContext} from './config-interfaces/node-start-context.js';
import {type NodeRestartContext} from './config-interfaces/node-restart-context.js';
import {type NodeSetupContext} from './config-interfaces/node-setup-context.js';
import {type NodeKeysContext} from './config-interfaces/node-keys-context.js';
import {type NodeKeysConfigClass} from './config-interfaces/node-keys-config-class.js';
import {type NodeStartConfigClass} from './config-interfaces/node-start-config-class.js';
import {type CheckedNodesConfigClass, type CheckedNodesContext} from './config-interfaces/node-common-config-class.js';
import {type NetworkNodeServices} from '../../core/network-node-services.js';
import {ComponentTypes} from '../../core/config/remote/enumerations/component-types.js';
import {DeploymentPhase} from '../../data/schema/model/remote/deployment-phase.js';
import {type RemoteConfigRuntimeStateApi} from '../../business/runtime-state/api/remote-config-runtime-state-api.js';
import {type ComponentFactoryApi} from '../../core/config/remote/api/component-factory-api.js';
import {type LocalConfigRuntimeState} from '../../business/runtime-state/config/local/local-config-runtime-state.js';
import {ClusterSchema} from '../../data/schema/model/common/cluster-schema.js';
import {LockManager} from '../../core/lock/lock-manager.js';
import {type NodeServiceMapping} from '../../types/mappings/node-service-mapping.js';
import {Pod} from '../../integration/kube/resources/pod/pod.js';
import {type Container} from '../../integration/kube/resources/container/container.js';
import {SemanticVersion} from '../../business/utils/semantic-version.js';
import {DeploymentStateSchema} from '../../data/schema/model/remote/deployment-state-schema.js';
import {type BaseStateSchema} from '../../data/schema/model/remote/state/base-state-schema.js';
import {ComponentStateMetadataSchema} from '../../data/schema/model/remote/state/component-state-metadata-schema.js';
import net from 'node:net';
import {type NodeConnectionsContext} from './config-interfaces/node-connections-context.js';
import {TDirectoryData} from '../../integration/kube/t-directory-data.js';
import {Service} from '../../integration/kube/resources/service/service.js';
import {Address} from '../../business/address/address.js';
import {Contexts} from '../../integration/kube/resources/context/contexts.js';
import {K8Helper} from '../../business/utils/k8-helper.js';
import {Secret} from '../../integration/kube/resources/secret/secret.js';
import {NodeUpgradeConfigClass} from './config-interfaces/node-upgrade-config-class.js';
import {NodeCollectJfrLogsContext} from './config-interfaces/node-collect-jfr-logs-context.js';
import {NodeCollectJfrLogsConfigClass} from './config-interfaces/node-collect-jfr-logs-config-class.js';
import {PackageDownloader} from '../../core/package-downloader.js';
import {DefaultHelmClient} from '../../integration/helm/impl/default-helm-client.js';
import {CommandFlag} from '../../types/flag-types.js';
import {ConsensusNodePathTemplates} from '../../core/consensus-node-path-templates.js';
import {type ConfigProvider} from '../../data/configuration/api/config-provider.js';
import {SoloConfig} from '../../business/runtime-state/config/solo/solo-config.js';
import {type Wraps} from '../../business/runtime-state/config/solo/wraps.js';

import {DiagnosticsAnalyzer} from '../util/diagnostics-analyzer.js';
import {NodesStartedEvent} from '../../core/events/event-types/nodes-started-event.js';
import {type SoloEventBus} from '../../core/events/solo-event-bus.js';
import {Listr} from 'listr2';
import {ConfigMap} from '../../integration/kube/resources/config-map/config-map.js';

const {gray, cyan, red, green, yellow} = chalk;

export type LeaseWrapper = {lease: Lock};

@injectable()
export class NodeCommandTasks {
  private readonly soloConfig: SoloConfig;

  public constructor(
    @inject(InjectTokens.SoloLogger) private readonly logger: SoloLogger,
    @inject(InjectTokens.AccountManager) private readonly accountManager: AccountManager,
    @inject(InjectTokens.ConfigManager) private readonly configManager: ConfigManager,
    @inject(InjectTokens.K8Factory) private readonly k8Factory: K8Factory,
    @inject(InjectTokens.PlatformInstaller) private readonly platformInstaller: PlatformInstaller,
    @inject(InjectTokens.KeyManager) private readonly keyManager: KeyManager,
    @inject(InjectTokens.ProfileManager) private readonly profileManager: ProfileManager,
    @inject(InjectTokens.ChartManager) private readonly chartManager: ChartManager,
    @inject(InjectTokens.CertificateManager) private readonly certificateManager: CertificateManager,
    @inject(InjectTokens.RemoteConfigRuntimeState) private readonly remoteConfig: RemoteConfigRuntimeStateApi,
    @inject(InjectTokens.LocalConfigRuntimeState) private readonly localConfig: LocalConfigRuntimeState,
    @inject(InjectTokens.ComponentFactory) private readonly componentFactory: ComponentFactoryApi,
    @inject(InjectTokens.OneShotState) private readonly oneShotState: OneShotState,
    @inject(InjectTokens.Zippy) private readonly zippy: Zippy,
    @inject(InjectTokens.PackageDownloader) private readonly downloader: PackageDownloader,
    @inject(InjectTokens.GitClient) private readonly gitClient: GitClient,
    @inject(InjectTokens.ConfigProvider) configProvider: ConfigProvider,
    @inject(InjectTokens.SoloEventBus) private readonly eventBus: SoloEventBus,
  ) {
    this.logger = patchInject(logger, InjectTokens.SoloLogger, this.constructor.name);
    this.accountManager = patchInject(accountManager, InjectTokens.AccountManager, this.constructor.name);
    this.configManager = patchInject(configManager, InjectTokens.ConfigManager, this.constructor.name);
    this.k8Factory = patchInject(k8Factory, InjectTokens.K8Factory, this.constructor.name);
    this.platformInstaller = patchInject(platformInstaller, InjectTokens.PlatformInstaller, this.constructor.name);
    this.keyManager = patchInject(keyManager, InjectTokens.KeyManager, this.constructor.name);
    this.profileManager = patchInject(profileManager, InjectTokens.ProfileManager, this.constructor.name);
    this.chartManager = patchInject(chartManager, InjectTokens.ChartManager, this.constructor.name);
    this.certificateManager = patchInject(certificateManager, InjectTokens.CertificateManager, this.constructor.name);
    this.localConfig = patchInject(localConfig, InjectTokens.LocalConfigRuntimeState, this.constructor.name);
    this.remoteConfig = patchInject(remoteConfig, InjectTokens.RemoteConfigRuntimeState, this.constructor.name);
    this.oneShotState = patchInject(oneShotState, InjectTokens.OneShotState, this.constructor.name);
    this.zippy = patchInject(zippy, InjectTokens.Zippy, this.constructor.name);
    this.downloader = patchInject(downloader, InjectTokens.PackageDownloader, this.constructor.name);
    this.gitClient = patchInject(gitClient, InjectTokens.GitClient, this.constructor.name);
    this.eventBus = patchInject(eventBus, InjectTokens.SoloEventBus, this.constructor.name);
    configProvider = patchInject(configProvider, InjectTokens.ConfigProvider, this.constructor.name);
    this.soloConfig = SoloConfig.getConfig(configProvider);
  }

  private getFileUpgradeId(deploymentName: DeploymentName): FileId {
    const realm: Realm = this.localConfig.configuration.realmForDeployment(deploymentName);
    const shard: Shard = this.localConfig.configuration.shardForDeployment(deploymentName);
    return FileId.fromString(entityId(shard, realm, constants.UPGRADE_FILE_ID_NUM));
  }

  private async _prepareUpgradeZip(stagingDirectory: string, upgradeVersion?: string): Promise<string> {
    // we build a mock upgrade.zip file as we really don't need to upgrade the network
    // also the platform zip file is ~80Mb in size requiring a lot of transactions since the max
    // transaction size is 6Kb and in practice we need to send the file as 4Kb chunks.
    // Note however that in DAB phase-2, we won't need to trigger this fake upgrade process
    const zipper: Zippy = new Zippy(this.logger);
    const upgradeConfigDirectory: string = PathEx.join(stagingDirectory, 'mock-upgrade', 'data', 'config');
    if (!fs.existsSync(upgradeConfigDirectory)) {
      fs.mkdirSync(upgradeConfigDirectory, {recursive: true});
    }

    // bump field hedera.config.version or use the version passed in
    const fileBytes: Buffer = fs.readFileSync(
      PathEx.joinWithRealPath(stagingDirectory, 'templates', constants.APPLICATION_PROPERTIES),
    );
    const lines: string[] = fileBytes.toString().split('\n');
    const newLines: string[] = [];
    for (let line of lines) {
      line = line.trim();
      const parts: string[] = line.split('=');
      if (parts.length === 2) {
        if (parts[0] === 'hedera.config.version') {
          const version: string = upgradeVersion ?? String(Number.parseInt(parts[1]) + 1);
          line = `hedera.config.version=${version}`;
        }
        newLines.push(line);
      }
    }
    fs.writeFileSync(PathEx.join(upgradeConfigDirectory, constants.APPLICATION_PROPERTIES), newLines.join('\n'));

    return await zipper.zip(
      PathEx.join(stagingDirectory, 'mock-upgrade'),
      PathEx.join(stagingDirectory, 'mock-upgrade.zip'),
    );
  }

  private async _uploadUpgradeZip(
    upgradeZipFile: string,
    nodeClient: Client,
    deploymentName: DeploymentName,
  ): Promise<string> {
    // get byte value of the zip file
    const zipBytes: Buffer = fs.readFileSync(upgradeZipFile);
    const zipHash: string = crypto.createHash('sha384').update(zipBytes).digest('hex');
    this.logger.debug(
      `loaded upgrade zip file [ zipHash = ${zipHash} zipBytes.length = ${zipBytes.length}, zipPath = ${upgradeZipFile}]`,
    );

    // create a file upload transaction to upload file to the network
    try {
      let start: number = 0;

      while (start < zipBytes.length) {
        const zipBytesChunk: Uint8Array<ArrayBuffer> = new Uint8Array(
          zipBytes.subarray(start, start + constants.UPGRADE_FILE_CHUNK_SIZE),
        );
        let fileTransaction: FileUpdateTransaction | FileAppendTransaction | undefined = undefined;

        fileTransaction =
          start === 0
            ? new FileUpdateTransaction().setFileId(this.getFileUpgradeId(deploymentName)).setContents(zipBytesChunk)
            : new FileAppendTransaction().setFileId(this.getFileUpgradeId(deploymentName)).setContents(zipBytesChunk);
        const resp: TransactionResponse = await fileTransaction.execute(nodeClient);
        const receipt: TransactionReceipt = await resp.getReceipt(nodeClient);
        this.logger.debug(
          `updated file ${this.getFileUpgradeId(deploymentName)} [chunkSize= ${zipBytesChunk.length}, txReceipt = ${receipt.toString()}]`,
        );

        start += constants.UPGRADE_FILE_CHUNK_SIZE;
        this.logger.debug(`uploaded ${start} bytes of ${zipBytes.length} bytes`);
      }

      return zipHash;
    } catch (error) {
      throw new SoloError(`failed to upload build.zip file: ${error.message}`, error);
    }
  }

  private async copyLocalBuildPathToNode(
    k8: K8,
    podReference: PodReference,
    configManager: ConfigManager,
    localDataLibraryBuildPath: string,
  ): Promise<void> {
    const container: Container = k8
      .containers()
      .readByRef(ContainerReference.of(podReference, constants.ROOT_CONTAINER));

    // Remove existing jars before copying to prevent mixed-version classpath (issue #3848)
    await container.execContainer([
      'bash',
      '-c',
      `rm -rf ${constants.HEDERA_HAPI_PATH}/${constants.HEDERA_DATA_LIB_DIR}/*.jar ${constants.HEDERA_HAPI_PATH}/${constants.HEDERA_DATA_APPS_DIR}/*.jar`,
    ]);

    await container.copyTo(localDataLibraryBuildPath, `${constants.HEDERA_HAPI_PATH}`, localBuildPathFilter);
    if (configManager.getFlag<string>(flags.appConfig)) {
      const testJsonFiles: string[] = configManager.getFlag<string>(flags.appConfig)!.split(',');
      for (const jsonFile of testJsonFiles) {
        if (fs.existsSync(jsonFile)) {
          await container.copyTo(jsonFile, `${constants.HEDERA_HAPI_PATH}`);
        }
      }
    }
  }

  private async validateNodePvcsForLocalBuildPath(namespace: NamespaceName, contexts: string[]): Promise<void> {
    await Promise.all(
      contexts.map(async (context): Promise<void> => {
        const pvcs: string[] = await this.k8Factory
          .getK8(context)
          .pvcs()
          .list(namespace, ['solo.hedera.com/type=node-pvc']);

        if (pvcs.length === 0) {
          throw new SoloError(
            'Custom JARs provided via --local-build-path require node PVCs to persist across pod restarts. ' +
              'Redeploy the consensus network with --pvcs true and run consensus node setup again.',
          );
        }
      }),
    );
  }

  private _uploadPlatformSoftware(
    nodeAliases: NodeAliases,
    podReferences: Record<NodeAlias, PodReference>,
    task: SoloListrTaskWrapper<AnyListrContext>,
    localBuildPath: string,
    consensusNodes: ConsensusNode[],
    releaseTag: string,
  ): SoloListr<AnyListrContext> {
    const subTasks: SoloListrTask<AnyListrContext>[] = [];

    this.logger.debug('no need to fetch, use local build jar files');

    const buildPathMap: Map<NodeAlias, string> = new Map<NodeAlias, string>();
    let defaultDataLibraryBuildPath: string;
    const parameterPairs: string[] = localBuildPath.split(',');
    for (const parameterPair of parameterPairs) {
      if (parameterPair.includes('=')) {
        const [nodeAlias, localDataLibraryBuildPath]: string[] = parameterPair.split('=');
        buildPathMap.set(nodeAlias as NodeAlias, localDataLibraryBuildPath);
      } else {
        defaultDataLibraryBuildPath = parameterPair;
      }
    }

    let localDataLibraryBuildPath: string;

    for (const nodeAlias of nodeAliases) {
      const podReference: PodReference = podReferences[nodeAlias];
      const context: string = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
      localDataLibraryBuildPath = buildPathMap.has(nodeAlias)
        ? buildPathMap.get(nodeAlias)
        : defaultDataLibraryBuildPath;

      if (!fs.existsSync(localDataLibraryBuildPath)) {
        throw new SoloError(`local build path does not exist: ${localDataLibraryBuildPath}`);
      }

      // The local build path points to the `data` directory itself (containing apps/ and lib/).
      // Validate that it contains jar files in each subdirectory to catch incorrect paths early.
      const applicationsSubDirectory: string = PathEx.join(localDataLibraryBuildPath, 'apps');
      const librarySubDirectory: string = PathEx.join(localDataLibraryBuildPath, 'lib');
      if (!fs.existsSync(applicationsSubDirectory) || !fs.existsSync(librarySubDirectory)) {
        throw new SoloError(
          `local build path '${localDataLibraryBuildPath}' must contain 'apps' and 'lib' subdirectories`,
        );
      }
      const applicationsJarFiles: string[] = fs
        .readdirSync(applicationsSubDirectory)
        .filter((file: string): boolean => file.endsWith('.jar'));
      if (applicationsJarFiles.length === 0) {
        throw new SoloError(`No jar files found in '${applicationsSubDirectory}'; please check your local build path`);
      }
      const libraryJarFiles: string[] = fs
        .readdirSync(librarySubDirectory)
        .filter((file: string): boolean => file.endsWith('.jar'));
      if (libraryJarFiles.length === 0) {
        throw new SoloError(`No jar files found in '${librarySubDirectory}'; please check your local build path`);
      }

      const k8: K8 = this.k8Factory.getK8(context);

      subTasks.push({
        title: `Copy local build to Node: ${chalk.yellow(nodeAlias)} from ${localDataLibraryBuildPath}`,
        task: async (): Promise<void> => {
          try {
            const retrievedReleaseTag: string = await this.gitClient.describeTag(localDataLibraryBuildPath);
            const expectedReleaseTag: string = releaseTag || HEDERA_PLATFORM_VERSION;
            if (retrievedReleaseTag !== expectedReleaseTag) {
              this.logger.showUser(
                chalk.cyan(
                  `Checkout version ${retrievedReleaseTag} does not match the release version ${expectedReleaseTag}`,
                ),
              );
            }
          } catch {
            // if we can't find the release tag in the local build path directory, we will skip the check and continue
            this.logger.warn('Could not find release tag in local build path directory');
            this.logger.showUser(
              chalk.yellowBright(
                'The release tag could not be verified, please ensure that the release tag passed on the command line ' +
                  'matches the release tag of the code in the local build path directory',
              ),
            );
          }

          // retry copying the build to the node to handle edge cases during performance testing
          let storedError: Error | null = null;
          let index: number = 0;
          for (; index < constants.LOCAL_BUILD_COPY_RETRY; index++) {
            storedError = null;
            try {
              // filter the data/config and data/keys to avoid failures due to config and secret mounts
              await this.copyLocalBuildPathToNode(k8, podReference, this.configManager, localDataLibraryBuildPath);
            } catch (error) {
              storedError = error;
            }
          }
          if (storedError) {
            throw new SoloError(`Error in copying local build to node: ${storedError.message}`, storedError);
          }
        },
      });
    }
    // set up the sub-tasks
    return task.newListr(subTasks, {
      concurrent: constants.NODE_COPY_CONCURRENT,
      rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION,
      fallbackRendererOptions: {
        timer: constants.LISTR_DEFAULT_RENDERER_TIMER_OPTION,
      },
    });
  }

  private async _fetchPlatformSoftware(
    nodeAliases: NodeAliases,
    podReferences: Record<NodeAlias, PodReference>,
    releaseTag: string,
    task: SoloListrTaskWrapper<AnyListrContext>,
    platformInstaller: PlatformInstaller,
    consensusNodes: ConsensusNode[],
    stagingDirectory: string,
  ): Promise<SoloListr<AnyListrContext>> {
    const subTasks: SoloListrTask<AnyListrContext>[] = [];
    const [zipPath, checksumPath] = await platformInstaller.getPlatformRelease(stagingDirectory, releaseTag);
    for (const nodeAlias of nodeAliases) {
      const context: string = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
      const podReference: PodReference = podReferences[nodeAlias];
      subTasks.push({
        title: `Update node: ${chalk.yellow(nodeAlias)} [ platformVersion = ${releaseTag}, context = ${context} ]`,
        task: async (): Promise<void> => {
          await platformInstaller.fetchPlatform(podReference, releaseTag, zipPath, checksumPath, context);
        },
      });
    }

    // set up the sub-tasks
    return task.newListr(subTasks, {
      concurrent: true, // since we download in the container directly, we want this to be in parallel across all nodes
      rendererOptions: {
        collapseSubtasks: false,
      },
    });
  }

  private _checkNodeActivenessTask(
    context_: AnyListrContext,
    task: SoloListrTaskWrapper<AnyListrContext>,
    nodeAliases: NodeAliases,
    status: NodeStatusCodes = NodeStatusCodes.ACTIVE,
  ): SoloListr<AnyListrContext> {
    const {
      config: {namespace},
    } = context_;

    const enableDebugger: boolean = context_.config.debugNodeAlias && status !== NodeStatusCodes.FREEZE_COMPLETE;
    const debugNodeAlias: NodeAlias | undefined = context_.config.debugNodeAlias;

    const subTasks: {
      title: string;
      task: (context_: AnyListrContext, task: SoloListrTaskWrapper<AnyListrContext>) => Promise<void>;
    }[] = nodeAliases.map(
      (
        nodeAlias,
      ): {
        title: string;
        task: (context_: AnyListrContext, task: SoloListrTaskWrapper<AnyListrContext>) => Promise<void>;
      } => {
        const isDebugNode: boolean = debugNodeAlias === nodeAlias && status !== NodeStatusCodes.FREEZE_COMPLETE;
        const reminder: string = isDebugNode ? 'Please attach JVM debugger now.' : '';
        const title: string = `Check network pod: ${chalk.yellow(nodeAlias)} ${chalk.red(reminder)}`;
        const context: string = helpers.extractContextFromConsensusNodes(
          nodeAlias,
          this.remoteConfig.getConsensusNodes(),
        );

        return {
          title,
          task: async (context_: AnyListrContext, task: SoloListrTaskWrapper<AnyListrContext>): Promise<void> => {
            if (enableDebugger && isDebugNode) {
              await task.prompt(ListrInquirerPromptAdapter).run(confirmPrompt, {
                message: `JVM debugger setup for ${nodeAlias}. Continue when debugging is complete?`,
                default: false,
              });
            }

            context_.config.podRefs[nodeAlias] = await this.checkNetworkNodeActiveness(
              namespace,
              nodeAlias,
              task,
              title,
              status,
              undefined,
              undefined,
              undefined,
              context,
            );
          },
        };
      },
    );

    return task.newListr(subTasks, {
      concurrent: !enableDebugger, // Run sequentially when debugging to avoid multiple prompts
      rendererOptions: {
        collapseSubtasks: false,
      },
    });
  }

  public async checkNetworkNodeActiveness(
    namespace: NamespaceName,
    nodeAlias: NodeAlias,
    task: SoloListrTaskWrapper<AnyListrContext>,
    title: string,
    status: NodeStatusCodes = NodeStatusCodes.ACTIVE,
    maxAttempts: number = constants.NETWORK_NODE_ACTIVE_MAX_ATTEMPTS,
    delay: number = constants.NETWORK_NODE_ACTIVE_DELAY,
    timeout: number = constants.NETWORK_NODE_ACTIVE_TIMEOUT,
    context?: string,
  ): Promise<PodReference> {
    const podName: PodName = Templates.renderNetworkPodName(nodeAlias);
    const podReference: PodReference = PodReference.of(namespace, podName);
    task.title = `${title} - status ${chalk.yellow('STARTING')}, attempt ${chalk.blueBright(`0/${maxAttempts}`)}`;

    const consensusNodes: ConsensusNode[] = this.remoteConfig.getConsensusNodes();

    if (typeof context !== 'string' || context.trim().length === 0) {
      context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
    }

    let attempt: number = 0;
    let success: boolean = false;
    while (attempt < maxAttempts) {
      const controller: AbortController = new AbortController();

      const timeoutId: NodeJS.Timeout = setTimeout((): void => {
        task.title = `${title} - status ${chalk.yellow('TIMEOUT')}, attempt ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
        controller.abort();
      }, timeout);

      try {
        const response: string = await container
          .resolve<NetworkNodes>(InjectTokens.NetworkNodes)
          .getNetworkNodePodStatus(podReference, context);

        if (!response) {
          task.title = `${title} - status ${chalk.yellow('UNKNOWN')}, attempt ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
          clearTimeout(timeoutId);
          throw new SoloError('empty response'); // Guard
        }

        const statusLine: string = response
          .split('\n')
          .find((line: string): boolean => line.startsWith('platform_PlatformStatus'));

        if (!statusLine) {
          task.title = `${title} - status ${chalk.yellow('STARTING')}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
          clearTimeout(timeoutId);
          throw new SoloError('missing status line'); // Guard
        }

        const statusNumber: number = Number.parseInt(statusLine.split(' ').pop());

        if (statusNumber === status) {
          task.title = `${title} - status ${chalk.green(NodeStatusEnums[status])}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
          success = true;
          clearTimeout(timeoutId);
          break;
        } else if (statusNumber === NodeStatusCodes.CATASTROPHIC_FAILURE) {
          task.title = `${title} - status ${chalk.red('CATASTROPHIC_FAILURE')}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
          break;
        } else if (statusNumber) {
          task.title = `${title} - status ${chalk.yellow(NodeStatusEnums[statusNumber])}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
        }
        clearTimeout(timeoutId);
      } catch (error) {
        this.logger.debug(
          `${title} : Error in checking node activeness: attempt: ${attempt}/${maxAttempts}: ${JSON.stringify(error)}`,
        );
      }

      attempt++;
      clearTimeout(timeoutId);
      await sleep(Duration.ofMillis(delay));
    }

    if (!success) {
      throw new SoloError(
        `node '${nodeAlias}' is not ${NodeStatusEnums[status]}` +
          `[ attempt = ${chalk.blueBright(`${attempt}/${maxAttempts}`)} ]`,
      );
    }

    if (constants.NETWORK_NODE_ACTIVE_EXTRA_DELAY_MS > 0) {
      await sleep(Duration.ofMillis(constants.NETWORK_NODE_ACTIVE_EXTRA_DELAY_MS)); // delaying prevents - gRPC service error
    }

    return podReference;
  }

  /** Return task for check if node proxies are ready */
  private _checkNodesProxiesTask(
    task: SoloListrTaskWrapper<{config: {consensusNodes: ConsensusNode[]; namespace: NamespaceName}}>,
    nodeAliases: NodeAliases,
  ): SoloListr<{config: {consensusNodes: ConsensusNode[]; namespace: NamespaceName}}> {
    const subTasks: SoloListrTask<{config: {consensusNodes: ConsensusNode[]; namespace: NamespaceName}}>[] = [];

    for (const nodeAlias of nodeAliases) {
      subTasks.push({
        title: `Check proxy for node: ${chalk.yellow(nodeAlias)}`,
        task: async (context_): Promise<void> => {
          const context: string = helpers.extractContextFromConsensusNodes(nodeAlias, context_.config.consensusNodes);
          const k8: K8 = this.k8Factory.getK8(context);
          await k8
            .pods()
            .waitForReadyStatus(
              context_.config.namespace,
              [`app=haproxy-${nodeAlias}`, 'solo.hedera.com/type=haproxy'],
              constants.NETWORK_PROXY_MAX_ATTEMPTS,
              constants.NETWORK_PROXY_DELAY,
            );
        },
      });
    }

    // set up the sub-tasks
    return task.newListr(subTasks, {
      concurrent: true,
      rendererOptions: {
        collapseSubtasks: false,
      },
    });
  }

  /**
   * When generating multiple all aliases are read from config.nodeAliases,
   * When generating a single key the alias in config.nodeAlias is used
   */
  private _generateGossipKeys(generateMultiple: boolean): SoloListrTask<NodeKeysContext | NodeAddContext> {
    return {
      title: 'Generate gossip keys',
      task: ({config}, task): any => {
        const nodeAliases: NodeAlias[] = generateMultiple
          ? (config as NodeKeysConfigClass).nodeAliases
          : [(config as NodeAddConfigClass).nodeAlias];
        const subTasks: SoloListrTask<NodeKeysContext | NodeAddContext>[] = this.keyManager.taskGenerateGossipKeys(
          nodeAliases,
          config.keysDir,
          config.curDate,
        );
        // set up the sub-tasks
        return task.newListr(subTasks, constants.LISTR_DEFAULT_OPTIONS.DEFAULT);
      },
      skip: (context_): boolean => !context_.config.generateGossipKeys,
    };
  }

  /**
   * When generating multiple all aliases are read from config.nodeAliases,
   * When generating a single key the alias in config.nodeAlias is used
   */
  private _generateGrpcTlsKeys(generateMultiple: boolean): SoloListrTask<NodeKeysContext | NodeAddContext> {
    return {
      title: 'Generate gRPC TLS Keys',
      task: (context_, task): SoloListr<NodeKeysContext | NodeAddContext> => {
        const config: NodeAddConfigClass | NodeKeysConfigClass = context_.config;
        const nodeAliases: NodeAlias[] = generateMultiple
          ? (config as NodeKeysConfigClass).nodeAliases
          : [(config as NodeAddConfigClass).nodeAlias];
        const subTasks: SoloListrTask<any>[] = this.keyManager.taskGenerateTLSKeys(
          nodeAliases,
          config.keysDir,
          config.curDate,
        );
        // set up the sub-tasks
        return task.newListr(subTasks, constants.LISTR_DEFAULT_OPTIONS.WITH_CONCURRENCY);
      },
      skip: (context_): boolean => !context_.config.generateTlsKeys,
    };
  }

  public copyGrpcTlsCertificates(): SoloListrTask<NodeAddContext> {
    return {
      title: 'Copy gRPC TLS Certificates',
      task: ({config}, task): SoloListr<AnyListrContext> =>
        this.certificateManager.buildCopyTlsCertificatesTasks(
          task,
          config.grpcTlsCertificatePath,
          config.grpcWebTlsCertificatePath,
          config.grpcTlsKeyPath,
          config.grpcWebTlsKeyPath,
        ),
      skip: (context_): boolean =>
        !context_.config.grpcTlsCertificatePath && !context_.config.grpcWebTlsCertificatePath,
    };
  }

  private async _addStake(
    namespace: NamespaceName,
    accountId: string,
    nodeAlias: NodeAlias,
    stakeAmount: number = HEDERA_NODE_DEFAULT_STAKE_AMOUNT,
  ): Promise<void> {
    try {
      const deploymentName: DeploymentName = this.configManager.getFlag(flags.deployment);
      await this.accountManager.loadNodeClient(
        namespace,
        this.remoteConfig.getClusterRefs(),
        deploymentName,
        this.configManager.getFlag<boolean>(flags.forcePortForward),
      );
      const client: Client = this.accountManager._nodeClient;
      const treasuryKey: AccountIdWithKeyPairObject = await this.accountManager.getTreasuryAccountKeys(
        namespace,
        deploymentName,
      );

      const treasuryPrivateKey: PrivateKey = PrivateKey.fromStringED25519(treasuryKey.privateKey);
      const treasuryAccountId: AccountId = this.accountManager.getTreasuryAccountId(deploymentName);
      client.setOperator(treasuryAccountId, treasuryPrivateKey);

      // check balance
      const treasuryBalance: AccountBalance = await new AccountBalanceQuery()
        .setAccountId(treasuryAccountId)
        .execute(client);

      this.logger.debug(`Account ${treasuryAccountId} balance: ${treasuryBalance.hbars}`);

      // get some initial balance
      await this.accountManager.transferAmount(treasuryAccountId, accountId, stakeAmount);

      // check balance
      const balance: AccountBalance = await new AccountBalanceQuery().setAccountId(accountId).execute(client);
      this.logger.debug(`Account ${accountId} balance: ${balance.hbars}`);

      // Create the transaction
      const transaction: AccountUpdateTransaction = new AccountUpdateTransaction()
        .setAccountId(accountId)
        .setStakedNodeId(Templates.nodeIdFromNodeAlias(nodeAlias))
        .freezeWith(client);

      // Sign the transaction with the account's private key
      const signTransaction: AccountUpdateTransaction = await transaction.sign(treasuryPrivateKey);

      const transactionResponse: TransactionResponse = await signTransaction.execute(client);

      const receipt: TransactionReceipt = await transactionResponse.getReceipt(client);

      this.logger.debug(`The transaction consensus status is ${receipt.status}`);
    } catch (error) {
      throw new SoloError(`Error in adding stake: ${error.message}`, error);
    }
  }

  public prepareUpgradeZip(): SoloListrTask<AnyListrContext> {
    return {
      title: 'Prepare upgrade zip file for node upgrade process',
      task: async (context_): Promise<void> => {
        const config: NodeAddConfigClass | NodeUpdateConfigClass | NodeUpgradeConfigClass | NodeDestroyConfigClass =
          context_.config;
        const {upgradeZipFile, deployment}: any = context_.config;
        if (upgradeZipFile) {
          context_.upgradeZipFile = upgradeZipFile;
          this.logger.debug(`Using upgrade zip file: ${context_.upgradeZipFile}`);
        } else {
          // download application.properties from the first node in the deployment
          const nodeAlias: NodeAlias = config.existingNodeAliases[0];

          const nodeFullyQualifiedPodName: PodName = Templates.renderNetworkPodName(nodeAlias);
          const podReference: PodReference = PodReference.of(config.namespace, nodeFullyQualifiedPodName);
          const containerReference: ContainerReference = ContainerReference.of(podReference, constants.ROOT_CONTAINER);

          const context: string = helpers.extractContextFromConsensusNodes(
            (context_ as NodeUpdateContext | NodeDestroyContext).config.nodeAlias,
            context_.config.consensusNodes,
          );

          const templatesDirectory: string = PathEx.join(config.stagingDir, 'templates');
          fs.mkdirSync(templatesDirectory, {recursive: true});

          await this.k8Factory
            .getK8(context)
            .containers()
            .readByRef(containerReference)
            .copyFrom(
              `${constants.HEDERA_HAPI_PATH}/data/config/${constants.APPLICATION_PROPERTIES}`,
              templatesDirectory,
            );

          const upgradeVersion: string | undefined =
            'upgradeVersion' in config ? (config.upgradeVersion as string) : undefined;
          context_.upgradeZipFile = await this._prepareUpgradeZip(config.stagingDir, upgradeVersion);
        }
        context_.upgradeZipHash = await this._uploadUpgradeZip(context_.upgradeZipFile, config.nodeClient, deployment);
      },
    };
  }

  public loadAdminKey(): SoloListrTask<NodeUpdateContext | NodeUpgradeContext | NodeDestroyContext> {
    return {
      title: 'Load node admin key',
      task: async (context_): Promise<void> => {
        const config: NodeUpdateConfigClass | NodeUpgradeConfigClass | NodeDestroyConfigClass = context_.config;
        if ((context_ as NodeUpdateContext | NodeDestroyContext).config.nodeAlias) {
          try {
            const context: string = helpers.extractContextFromConsensusNodes(
              (context_ as NodeUpdateContext | NodeDestroyContext).config.nodeAlias,
              context_.config.consensusNodes,
            );

            // load nodeAdminKey from k8s if exist
            const keyFromK8: Secret = await this.k8Factory
              .getK8(context)
              .secrets()
              .read(
                config.namespace,
                Templates.renderNodeAdminKeyName((context_ as NodeUpdateContext | NodeDestroyContext).config.nodeAlias),
              );
            const privateKey: string = Base64.decode(keyFromK8.data.privateKey);
            config.adminKey = PrivateKey.fromStringED25519(privateKey);
          } catch (error) {
            this.logger.debug(`Error in loading node admin key: ${error.message}, use default key`);
            config.adminKey = PrivateKey.fromStringED25519(constants.GENESIS_KEY);
          }
        } else {
          config.adminKey = PrivateKey.fromStringED25519(constants.GENESIS_KEY);
        }
      },
    };
  }

  public checkExistingNodesStakedAmount(): SoloListrTask<
    NodeUpdateContext | NodeAddContext | NodeDestroyContext | NodeUpgradeContext
  > {
    return {
      title: 'Check existing nodes staked amount',
      task: async ({config}): Promise<void> => {
        // Transfer some hbar to the node for staking purpose
        const deploymentName: DeploymentName = this.configManager.getFlag(flags.deployment);
        const accountMap: Map<NodeAlias, string> = this.accountManager.getNodeAccountMap(
          config.existingNodeAliases,
          deploymentName,
        );
        const treasuryAccountId: AccountId = this.accountManager.getTreasuryAccountId(deploymentName);
        for (const nodeAlias of config.existingNodeAliases) {
          const accountId: string = accountMap.get(nodeAlias)!;
          await this.accountManager.transferAmount(treasuryAccountId, accountId, 1);
        }
      },
    };
  }

  public sendPrepareUpgradeTransaction(): SoloListrTask<
    NodeUpdateContext | NodeAddContext | NodeDestroyContext | NodeUpgradeContext
  > {
    return {
      title: 'Send prepare upgrade transaction',
      task: async (context_): Promise<void> => {
        const {upgradeZipHash} = context_;
        const {nodeClient, freezeAdminPrivateKey, deployment} = context_.config;
        try {
          const freezeAccountId: AccountId = this.accountManager.getFreezeAccountId(deployment);
          const treasuryAccountId: AccountId = this.accountManager.getTreasuryAccountId(deployment);

          // query the balance
          const balance: AccountBalance = await new AccountBalanceQuery()
            .setAccountId(freezeAccountId)
            .execute(nodeClient);

          this.logger.debug(`Freeze admin account balance: ${balance.hbars}`);

          // transfer some tiny amount to the freeze admin account
          await this.accountManager.transferAmount(treasuryAccountId, freezeAccountId, 100_000);

          // set operator of freeze transaction as freeze admin account
          nodeClient.setOperator(freezeAccountId, freezeAdminPrivateKey);

          const prepareUpgradeTransaction: TransactionResponse = await new FreezeTransaction()
            .setFreezeType(FreezeType.PrepareUpgrade)
            .setFileId(this.getFileUpgradeId(deployment))
            .setFileHash(upgradeZipHash)
            .freezeWith(nodeClient)
            .execute(nodeClient);

          const prepareUpgradeReceipt: TransactionReceipt = await prepareUpgradeTransaction.getReceipt(nodeClient);

          this.logger.debug(
            `sent prepare upgrade transaction [id: ${prepareUpgradeTransaction.transactionId.toString()}]`,
            prepareUpgradeReceipt.status.toString(),
          );

          if (prepareUpgradeReceipt.status !== Status.Success) {
            throw new SoloError(`Prepare upgrade transaction failed: ${prepareUpgradeReceipt.status}`);
          }
        } catch (error) {
          throw new SoloError(`Error in prepare upgrade: ${error.message}`, error);
        }
      },
    };
  }

  public sendFreezeUpgradeTransaction(): SoloListrTask<
    NodeUpdateContext | NodeAddContext | NodeDestroyContext | NodeUpgradeContext
  > {
    return {
      title: 'Send freeze upgrade transaction',
      task: async (context_): Promise<void> => {
        const {upgradeZipHash} = context_;
        const {freezeAdminPrivateKey, nodeClient, deployment} = context_.config;
        try {
          const futureDate: Date = new Date();
          this.logger.debug(`Current time: ${futureDate}`);

          futureDate.setTime(futureDate.getTime() + 5000); // 5 seconds in the future
          this.logger.debug(`Freeze time: ${futureDate}`);

          const freezeAdminAccountId: AccountId = this.accountManager.getFreezeAccountId(deployment);

          // query the balance
          const balance: AccountBalance = await new AccountBalanceQuery()
            .setAccountId(freezeAdminAccountId)
            .execute(nodeClient);

          this.logger.debug(`Freeze admin account balance: ${balance.hbars}`);

          nodeClient.setOperator(freezeAdminAccountId, freezeAdminPrivateKey);
          const freezeUpgradeTx: TransactionResponse = await new FreezeTransaction()
            .setFreezeType(FreezeType.FreezeUpgrade)
            .setStartTimestamp(Timestamp.fromDate(futureDate))
            .setFileId(this.getFileUpgradeId(deployment))
            .setFileHash(upgradeZipHash)
            .freezeWith(nodeClient)
            .execute(nodeClient);

          const freezeUpgradeReceipt: TransactionReceipt = await freezeUpgradeTx.getReceipt(nodeClient);
          this.logger.debug(
            `Upgrade frozen with transaction id: ${freezeUpgradeTx.transactionId.toString()}`,
            freezeUpgradeReceipt.status.toString(),
          );
        } catch (error) {
          throw new SoloError(`Error in freeze upgrade: ${error.message}`, error);
        }
      },
    };
  }

  public sendFreezeTransaction(): SoloListrTask<NodeFreezeContext> {
    return {
      title: 'Send freeze only transaction',
      task: async (context_): Promise<void> => {
        const {freezeAdminPrivateKey, deployment, namespace}: any = context_.config;
        try {
          const nodeClient: Client = await this.accountManager.loadNodeClient(
            namespace,
            this.remoteConfig.getClusterRefs(),
            deployment,
          );
          const futureDate: Date = new Date();
          this.logger.debug(`Current time: ${futureDate}`);

          futureDate.setTime(futureDate.getTime() + 5000); // 5 seconds in the future
          this.logger.debug(`Freeze time: ${futureDate}`);

          const freezeAdminAccountId: AccountId = this.accountManager.getFreezeAccountId(deployment);
          nodeClient.setOperator(freezeAdminAccountId, freezeAdminPrivateKey);
          const freezeOnlyTransaction: TransactionResponse = await new FreezeTransaction()
            .setFreezeType(FreezeType.FreezeOnly)
            .setStartTimestamp(Timestamp.fromDate(futureDate))
            .freezeWith(nodeClient)
            .execute(nodeClient);

          const freezeOnlyReceipt: TransactionReceipt = await freezeOnlyTransaction.getReceipt(nodeClient);

          this.logger.debug(
            `sent prepare transaction [id: ${freezeOnlyTransaction.transactionId.toString()}]`,
            freezeOnlyReceipt.status.toString(),
          );
        } catch (error) {
          throw new SoloError(`Error in sending freeze transaction: ${error.message}`, error);
        }
      },
    };
  }

  /** Download generated config files and key files from the network node,
   *  This function should only be called when updating or destroying a node
   * */
  public downloadNodeGeneratedFilesForDynamicAddressBook(): SoloListrTask<
    NodeUpdateContext | NodeAddContext | NodeDestroyContext
  > {
    return {
      title: 'Download generated files from an existing node',
      task: async ({
        config: {nodeAlias, existingNodeAliases, consensusNodes, stagingDir, keysDir, namespace},
      }): Promise<void> => {
        // don't try to download from the same node we are deleting, it won't work
        const targetNodeAlias: NodeAlias =
          nodeAlias === existingNodeAliases[0] && existingNodeAliases.length > 1
            ? existingNodeAliases[1]
            : existingNodeAliases[0];

        const nodeFullyQualifiedPodName: PodName = Templates.renderNetworkPodName(targetNodeAlias);
        const podReference: PodReference = PodReference.of(namespace, nodeFullyQualifiedPodName);
        const containerReference: ContainerReference = ContainerReference.of(podReference, constants.ROOT_CONTAINER);

        const context: Context = helpers.extractContextFromConsensusNodes(targetNodeAlias, consensusNodes);

        const k8Container: Container = this.k8Factory.getK8(context).containers().readByRef(containerReference);

        const consensusVersion: SemanticVersion<string> | undefined =
          this.remoteConfig.configuration?.versions?.consensusNode;
        const releaseTag: string = consensusVersion?.toString() || HEDERA_PLATFORM_VERSION;
        const needsConfigTxt: boolean = needsConfigTxtForConsensusVersion(releaseTag);
        const configSource: string = `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/config.txt`;
        if (needsConfigTxt && (await k8Container.hasFile(configSource))) {
          // copy the config.txt file from the node1 upgrade directory if it exists
          await k8Container.copyFrom(configSource, stagingDir);
        }

        // if directory data/upgrade/current/data/keys does not exist, then