@hashgraph/solo/dist/src/commands/node/tasks.js

An opinionated CLI tool to deploy and manage private Hedera Networks.

import { Zippy } from '../../core/zippy.js';
import * as constants from '../../core/constants.js';
import { DEFAULT_NETWORK_NODE_NAME, FREEZE_ADMIN_ACCOUNT, HEDERA_NODE_DEFAULT_STAKE_AMOUNT, IGNORED_NODE_ACCOUNT_ID, TREASURY_ACCOUNT_ID, } from '../../core/constants.js';
import { Templates } from '../../core/templates.js';
import { Task } from '../../core/task.js';
import { AccountBalanceQuery, AccountId, AccountUpdateTransaction, FileAppendTransaction, FileUpdateTransaction, FreezeTransaction, FreezeType, Long, NodeCreateTransaction, NodeDeleteTransaction, NodeUpdateTransaction, PrivateKey, Timestamp, } from '@hashgraph/sdk';
import { IllegalArgumentError, MissingArgumentError, SoloError } from '../../core/errors.js';
import path from 'path';
import fs from 'fs';
import crypto from 'crypto';
import * as helpers from '../../core/helpers.js';
import { addDebugOptions, getNodeAccountMap, prepareEndpoints, renameAndCopyFile, sleep, splitFlagInput, } from '../../core/helpers.js';
import chalk from 'chalk';
import { Flags as flags } from '../flags.js';
import { PodName } from '../../core/kube/resources/pod/pod_name.js';
import { NodeStatusCodes, NodeStatusEnums, NodeSubcommandType } from '../../core/enumerations.js';
import { ListrLease } from '../../core/lease/listr_lease.js';
import { Duration } from '../../core/time/duration.js';
import { GenesisNetworkDataConstructor } from '../../core/genesis_network_models/genesis_network_data_constructor.js';
import { NodeOverridesModel } from '../../core/node_overrides_model.js';
import { PodRef } from '../../core/kube/resources/pod/pod_ref.js';
import { ContainerRef } from '../../core/kube/resources/container/container_ref.js';
import { NetworkNodes } from '../../core/network_nodes.js';
import { container } from 'tsyringe-neo';
import { ConsensusNode } from '../../core/model/consensus_node.js';
import { Base64 } from 'js-base64';
export class NodeCommandTasks {
    accountManager;
    configManager;
    keyManager;
    profileManager;
    platformInstaller;
    logger;
    k8Factory;
    parent;
    chartManager;
    certificateManager;
    prepareValuesFiles;
    constructor(opts) {
        if (!opts || !opts.accountManager)
            throw new IllegalArgumentError('An instance of core/AccountManager is required', opts.accountManager);
        if (!opts || !opts.configManager)
            throw new Error('An instance of core/ConfigManager is required');
        if (!opts || !opts.logger)
            throw new Error('An instance of core/Logger is required');
        if (!opts || !opts.k8Factory)
            throw new Error('An instance of core/K8Factory is required');
        if (!opts || !opts.platformInstaller)
            throw new IllegalArgumentError('An instance of core/PlatformInstaller is required', opts.platformInstaller);
        if (!opts || !opts.keyManager)
            throw new IllegalArgumentError('An instance of core/KeyManager is required', opts.keyManager);
        if (!opts || !opts.profileManager)
            throw new IllegalArgumentError('An instance of ProfileManager is required', opts.profileManager);
        if (!opts || !opts.certificateManager)
            throw new IllegalArgumentError('An instance of CertificateManager is required', opts.certificateManager);
        if (!opts || !opts.parent)
            throw new IllegalArgumentError('An instance of parents as BaseCommand is required', opts.parent);
        this.accountManager = opts.accountManager;
        this.configManager = opts.configManager;
        this.logger = opts.logger;
        this.k8Factory = opts.k8Factory;
        this.platformInstaller = opts.platformInstaller;
        this.profileManager = opts.profileManager;
        this.keyManager = opts.keyManager;
        this.chartManager = opts.chartManager;
        this.certificateManager = opts.certificateManager;
        this.prepareValuesFiles = opts.parent.prepareValuesFiles.bind(opts.parent);
        this.parent = opts.parent;
    }
    async _prepareUpgradeZip(stagingDir) {
        // we build a mock upgrade.zip file as we really don't need to upgrade the network
        // also the platform zip file is ~80Mb in size requiring a lot of transactions since the max
        // transaction size is 6Kb and in practice we need to send the file as 4Kb chunks.
        // Note however that in DAB phase-2, we won't need to trigger this fake upgrade process
        const zipper = new Zippy(this.logger);
        const upgradeConfigDir = path.join(stagingDir, 'mock-upgrade', 'data', 'config');
        if (!fs.existsSync(upgradeConfigDir)) {
            fs.mkdirSync(upgradeConfigDir, { recursive: true });
        }
        // bump field hedera.config.version
        const fileBytes = fs.readFileSync(path.join(stagingDir, 'templates', 'application.properties'));
        const lines = fileBytes.toString().split('\n');
        const newLines = [];
        for (let line of lines) {
            line = line.trim();
            const parts = line.split('=');
            if (parts.length === 2) {
                if (parts[0] === 'hedera.config.version') {
                    let version = parseInt(parts[1]);
                    line = `hedera.config.version=${++version}`;
                }
                newLines.push(line);
            }
        }
        fs.writeFileSync(path.join(upgradeConfigDir, 'application.properties'), newLines.join('\n'));
        return await zipper.zip(path.join(stagingDir, 'mock-upgrade'), path.join(stagingDir, 'mock-upgrade.zip'));
    }
    async _uploadUpgradeZip(upgradeZipFile, nodeClient) {
        // get byte value of the zip file
        const zipBytes = fs.readFileSync(upgradeZipFile);
        // @ts-ignore
        const zipHash = crypto.createHash('sha384').update(zipBytes).digest('hex');
        this.logger.debug(`loaded upgrade zip file [ zipHash = ${zipHash} zipBytes.length = ${zipBytes.length}, zipPath = ${upgradeZipFile}]`);
        // create a file upload transaction to upload file to the network
        try {
            let start = 0;
            while (start < zipBytes.length) {
                const zipBytesChunk = new Uint8Array(zipBytes.subarray(start, start + constants.UPGRADE_FILE_CHUNK_SIZE));
                let fileTransaction = null;
                if (start === 0) {
                    fileTransaction = new FileUpdateTransaction().setFileId(constants.UPGRADE_FILE_ID).setContents(zipBytesChunk);
                }
                else {
                    fileTransaction = new FileAppendTransaction().setFileId(constants.UPGRADE_FILE_ID).setContents(zipBytesChunk);
                }
                const resp = await fileTransaction.execute(nodeClient);
                const receipt = await resp.getReceipt(nodeClient);
                this.logger.debug(`updated file ${constants.UPGRADE_FILE_ID} [chunkSize= ${zipBytesChunk.length}, txReceipt = ${receipt.toString()}]`);
                start += constants.UPGRADE_FILE_CHUNK_SIZE;
                this.logger.debug(`uploaded ${start} bytes of ${zipBytes.length} bytes`);
            }
            return zipHash;
        }
        catch (e) {
            throw new SoloError(`failed to upload build.zip file: ${e.message}`, e);
        }
    }
    async copyLocalBuildPathToNode(k8, podRef, configManager, localDataLibBuildPath) {
        const filterFunction = (path, stat) => {
            return !(path.includes('data/keys') || path.includes('data/config'));
        };
        await k8
            .containers()
            .readByRef(ContainerRef.of(podRef, constants.ROOT_CONTAINER))
            .copyTo(localDataLibBuildPath, `${constants.HEDERA_HAPI_PATH}`, filterFunction);
        if (configManager.getFlag(flags.appConfig)) {
            const testJsonFiles = configManager.getFlag(flags.appConfig).split(',');
            for (const jsonFile of testJsonFiles) {
                if (fs.existsSync(jsonFile)) {
                    await k8
                        .containers()
                        .readByRef(ContainerRef.of(podRef, constants.ROOT_CONTAINER))
                        .copyTo(jsonFile, `${constants.HEDERA_HAPI_PATH}`);
                }
            }
        }
    }
    _uploadPlatformSoftware(nodeAliases, podRefs, task, localBuildPath, consensusNodes) {
        const subTasks = [];
        this.logger.debug('no need to fetch, use local build jar files');
        const buildPathMap = new Map();
        let defaultDataLibBuildPath;
        const parameterPairs = localBuildPath.split(',');
        for (const parameterPair of parameterPairs) {
            if (parameterPair.includes('=')) {
                const [nodeAlias, localDataLibBuildPath] = parameterPair.split('=');
                buildPathMap.set(nodeAlias, localDataLibBuildPath);
            }
            else {
                defaultDataLibBuildPath = parameterPair;
            }
        }
        let localDataLibBuildPath;
        for (const nodeAlias of nodeAliases) {
            const podRef = podRefs[nodeAlias];
            const context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
            if (buildPathMap.has(nodeAlias)) {
                localDataLibBuildPath = buildPathMap.get(nodeAlias);
            }
            else {
                localDataLibBuildPath = defaultDataLibBuildPath;
            }
            if (!fs.existsSync(localDataLibBuildPath)) {
                throw new SoloError(`local build path does not exist: ${localDataLibBuildPath}`);
            }
            const self = this;
            const k8 = self.k8Factory.getK8(context);
            subTasks.push({
                title: `Copy local build to Node: ${chalk.yellow(nodeAlias)} from ${localDataLibBuildPath}`,
                task: async () => {
                    // retry copying the build to the node to handle edge cases during performance testing
                    let error = null;
                    let i = 0;
                    for (; i < constants.LOCAL_BUILD_COPY_RETRY; i++) {
                        error = null;
                        try {
                            // filter the data/config and data/keys to avoid failures due to config and secret mounts
                            await self.copyLocalBuildPathToNode(k8, podRef, self.configManager, localDataLibBuildPath);
                        }
                        catch (e) {
                            error = e;
                        }
                    }
                    if (error) {
                        throw new SoloError(`Error in copying local build to node: ${error.message}`, error);
                    }
                },
            });
        }
        // set up the sub-tasks
        return task.newListr(subTasks, {
            concurrent: constants.NODE_COPY_CONCURRENT,
            rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION,
        });
    }
    _fetchPlatformSoftware(nodeAliases, podRefs, releaseTag, task, platformInstaller, consensusNodes) {
        const subTasks = [];
        for (const nodeAlias of nodeAliases) {
            const context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
            const podRef = podRefs[nodeAlias];
            subTasks.push({
                title: `Update node: ${chalk.yellow(nodeAlias)} [ platformVersion = ${releaseTag}, context = ${context} ]`,
                task: async () => await platformInstaller.fetchPlatform(podRef, releaseTag, context),
            });
        }
        // set up the sub-tasks
        return task.newListr(subTasks, {
            concurrent: true, // since we download in the container directly, we want this to be in parallel across all nodes
            rendererOptions: {
                collapseSubtasks: false,
            },
        });
    }
    _checkNodeActivenessTask(ctx, task, nodeAliases, status = NodeStatusCodes.ACTIVE) {
        const { config: { namespace }, } = ctx;
        const enableDebugger = ctx.config.debugNodeAlias && status !== NodeStatusCodes.FREEZE_COMPLETE;
        const subTasks = nodeAliases.map((nodeAlias, i) => {
            const reminder = 'debugNodeAlias' in ctx.config &&
                ctx.config.debugNodeAlias === nodeAlias &&
                status !== NodeStatusCodes.FREEZE_COMPLETE
                ? 'Please attach JVM debugger now.  Sleeping for 1 hour, hit ctrl-c once debugging is complete.'
                : '';
            const title = `Check network pod: ${chalk.yellow(nodeAlias)} ${chalk.red(reminder)}`;
            const context = helpers.extractContextFromConsensusNodes(nodeAlias, ctx.config.consensusNodes);
            const subTask = async (ctx, task) => {
                if (enableDebugger) {
                    await sleep(Duration.ofHours(1));
                }
                ctx.config.podRefs[nodeAlias] = await this._checkNetworkNodeActiveness(namespace, nodeAlias, task, title, i, status, undefined, undefined, undefined, context);
            };
            return { title, task: subTask };
        });
        return task.newListr(subTasks, {
            concurrent: true,
            rendererOptions: {
                collapseSubtasks: false,
            },
        });
    }
    async _checkNetworkNodeActiveness(namespace, nodeAlias, task, title, index, status = NodeStatusCodes.ACTIVE, maxAttempts = constants.NETWORK_NODE_ACTIVE_MAX_ATTEMPTS, delay = constants.NETWORK_NODE_ACTIVE_DELAY, timeout = constants.NETWORK_NODE_ACTIVE_TIMEOUT, context) {
        nodeAlias = nodeAlias.trim();
        const podName = Templates.renderNetworkPodName(nodeAlias);
        const podRef = PodRef.of(namespace, podName);
        task.title = `${title} - status ${chalk.yellow('STARTING')}, attempt ${chalk.blueBright(`0/${maxAttempts}`)}`;
        let attempt = 0;
        let success = false;
        while (attempt < maxAttempts) {
            const controller = new AbortController();
            const timeoutId = setTimeout(() => {
                task.title = `${title} - status ${chalk.yellow('TIMEOUT')}, attempt ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
                controller.abort();
            }, timeout);
            try {
                const response = await this.k8Factory
                    .getK8(context)
                    .containers()
                    .readByRef(ContainerRef.of(podRef, constants.ROOT_CONTAINER))
                    .execContainer([
                    'bash',
                    '-c',
                    'curl -s http://localhost:9999/metrics | grep platform_PlatformStatus | grep -v \\#',
                ]);
                if (!response) {
                    task.title = `${title} - status ${chalk.yellow('UNKNOWN')}, attempt ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
                    clearTimeout(timeoutId);
                    throw new Error('empty response'); // Guard
                }
                const statusLine = response.split('\n').find(line => line.startsWith('platform_PlatformStatus'));
                if (!statusLine) {
                    task.title = `${title} - status ${chalk.yellow('STARTING')}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
                    clearTimeout(timeoutId);
                    throw new Error('missing status line'); // Guard
                }
                const statusNumber = parseInt(statusLine.split(' ').pop());
                if (statusNumber === status) {
                    task.title = `${title} - status ${chalk.green(NodeStatusEnums[status])}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
                    success = true;
                    clearTimeout(timeoutId);
                    break;
                }
                else if (statusNumber === NodeStatusCodes.CATASTROPHIC_FAILURE) {
                    task.title = `${title} - status ${chalk.red('CATASTROPHIC_FAILURE')}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
                    break;
                }
                else if (statusNumber) {
                    task.title = `${title} - status ${chalk.yellow(NodeStatusEnums[statusNumber])}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
                }
                clearTimeout(timeoutId);
            }
            catch (e) {
                this.logger.debug(`${title} : Error in checking node activeness: attempt: ${attempt}/${maxAttempts}: ${JSON.stringify(e)}`);
            }
            attempt++;
            clearTimeout(timeoutId);
            await sleep(Duration.ofMillis(delay));
        }
        if (!success) {
            throw new SoloError(`node '${nodeAlias}' is not ${NodeStatusEnums[status]}` +
                `[ attempt = ${chalk.blueBright(`${attempt}/${maxAttempts}`)} ]`);
        }
        await sleep(Duration.ofSeconds(2)); // delaying prevents - gRPC service error
        return podRef;
    }
    /** Return task for check if node proxies are ready */
    _checkNodesProxiesTask(ctx, task, nodeAliases) {
        const subTasks = [];
        for (const nodeAlias of nodeAliases) {
            subTasks.push({
                title: `Check proxy for node: ${chalk.yellow(nodeAlias)}`,
                task: async (ctx) => {
                    const context = helpers.extractContextFromConsensusNodes(nodeAlias, ctx.config.consensusNodes);
                    const k8 = this.k8Factory.getK8(context);
                    await k8
                        .pods()
                        .waitForReadyStatus(ctx.config.namespace, [`app=haproxy-${nodeAlias}`, 'solo.hedera.com/type=haproxy'], constants.NETWORK_PROXY_MAX_ATTEMPTS, constants.NETWORK_PROXY_DELAY);
                },
            });
        }
        // set up the sub-tasks
        return task.newListr(subTasks, {
            concurrent: false,
            rendererOptions: {
                collapseSubtasks: false,
            },
        });
    }
    /**
     * When generating multiple all aliases are read from config.nodeAliases,
     * When generating a single key the alias in config.nodeAlias is used
     */
    _generateGossipKeys(generateMultiple) {
        const self = this;
        return new Task('Generate gossip keys', (ctx, task) => {
            const config = ctx.config;
            const nodeAliases = generateMultiple ? config.nodeAliases : [config.nodeAlias];
            const subTasks = self.keyManager.taskGenerateGossipKeys(nodeAliases, config.keysDir, config.curDate);
            // set up the sub-tasks
            return task.newListr(subTasks, {
                concurrent: false,
                rendererOptions: {
                    collapseSubtasks: false,
                    timer: constants.LISTR_DEFAULT_RENDERER_TIMER_OPTION,
                },
            });
        }, (ctx) => !ctx.config.generateGossipKeys);
    }
    /**
     * When generating multiple all aliases are read from config.nodeAliases,
     * When generating a single key the alias in config.nodeAlias is used
     */
    _generateGrpcTlsKeys(generateMultiple) {
        const self = this;
        return new Task('Generate gRPC TLS Keys', (ctx, task) => {
            const config = ctx.config;
            const nodeAliases = generateMultiple ? config.nodeAliases : [config.nodeAlias];
            const subTasks = self.keyManager.taskGenerateTLSKeys(nodeAliases, config.keysDir, config.curDate);
            // set up the sub-tasks
            return task.newListr(subTasks, {
                concurrent: true,
                rendererOptions: {
                    collapseSubtasks: false,
                    timer: constants.LISTR_DEFAULT_RENDERER_TIMER_OPTION,
                },
            });
        }, (ctx) => !ctx.config.generateTlsKeys);
    }
    copyGrpcTlsCertificates() {
        const self = this;
        return new Task('Copy gRPC TLS Certificates', (ctx, parentTask) => self.certificateManager.buildCopyTlsCertificatesTasks(parentTask, ctx.config.grpcTlsCertificatePath, ctx.config.grpcWebTlsCertificatePath, ctx.config.grpcTlsKeyPath, ctx.config.grpcWebTlsKeyPath), (ctx) => !ctx.config.grpcTlsCertificatePath && !ctx.config.grpcWebTlsCertificatePath);
    }
    async _addStake(namespace, accountId, nodeAlias, stakeAmount = HEDERA_NODE_DEFAULT_STAKE_AMOUNT, context) {
        try {
            const deploymentName = this.configManager.getFlag(flags.deployment);
            await this.accountManager.loadNodeClient(namespace, this.parent.getClusterRefs(), deploymentName, this.configManager.getFlag(flags.forcePortForward), context);
            const client = this.accountManager._nodeClient;
            const treasuryKey = await this.accountManager.getTreasuryAccountKeys(namespace);
            const treasuryPrivateKey = PrivateKey.fromStringED25519(treasuryKey.privateKey);
            client.setOperator(TREASURY_ACCOUNT_ID, treasuryPrivateKey);
            // check balance
            const treasuryBalance = await new AccountBalanceQuery().setAccountId(TREASURY_ACCOUNT_ID).execute(client);
            this.logger.debug(`Account ${TREASURY_ACCOUNT_ID} balance: ${treasuryBalance.hbars}`);
            // get some initial balance
            await this.accountManager.transferAmount(constants.TREASURY_ACCOUNT_ID, accountId, stakeAmount);
            // check balance
            const balance = await new AccountBalanceQuery().setAccountId(accountId).execute(client);
            this.logger.debug(`Account ${accountId} balance: ${balance.hbars}`);
            // Create the transaction
            const transaction = new AccountUpdateTransaction()
                .setAccountId(accountId)
                .setStakedNodeId(Templates.nodeIdFromNodeAlias(nodeAlias))
                .freezeWith(client);
            // Sign the transaction with the account's private key
            const signTx = await transaction.sign(treasuryPrivateKey);
            // Submit the transaction to a Hedera network
            const txResponse = await signTx.execute(client);
            // Request the receipt of the transaction
            const receipt = await txResponse.getReceipt(client);
            // Get the transaction status
            const transactionStatus = receipt.status;
            this.logger.debug(`The transaction consensus status is ${transactionStatus.toString()}`);
        }
        catch (e) {
            throw new SoloError(`Error in adding stake: ${e.message}`, e);
        }
    }
    prepareUpgradeZip() {
        const self = this;
        return new Task('Prepare upgrade zip file for node upgrade process', async (ctx, task) => {
            const config = ctx.config;
            const { upgradeZipFile } = ctx.config;
            if (upgradeZipFile) {
                this.logger.debug(`Using upgrade zip file: ${ctx.upgradeZipFile}`);
                ctx.upgradeZipFile = upgradeZipFile;
            }
            else {
                ctx.upgradeZipFile = await self._prepareUpgradeZip(config.stagingDir);
            }
            ctx.upgradeZipHash = await self._uploadUpgradeZip(ctx.upgradeZipFile, config.nodeClient);
        });
    }
    loadAdminKey() {
        return new Task('Load node admin key', async (ctx, task) => {
            const config = ctx.config;
            if (ctx.config.nodeAlias) {
                try {
                    // load nodeAdminKey form k8s if exist
                    const keyFromK8 = await this.k8Factory
                        .default()
                        .secrets()
                        .read(config.namespace, Templates.renderNodeAdminKeyName(config.nodeAlias));
                    const privateKey = Base64.decode(keyFromK8.data.privateKey);
                    config.adminKey = PrivateKey.fromStringED25519(privateKey);
                }
                catch (e) {
                    this.logger.debug(`Error in loading node admin key: ${e.message}, use default key`);
                    config.adminKey = PrivateKey.fromStringED25519(constants.GENESIS_KEY);
                }
            }
            else {
                config.adminKey = PrivateKey.fromStringED25519(constants.GENESIS_KEY);
            }
        });
    }
    checkExistingNodesStakedAmount() {
        const self = this;
        return new Task('Check existing nodes staked amount', async (ctx, task) => {
            const config = ctx.config;
            // Transfer some hbar to the node for staking purpose
            const accountMap = getNodeAccountMap(config.existingNodeAliases);
            for (const nodeAlias of config.existingNodeAliases) {
                const accountId = accountMap.get(nodeAlias);
                await self.accountManager.transferAmount(constants.TREASURY_ACCOUNT_ID, accountId, 1);
            }
        });
    }
    sendPrepareUpgradeTransaction() {
        const self = this;
        return new Task('Send prepare upgrade transaction', async (ctx, task) => {
            const { upgradeZipHash } = ctx;
            const { nodeClient, freezeAdminPrivateKey } = ctx.config;
            try {
                // query the balance
                const balance = await new AccountBalanceQuery().setAccountId(FREEZE_ADMIN_ACCOUNT).execute(nodeClient);
                self.logger.debug(`Freeze admin account balance: ${balance.hbars}`);
                // transfer some tiny amount to the freeze admin account
                await self.accountManager.transferAmount(constants.TREASURY_ACCOUNT_ID, FREEZE_ADMIN_ACCOUNT, 100000);
                // set operator of freeze transaction as freeze admin account
                nodeClient.setOperator(FREEZE_ADMIN_ACCOUNT, freezeAdminPrivateKey);
                const prepareUpgradeTx = await new FreezeTransaction()
                    .setFreezeType(FreezeType.PrepareUpgrade)
                    .setFileId(constants.UPGRADE_FILE_ID)
                    .setFileHash(upgradeZipHash)
                    .freezeWith(nodeClient)
                    .execute(nodeClient);
                const prepareUpgradeReceipt = await prepareUpgradeTx.getReceipt(nodeClient);
                self.logger.debug(`sent prepare upgrade transaction [id: ${prepareUpgradeTx.transactionId.toString()}]`, prepareUpgradeReceipt.status.toString());
            }
            catch (e) {
                self.logger.error(`Error in prepare upgrade: ${e.message}`, e);
                throw new SoloError(`Error in prepare upgrade: ${e.message}`, e);
            }
        });
    }
    sendFreezeUpgradeTransaction() {
        const self = this;
        return new Task('Send freeze upgrade transaction', async (ctx, task) => {
            const { upgradeZipHash } = ctx;
            const { freezeAdminPrivateKey, nodeClient } = ctx.config;
            try {
                const futureDate = new Date();
                self.logger.debug(`Current time: ${futureDate}`);
                futureDate.setTime(futureDate.getTime() + 5000); // 5 seconds in the future
                self.logger.debug(`Freeze time: ${futureDate}`);
                // query the balance
                const balance = await new AccountBalanceQuery().setAccountId(FREEZE_ADMIN_ACCOUNT).execute(nodeClient);
                self.logger.debug(`Freeze admin account balance: ${balance.hbars}`);
                nodeClient.setOperator(FREEZE_ADMIN_ACCOUNT, freezeAdminPrivateKey);
                const freezeUpgradeTx = await new FreezeTransaction()
                    .setFreezeType(FreezeType.FreezeUpgrade)
                    .setStartTimestamp(Timestamp.fromDate(futureDate))
                    .setFileId(constants.UPGRADE_FILE_ID)
                    .setFileHash(upgradeZipHash)
                    .freezeWith(nodeClient)
                    .execute(nodeClient);
                const freezeUpgradeReceipt = await freezeUpgradeTx.getReceipt(nodeClient);
                self.logger.debug(`Upgrade frozen with transaction id: ${freezeUpgradeTx.transactionId.toString()}`, freezeUpgradeReceipt.status.toString());
            }
            catch (e) {
                self.logger.error(`Error in freeze upgrade: ${e.message}`, e);
                throw new SoloError(`Error in freeze upgrade: ${e.message}`, e);
            }
        });
    }
    /** Download generated config files and key files from the network node */
    downloadNodeGeneratedFiles() {
        const self = this;
        return new Task('Download generated files from an existing node', async (ctx, task) => {
            const config = ctx.config;
            // don't try to download from the same node we are deleting, it won't work
            const nodeAlias = ctx.config.nodeAlias === config.existingNodeAliases[0] && config.existingNodeAliases.length > 1
                ? config.existingNodeAliases[1]
                : config.existingNodeAliases[0];
            const nodeFullyQualifiedPodName = Templates.renderNetworkPodName(nodeAlias);
            const podRef = PodRef.of(config.namespace, nodeFullyQualifiedPodName);
            const containerRef = ContainerRef.of(podRef, constants.ROOT_CONTAINER);
            // copy the config.txt file from the node1 upgrade directory
            await self.k8Factory
                .default()
                .containers()
                .readByRef(containerRef)
                .copyFrom(`${constants.HEDERA_HAPI_PATH}/data/upgrade/current/config.txt`, config.stagingDir);
            // if directory data/upgrade/current/data/keys does not exist, then use data/upgrade/current
            let keyDir = `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/data/keys`;
            if (!(await self.k8Factory.default().containers().readByRef(containerRef).hasDir(keyDir))) {
                keyDir = `${constants.HEDERA_HAPI_PATH}/data/upgrade/current`;
            }
            const signedKeyFiles = (await self.k8Factory.default().containers().readByRef(containerRef).listDir(keyDir)).filter(file => file.name.startsWith(constants.SIGNING_KEY_PREFIX));
            await self.k8Factory
                .default()
                .containers()
                .readByRef(containerRef)
                .execContainer([
                'bash',
                '-c',
                `mkdir -p ${constants.HEDERA_HAPI_PATH}/data/keys_backup && cp -r ${keyDir} ${constants.HEDERA_HAPI_PATH}/data/keys_backup/`,
            ]);
            for (const signedKeyFile of signedKeyFiles) {
                await self.k8Factory
                    .default()
                    .containers()
                    .readByRef(containerRef)
                    .copyFrom(`${keyDir}/${signedKeyFile.name}`, `${config.keysDir}`);
            }
            if (await self.k8Factory
                .default()
                .containers()
                .readByRef(containerRef)
                .hasFile(`${constants.HEDERA_HAPI_PATH}/data/upgrade/current/application.properties`)) {
                await self.k8Factory
                    .default()
                    .containers()
                    .readByRef(containerRef)
                    .copyFrom(`${constants.HEDERA_HAPI_PATH}/data/upgrade/current/application.properties`, `${config.stagingDir}/templates`);
            }
        });
    }
    downloadNodeUpgradeFiles() {
        const self = this;
        return new Task('Download upgrade files from an existing node', async (ctx, task) => {
            const config = ctx.config;
            const nodeAlias = ctx.config.nodeAliases[0];
            const nodeFullyQualifiedPodName = Templates.renderNetworkPodName(nodeAlias);
            const podRef = PodRef.of(config.namespace, nodeFullyQualifiedPodName);
            // found all files under ${constants.HEDERA_HAPI_PATH}/data/upgrade/current/
            const upgradeDirectories = [
                `${constants.HEDERA_HAPI_PATH}/data/upgrade/current`,
                `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/data/apps`,
                `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/data/libs`,
            ];
            const containerRef = ContainerRef.of(podRef, constants.ROOT_CONTAINER);
            for (const upgradeDir of upgradeDirectories) {
                // check if directory upgradeDir exist in root container
                if (!(await self.k8Factory.default().containers().readByRef(containerRef).hasDir(upgradeDir))) {
                    continue;
                }
                const files = await self.k8Factory.default().containers().readByRef(containerRef).listDir(upgradeDir);
                // iterate all files and copy them to the staging directory
                for (const file of files) {
                    if (file.name.endsWith('.mf')) {
                        continue;
                    }
                    if (file.directory) {
                        continue;
                    }
                    this.logger.debug(`Copying file: ${file.name}`);
                    await self.k8Factory
                        .default()
                        .containers()
                        .readByRef(containerRef)
                        .copyFrom(`${upgradeDir}/${file.name}`, `${config.stagingDir}`);
                }
            }
        });
    }
    taskCheckNetworkNodePods(ctx, task, nodeAliases, maxAttempts = undefined) {
        if (!ctx.config)
            ctx.config = {};
        ctx.config.podRefs = {};
        const consensusNodes = ctx.config.consensusNodes;
        const subTasks = [];
        const self = this;
        for (const nodeAlias of nodeAliases) {
            const context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
            subTasks.push({
                title: `Check network pod: ${chalk.yellow(nodeAlias)}`,
                task: async (ctx) => {
                    try {
                        ctx.config.podRefs[nodeAlias] = await self.checkNetworkNodePod(ctx.config.namespace, nodeAlias, maxAttempts, undefined, context);
                    }
                    catch (_) {
                        ctx.config.skipStop = true;
                    }
                },
            });
        }
        // setup the sub-tasks
        return task.newListr(subTasks, {
            concurrent: true,
            rendererOptions: {
                collapseSubtasks: false,
            },
        });
    }
    /** Check if the network node pod is running */
    async checkNetworkNodePod(namespace, nodeAlias, maxAttempts = constants.PODS_RUNNING_MAX_ATTEMPTS, delay = constants.PODS_RUNNING_DELAY, context) {
        nodeAlias = nodeAlias.trim();
        const podName = Templates.renderNetworkPodName(nodeAlias);
        const podRef = PodRef.of(namespace, podName);
        try {
            const k8 = this.k8Factory.getK8(context);
            await k8
                .pods()
                .waitForRunningPhase(namespace, [`solo.hedera.com/node-name=${nodeAlias}`, 'solo.hedera.com/type=network-node'], maxAttempts, delay);
            return podRef;
        }
        catch (e) {
            throw new SoloError(`no pod found for nodeAlias: ${nodeAlias}`, e);
        }
    }
    identifyExistingNodes() {
        const self = this;
        return new Task('Identify existing network nodes', async (ctx, task) => {
            const config = ctx.config;
            config.existingNodeAliases = [];
            const clusterRefs = this.parent.getClusterRefs();
            config.serviceMap = await self.accountManager.getNodeServiceMap(config.namespace, clusterRefs, config.deployment);
            for (const networkNodeServices of config.serviceMap.values()) {
                config.existingNodeAliases.push(networkNodeServices.nodeAlias);
            }
            config.allNodeAliases = [...config.existingNodeAliases];
            return self.taskCheckNetworkNodePods(ctx, task, config.existingNodeAliases);
        });
    }
    uploadStateFiles(skip) {
        const self = this;
        return new Task('Upload state files network nodes', async (ctx, task) => {
            const config = ctx.config;
            const zipFile = config.stateFile;
            self.logger.debug(`zip file: ${zipFile}`);
            for (const nodeAlias of ctx.config.nodeAliases) {
                const context = helpers.extractContextFromConsensusNodes(nodeAlias, config.consensusNodes);
                const k8 = this.k8Factory.getK8(context);
                const podRef = ctx.config.podRefs[nodeAlias];
                const containerRef = ContainerRef.of(podRef, constants.ROOT_CONTAINER);
                self.logger.debug(`Uploading state files to pod ${podRef.name}`);
                await k8.containers().readByRef(containerRef).copyTo(zipFile, `${constants.HEDERA_HAPI_PATH}/data`);
                self.logger.info(`Deleting the previous state files in pod ${podRef.name} directory ${constants.HEDERA_HAPI_PATH}/data/saved`);
                await k8
                    .containers()
                    .readByRef(containerRef)
                    .execContainer(['rm', '-rf', `${constants.HEDERA_HAPI_PATH}/data/saved/*`]);
                await k8
                    .containers()
                    .readByRef(containerRef)
                    .execContainer([
                    'tar',
                    '-xvf',
                    `${constants.HEDERA_HAPI_PATH}/data/${path.basename(zipFile)}`,
                    '-C',
                    `${constants.HEDERA_HAPI_PATH}/data/saved`,
                ]);
            }
        }, skip);
    }
    identifyNetworkPods(maxAttempts) {
        const self = this;
        return new Task('Identify network pods', (ctx, task) => {
            return self.taskCheckNetworkNodePods(ctx, task, ctx.config.nodeAliases, maxAttempts);
        });
    }
    fetchPlatformSoftware(aliasesField) {
        const self = this;
        return new Task('Fetch platform software into network nodes', (ctx, task) => {
            const { podRefs, releaseTag, localBuildPath } = ctx.config;
            return localBuildPath !== ''
                ? self._uploadPlatformSoftware(ctx.config[aliasesField], podRefs, task, localBuildPath, ctx.config.consensusNodes)
                : self._fetchPlatformSoftware(ctx.config[aliasesField], podRefs, releaseTag, task, this.platformInstaller, ctx.config.consensusNodes);
        });
    }
    populateServiceMap() {
        return new Task('Populate serviceMap', async (ctx, task) => {
            ctx.config.serviceMap = await this.accountManager.getNodeServiceMap(ctx.config.namespace, this.parent.getClusterRefs(), ctx.config.deployment);
            ctx.config.podRefs[ctx.config.nodeAlias] = PodRef.of(ctx.config.namespace, ctx.config.serviceMap.get(ctx.config.nodeAlias).nodePodName);
        });
    }
    setupNetworkNodes(nodeAliasesProperty, isGenesis) {
        return new Task('Setup network nodes', async (ctx, task) => {
            ctx.config.nodeAliases = helpers.parseNodeAliases(ctx.config.nodeAliasesUnparsed);
            if (isGenesis) {
                await this.generateGenesisNetworkJson(ctx.config.namespace, ctx.config.consensusNodes, ctx.config.keysDir, ctx.config.stagingDir);
            }
            await this.generateNodeOverridesJson(ctx.config.namespace, ctx.config.nodeAliases, ctx.config.stagingDir);
            const consensusNodes = ctx.config.consensusNodes;
            const subTasks = [];
            for (const nodeAlias of ctx.config[nodeAliasesProperty]) {
                const podRef = ctx.config.podRefs[nodeAlias];
                let context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
                // temporary, bridge for node add
                if (!context) {
                    context = this.k8Factory.default().contexts().readCurrent();
                }
                subTasks.push({
                    title: `Node: ${chalk.yellow(nodeAlias)}`,
                    task: () => this.platformInstaller.taskSetup(podRef, ctx.config.stagingDir, isGenesis, context),
                });
            }
            // set up the sub-tasks
            return task.newListr(subTasks, {
                concurrent: true,
                rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION,
            });
        });
    }
    async generateNodeOverridesJson(namespace, nodeAliases, stagingDir) {
        const deploymentName = this.configManager.getFlag(flags.deployment);
        const networkNodeServiceMap = await this.accountManager.getNodeServiceMap(namespace, this.parent.getClusterRefs(), deploymentName);
        const nodeOverridesModel = new NodeOverridesModel(nodeAliases, networkNodeServiceMap);
        const nodeOverridesJson = path.join(stagingDir, constants.NODE_OVERRIDE_FILE);
        fs.writeFileSync(nodeOverridesJson, nodeOverridesModel.toYAML());
    }
    /**
     * Generate genesis network json file
     * @private
     * @param namespace - namespace
     * @param consensusNodes - consensus nodes
     * @param keysDir - keys directory
     * @param stagingDir - staging directory
     */
    async generateGenesisNetworkJson(namespace, consensusNodes, keysDir, stagingDir) {
        const deploymentName = this.configManager.getFlag(flags.deployment);
        const networkNodeServiceMap = await this.accountManager.getNodeServiceMap(namespace, this.parent.getClusterRefs(), deploymentName);
        const adminPublicKeys = splitFlagInput(this.configManager.getFlag(flags.adminPublicKeys));
        const genesisNetworkData = await GenesisNetworkDataConstructor.initialize(consensusNodes, this.keyManager, this.accountManager, keysDir, networkNodeServiceMap, adminPublicKeys);
        const genesisNetworkJson = path.join(stagingDir, 'genesis-network.json');
        fs.writeFileSync(genesisNetworkJson, genesisNetworkData.toJSON());
    }
    prepareStagingDirectory(nodeAliasesProperty) {
        return new Task('Prepare staging directory', (ctx, task) => {
            const config = ctx.config;
            const nodeAliases = config[nodeAliasesProperty];
            const subTasks = [
                {
                    title: 'Copy Gossip keys to staging',
                    task: async () => {
                        this.keyManager.copyGossipKeysToStaging(config.keysDir, config.stagingKeysDir, nodeAliases);
                    },
                },
                {
                    title: 'Copy gRPC TLS keys to staging',
                    task: async () => {
                        for (const nodeAlias of nodeAliases) {
                            const tlsKeyFiles = this.keyManager.prepareTLSKeyFilePaths(nodeAlias, config.keysDir);
                            this.keyManager.copyNodeKeysToStaging(tlsKeyFiles, config.stagingKeysDir);
                        }
                    },
                },
            ];
            return task.newListr(subTasks, {
                concurrent: false,
                rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION,
            });
        });
    }
    startNodes(nodeAliasesProperty) {
        return new Task('Starting nodes', (ctx, task) => {
            const config = ctx.config;
            const nodeAliases = config[nodeAliasesProperty];
            const subTasks = [];
            for (const nodeAlias of nodeAliases) {
                const podRef = config.podRefs[nodeAlias];
                const containerRef = ContainerRef.of(podRef, constants.ROOT_CONTAINER);
                subTasks.push({
                    title: `Start node: ${chalk.yellow(nodeAlias)}`,
                    task: async () => {
                        const context = helpers.extractContextFromConsensusNodes(nodeAlias, config.consensusNodes);
                        const k8 = this.k8Factory.getK8(context);
                        await k8.containers().readByRef(containerRef).execContainer(['systemctl', 'restart', 'network-node']);
                    },
                });
            }
            // set up the sub-tasks
            return task.newListr(subTasks, {
                concurrent: true,
                rendererOptions: {
                    collapseSubtasks: false,
                    timer: constants.LISTR_DEFAULT_RENDERER_TIMER_OPTION,
                },
            });
        });
    }
    enablePortForwarding() {
        return new Task('Enable port forwarding for JVM debugger', async (ctx, task) => {
            const podRef = PodRef.of(ctx.config.namespace, PodName.of(`network-${ctx.config.debugNodeAlias}-0`));
            this.logger.debug(`Enable port forwarding for JVM debugger on pod ${podRef.name}`);
            await this.k8Factory
                .default()
                .pods()
                .readByRef(podRef)
                .portForward(constants.JVM_DEBUG_PORT, constants.JVM_DEBUG_PORT);
        }, (ctx) => !ctx.config.debugNodeAlias);
    }
    checkAllNodesAreActive(nodeAliasesProperty) {
        return new Task('Check all nodes are ACTIVE', (ctx, task) => {
            return this._checkNodeActivenessTask(ctx, task, ctx.config[nodeAliasesProperty]);
        });
    }
    checkAllNodesAreFrozen(nodeAliasesProperty) {
        return new Task('Check all nodes are FROZEN', (ctx, task) => {
            return this._checkNodeActivenessTask(ctx, task, ctx.config[nodeAliasesProperty], NodeStatusCodes.FREEZE_COMPLETE);
        });
    }
    checkNodeProxiesAreActive() {
        return new Task('Check node proxies are ACTIVE', (ctx, task) => {
            // this is more reliable than checking the nodes logs for ACTIVE, as the
            // logs will have a lot of white noise from being behind
            return this._checkNodesProxiesTask(ctx, task, ctx.config.nodeAliases);
        }, async (ctx) => ctx.config.app !== '' && ctx.config.app !== constants.HEDERA_APP_NAME);
    }
    checkAllNodeProxiesAreActive() {
        return new Task('Check all node proxies are ACTIVE', (ctx, task) => {
            // this is more reliable than checking the nodes logs for ACTIVE, as the
            // logs will have a lot of white noise from being behind
            return this._checkNodesProxiesTask(ctx, task, ctx.config.allNodeAliases);
        });
    }
    // Update account manager and transfer hbar for staking purpose
    triggerStakeWeightCalculate(transactionType) {
        const self = this;
        return new Task('Trigger stake weight calculate', async (ctx, task) => {
            const config = ctx.config;
            self.logger.info('sleep 60 seconds for the handler to be able to trigger the network node stake weight recalculate');
            await sleep(Duration.ofSeconds(60));
            const accountMap = getNodeAccountMap(config.allNodeAliases);
            let skipNodeAlias;
            switch (transactionType) {
                case NodeSubcommandType.ADD:
                    break;
                case NodeSubcommandType.UPDATE:
                    if (config.newAccountNumber) {
                        // update map with current account ids
                        accountMap.set(config.nodeAlias, config.newAccountNumber);
                        skipNodeAlias = config.nodeAlias;
                    }
                    break;
                case NodeSubcommandType.DELETE:
                    if (config.nodeAlias) {
                        accountMap.delete(config.nodeAlias);
                        skipNodeAlias = config.nodeAlias;
                    }
            }
            config.nodeClient = await self.accountManager.refreshNodeClient(config.namespace, skipNodeAlias, this.parent.getClusterRefs(), this.configManager.getFlag(flags.deployment));
            // send some write transactions to invoke the handler that will trigger the stake weight recalculate
            for (const nodeAlias of accountMap.keys()) {
                const accountId = accountMap.get(nodeAlias);
                config.nodeClient.setOperator(TREASURY_ACCOUNT_ID, config.treasuryKey);
                await self.accountManager.transferAmount(constants.TREASURY_ACCOUNT_ID, accountId, 1);
            }
        });
    }
    addNodeStakes() {
        const self = this;
        // @ts-ignore
        return new Task('Add node stakes', (ctx, task) => {
            if (ctx.config.app === '' || ctx.config.app === constants.HEDERA_APP_NAME) {
                const subTasks = [];
                const accountMap = getNodeAccountMap(ctx.config.nodeAliases);
                const stakeAmountParsed = ctx.config.stakeAmount ? splitFlagInput(ctx.config.stakeAmount) : [];
                let nodeIndex = 0;
                for (const nodeAlias of ctx.config.nodeAliases) {
                    const accountId = accountMap.get(nodeAlias);
                    const context = helpers.extractContextFromConsensusNodes(nodeAlias, ctx.config.consensusNodes);
                    const stakeAmount = stakeAmountParsed.length > 0 ? stakeAmountParsed[nodeIndex] : HEDERA_NODE_DEFAULT_STAKE_AMOUNT;
                    subTasks.push({
                        title: `Adding stake for node: ${chalk.yellow(nodeAlias)}`,
                        task: async () => await self._addStake(ctx.config.namespace, accountId, nodeAlias, +stakeAmount, context),
                    });
                    nodeIndex++;
                }
                // set up the sub-tasks
                return task.newListr(subTasks, {
                    concurrent: false,
                    rendererOptions: {
                        collapseSubtasks: false,
                    },
                });
            }
        });
    }
    stakeNewNode() {
        const self = this;
        return new Task('Stake new node', async (ctx, task) => {
            const context = helpers.extractContextFromConsensusNodes(ctx.config.nodeAlias, ctx.config.consensusNodes);
            await self.accountManager.refreshNodeClient(ctx.config.namespace, ctx.config.nodeAlias, this.parent.getClusterRefs(), this.configManager.getFlag(flags.deployment), context, this.configManager.getFlag(flags.forcePortForward));
            await this._addStake(ctx.config.namespace, ctx.newNode