@hashgraph/solo
Version:
An opinionated CLI tool to deploy and manage private Hedera Networks.
854 lines • 185 kB
JavaScript
// SPDX-License-Identifier: Apache-2.0
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
import { Zippy } from '../../core/zippy.js';
import * as constants from '../../core/constants.js';
import { CHECK_WRAPS_DIRECTORY_BACKOFF_MS, CHECK_WRAPS_DIRECTORY_MAX_ATTEMPTS, DEFAULT_NETWORK_NODE_NAME, HEDERA_HAPI_PATH, HEDERA_NODE_DEFAULT_STAKE_AMOUNT, } from '../../core/constants.js';
const localBuildPathFilter = (path) => {
return !(path.includes('data/keys') || path.includes('data/config'));
};
import { Templates } from '../../core/templates.js';
import { AccountBalanceQuery, AccountId, AccountUpdateTransaction, FileAppendTransaction, FileId, FileUpdateTransaction, FreezeTransaction, FreezeType, Long, NodeCreateTransaction, NodeDeleteTransaction, NodeUpdateTransaction, PrivateKey, ServiceEndpoint, Status, Timestamp, } from '@hiero-ledger/sdk';
import { SoloError } from '../../core/errors/solo-error.js';
import { MissingArgumentError } from '../../core/errors/missing-argument-error.js';
import fs from 'node:fs';
import crypto from 'node:crypto';
import { execSync } from 'node:child_process';
import find from 'find-process';
import * as helpers from '../../core/helpers.js';
import { addRootImageValues, createAndCopyBlockNodeJsonFileForConsensusNode, entityId, extractContextFromConsensusNodes, prepareEndpoints, prepareValuesFilesMap, prepareValuesFilesMapMultipleCluster, renameAndCopyFile, showVersionBanner, sleep, splitFlagInput, } from '../../core/helpers.js';
import chalk from 'chalk';
import { Flags as flags } from '../flags.js';
import * as versions from '../../../version.js';
import { HEDERA_PLATFORM_VERSION, MINIMUM_HIERO_PLATFORM_VERSION_FOR_GRPC_WEB_ENDPOINTS, needsConfigTxtForConsensusVersion, } from '../../../version.js';
import { ListrInquirerPromptAdapter } from '@listr2/prompt-adapter-inquirer';
import { confirm as confirmPrompt } from '@inquirer/prompts';
import { PodName } from '../../integration/kube/resources/pod/pod-name.js';
import { NodeStatusCodes, NodeStatusEnums, NodeSubcommandType } from '../../core/enumerations.js';
import { ListrLock } from '../../core/lock/listr-lock.js';
import { Duration } from '../../core/time/duration.js';
import { GenesisNetworkDataConstructor } from '../../core/genesis-network-models/genesis-network-data-constructor.js';
import { NodeOverridesModel } from '../../core/node-overrides-model.js';
import { NamespaceName } from '../../types/namespace/namespace-name.js';
import { PodReference } from '../../integration/kube/resources/pod/pod-reference.js';
import { ContainerReference } from '../../integration/kube/resources/container/container-reference.js';
import { container, inject, injectable } from 'tsyringe-neo';
import { patchInject } from '../../core/dependency-injection/container-helper.js';
import { ConsensusNode } from '../../core/model/consensus-node.js';
import { Base64 } from 'js-base64';
import { SecretType } from '../../integration/kube/resources/secret/secret-type.js';
import { InjectTokens } from '../../core/dependency-injection/inject-tokens.js';
import { PathEx } from '../../business/utils/path-ex.js';
import { helmValuesHelper } from '../../core/helm-values-helper.js';
import { ComponentTypes } from '../../core/config/remote/enumerations/component-types.js';
import { DeploymentPhase } from '../../data/schema/model/remote/deployment-phase.js';
import { SemanticVersion } from '../../business/utils/semantic-version.js';
import net from 'node:net';
import { Address } from '../../business/address/address.js';
import { K8Helper } from '../../business/utils/k8-helper.js';
import { PackageDownloader } from '../../core/package-downloader.js';
import { DefaultHelmClient } from '../../integration/helm/impl/default-helm-client.js';
import { ConsensusNodePathTemplates } from '../../core/consensus-node-path-templates.js';
import { SoloConfig } from '../../business/runtime-state/config/solo/solo-config.js';
import { DiagnosticsAnalyzer } from '../util/diagnostics-analyzer.js';
import { NodesStartedEvent } from '../../core/events/event-types/nodes-started-event.js';
const { gray, cyan, red, green, yellow } = chalk;
let NodeCommandTasks = class NodeCommandTasks {
logger;
accountManager;
configManager;
k8Factory;
platformInstaller;
keyManager;
profileManager;
chartManager;
certificateManager;
remoteConfig;
localConfig;
componentFactory;
oneShotState;
zippy;
downloader;
gitClient;
eventBus;
soloConfig;
constructor(logger, accountManager, configManager, k8Factory, platformInstaller, keyManager, profileManager, chartManager, certificateManager, remoteConfig, localConfig, componentFactory, oneShotState, zippy, downloader, gitClient, configProvider, eventBus) {
this.logger = logger;
this.accountManager = accountManager;
this.configManager = configManager;
this.k8Factory = k8Factory;
this.platformInstaller = platformInstaller;
this.keyManager = keyManager;
this.profileManager = profileManager;
this.chartManager = chartManager;
this.certificateManager = certificateManager;
this.remoteConfig = remoteConfig;
this.localConfig = localConfig;
this.componentFactory = componentFactory;
this.oneShotState = oneShotState;
this.zippy = zippy;
this.downloader = downloader;
this.gitClient = gitClient;
this.eventBus = eventBus;
this.logger = patchInject(logger, InjectTokens.SoloLogger, this.constructor.name);
this.accountManager = patchInject(accountManager, InjectTokens.AccountManager, this.constructor.name);
this.configManager = patchInject(configManager, InjectTokens.ConfigManager, this.constructor.name);
this.k8Factory = patchInject(k8Factory, InjectTokens.K8Factory, this.constructor.name);
this.platformInstaller = patchInject(platformInstaller, InjectTokens.PlatformInstaller, this.constructor.name);
this.keyManager = patchInject(keyManager, InjectTokens.KeyManager, this.constructor.name);
this.profileManager = patchInject(profileManager, InjectTokens.ProfileManager, this.constructor.name);
this.chartManager = patchInject(chartManager, InjectTokens.ChartManager, this.constructor.name);
this.certificateManager = patchInject(certificateManager, InjectTokens.CertificateManager, this.constructor.name);
this.localConfig = patchInject(localConfig, InjectTokens.LocalConfigRuntimeState, this.constructor.name);
this.remoteConfig = patchInject(remoteConfig, InjectTokens.RemoteConfigRuntimeState, this.constructor.name);
this.oneShotState = patchInject(oneShotState, InjectTokens.OneShotState, this.constructor.name);
this.zippy = patchInject(zippy, InjectTokens.Zippy, this.constructor.name);
this.downloader = patchInject(downloader, InjectTokens.PackageDownloader, this.constructor.name);
this.gitClient = patchInject(gitClient, InjectTokens.GitClient, this.constructor.name);
this.eventBus = patchInject(eventBus, InjectTokens.SoloEventBus, this.constructor.name);
configProvider = patchInject(configProvider, InjectTokens.ConfigProvider, this.constructor.name);
this.soloConfig = SoloConfig.getConfig(configProvider);
}
getFileUpgradeId(deploymentName) {
const realm = this.localConfig.configuration.realmForDeployment(deploymentName);
const shard = this.localConfig.configuration.shardForDeployment(deploymentName);
return FileId.fromString(entityId(shard, realm, constants.UPGRADE_FILE_ID_NUM));
}
async _prepareUpgradeZip(stagingDirectory, upgradeVersion) {
// we build a mock upgrade.zip file as we really don't need to upgrade the network
// also the platform zip file is ~80Mb in size requiring a lot of transactions since the max
// transaction size is 6Kb and in practice we need to send the file as 4Kb chunks.
// Note however that in DAB phase-2, we won't need to trigger this fake upgrade process
const zipper = new Zippy(this.logger);
const upgradeConfigDirectory = PathEx.join(stagingDirectory, 'mock-upgrade', 'data', 'config');
if (!fs.existsSync(upgradeConfigDirectory)) {
fs.mkdirSync(upgradeConfigDirectory, { recursive: true });
}
// bump field hedera.config.version or use the version passed in
const fileBytes = fs.readFileSync(PathEx.joinWithRealPath(stagingDirectory, 'templates', constants.APPLICATION_PROPERTIES));
const lines = fileBytes.toString().split('\n');
const newLines = [];
for (let line of lines) {
line = line.trim();
const parts = line.split('=');
if (parts.length === 2) {
if (parts[0] === 'hedera.config.version') {
const version = upgradeVersion ?? String(Number.parseInt(parts[1]) + 1);
line = `hedera.config.version=${version}`;
}
newLines.push(line);
}
}
fs.writeFileSync(PathEx.join(upgradeConfigDirectory, constants.APPLICATION_PROPERTIES), newLines.join('\n'));
return await zipper.zip(PathEx.join(stagingDirectory, 'mock-upgrade'), PathEx.join(stagingDirectory, 'mock-upgrade.zip'));
}
async _uploadUpgradeZip(upgradeZipFile, nodeClient, deploymentName) {
// get byte value of the zip file
const zipBytes = fs.readFileSync(upgradeZipFile);
const zipHash = crypto.createHash('sha384').update(zipBytes).digest('hex');
this.logger.debug(`loaded upgrade zip file [ zipHash = ${zipHash} zipBytes.length = ${zipBytes.length}, zipPath = ${upgradeZipFile}]`);
// create a file upload transaction to upload file to the network
try {
let start = 0;
while (start < zipBytes.length) {
const zipBytesChunk = new Uint8Array(zipBytes.subarray(start, start + constants.UPGRADE_FILE_CHUNK_SIZE));
let fileTransaction = undefined;
fileTransaction =
start === 0
? new FileUpdateTransaction().setFileId(this.getFileUpgradeId(deploymentName)).setContents(zipBytesChunk)
: new FileAppendTransaction().setFileId(this.getFileUpgradeId(deploymentName)).setContents(zipBytesChunk);
const resp = await fileTransaction.execute(nodeClient);
const receipt = await resp.getReceipt(nodeClient);
this.logger.debug(`updated file ${this.getFileUpgradeId(deploymentName)} [chunkSize= ${zipBytesChunk.length}, txReceipt = ${receipt.toString()}]`);
start += constants.UPGRADE_FILE_CHUNK_SIZE;
this.logger.debug(`uploaded ${start} bytes of ${zipBytes.length} bytes`);
}
return zipHash;
}
catch (error) {
throw new SoloError(`failed to upload build.zip file: ${error.message}`, error);
}
}
async copyLocalBuildPathToNode(k8, podReference, configManager, localDataLibraryBuildPath) {
const container = k8
.containers()
.readByRef(ContainerReference.of(podReference, constants.ROOT_CONTAINER));
// Remove existing jars before copying to prevent mixed-version classpath (issue #3848)
await container.execContainer([
'bash',
'-c',
`rm -rf ${constants.HEDERA_HAPI_PATH}/${constants.HEDERA_DATA_LIB_DIR}/*.jar ${constants.HEDERA_HAPI_PATH}/${constants.HEDERA_DATA_APPS_DIR}/*.jar`,
]);
await container.copyTo(localDataLibraryBuildPath, `${constants.HEDERA_HAPI_PATH}`, localBuildPathFilter);
if (configManager.getFlag(flags.appConfig)) {
const testJsonFiles = configManager.getFlag(flags.appConfig).split(',');
for (const jsonFile of testJsonFiles) {
if (fs.existsSync(jsonFile)) {
await container.copyTo(jsonFile, `${constants.HEDERA_HAPI_PATH}`);
}
}
}
}
async validateNodePvcsForLocalBuildPath(namespace, contexts) {
await Promise.all(contexts.map(async (context) => {
const pvcs = await this.k8Factory
.getK8(context)
.pvcs()
.list(namespace, ['solo.hedera.com/type=node-pvc']);
if (pvcs.length === 0) {
throw new SoloError('Custom JARs provided via --local-build-path require node PVCs to persist across pod restarts. ' +
'Redeploy the consensus network with --pvcs true and run consensus node setup again.');
}
}));
}
_uploadPlatformSoftware(nodeAliases, podReferences, task, localBuildPath, consensusNodes, releaseTag) {
const subTasks = [];
this.logger.debug('no need to fetch, use local build jar files');
const buildPathMap = new Map();
let defaultDataLibraryBuildPath;
const parameterPairs = localBuildPath.split(',');
for (const parameterPair of parameterPairs) {
if (parameterPair.includes('=')) {
const [nodeAlias, localDataLibraryBuildPath] = parameterPair.split('=');
buildPathMap.set(nodeAlias, localDataLibraryBuildPath);
}
else {
defaultDataLibraryBuildPath = parameterPair;
}
}
let localDataLibraryBuildPath;
for (const nodeAlias of nodeAliases) {
const podReference = podReferences[nodeAlias];
const context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
localDataLibraryBuildPath = buildPathMap.has(nodeAlias)
? buildPathMap.get(nodeAlias)
: defaultDataLibraryBuildPath;
if (!fs.existsSync(localDataLibraryBuildPath)) {
throw new SoloError(`local build path does not exist: ${localDataLibraryBuildPath}`);
}
// The local build path points to the `data` directory itself (containing apps/ and lib/).
// Validate that it contains jar files in each subdirectory to catch incorrect paths early.
const applicationsSubDirectory = PathEx.join(localDataLibraryBuildPath, 'apps');
const librarySubDirectory = PathEx.join(localDataLibraryBuildPath, 'lib');
if (!fs.existsSync(applicationsSubDirectory) || !fs.existsSync(librarySubDirectory)) {
throw new SoloError(`local build path '${localDataLibraryBuildPath}' must contain 'apps' and 'lib' subdirectories`);
}
const applicationsJarFiles = fs
.readdirSync(applicationsSubDirectory)
.filter((file) => file.endsWith('.jar'));
if (applicationsJarFiles.length === 0) {
throw new SoloError(`No jar files found in '${applicationsSubDirectory}'; please check your local build path`);
}
const libraryJarFiles = fs
.readdirSync(librarySubDirectory)
.filter((file) => file.endsWith('.jar'));
if (libraryJarFiles.length === 0) {
throw new SoloError(`No jar files found in '${librarySubDirectory}'; please check your local build path`);
}
const k8 = this.k8Factory.getK8(context);
subTasks.push({
title: `Copy local build to Node: ${chalk.yellow(nodeAlias)} from ${localDataLibraryBuildPath}`,
task: async () => {
try {
const retrievedReleaseTag = await this.gitClient.describeTag(localDataLibraryBuildPath);
const expectedReleaseTag = releaseTag || HEDERA_PLATFORM_VERSION;
if (retrievedReleaseTag !== expectedReleaseTag) {
this.logger.showUser(chalk.cyan(`Checkout version ${retrievedReleaseTag} does not match the release version ${expectedReleaseTag}`));
}
}
catch {
// if we can't find the release tag in the local build path directory, we will skip the check and continue
this.logger.warn('Could not find release tag in local build path directory');
this.logger.showUser(chalk.yellowBright('The release tag could not be verified, please ensure that the release tag passed on the command line ' +
'matches the release tag of the code in the local build path directory'));
}
// retry copying the build to the node to handle edge cases during performance testing
let storedError = null;
let index = 0;
for (; index < constants.LOCAL_BUILD_COPY_RETRY; index++) {
storedError = null;
try {
// filter the data/config and data/keys to avoid failures due to config and secret mounts
await this.copyLocalBuildPathToNode(k8, podReference, this.configManager, localDataLibraryBuildPath);
}
catch (error) {
storedError = error;
}
}
if (storedError) {
throw new SoloError(`Error in copying local build to node: ${storedError.message}`, storedError);
}
},
});
}
// set up the sub-tasks
return task.newListr(subTasks, {
concurrent: constants.NODE_COPY_CONCURRENT,
rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION,
fallbackRendererOptions: {
timer: constants.LISTR_DEFAULT_RENDERER_TIMER_OPTION,
},
});
}
async _fetchPlatformSoftware(nodeAliases, podReferences, releaseTag, task, platformInstaller, consensusNodes, stagingDirectory) {
const subTasks = [];
const [zipPath, checksumPath] = await platformInstaller.getPlatformRelease(stagingDirectory, releaseTag);
for (const nodeAlias of nodeAliases) {
const context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
const podReference = podReferences[nodeAlias];
subTasks.push({
title: `Update node: ${chalk.yellow(nodeAlias)} [ platformVersion = ${releaseTag}, context = ${context} ]`,
task: async () => {
await platformInstaller.fetchPlatform(podReference, releaseTag, zipPath, checksumPath, context);
},
});
}
// set up the sub-tasks
return task.newListr(subTasks, {
concurrent: true, // since we download in the container directly, we want this to be in parallel across all nodes
rendererOptions: {
collapseSubtasks: false,
},
});
}
_checkNodeActivenessTask(context_, task, nodeAliases, status = NodeStatusCodes.ACTIVE) {
const { config: { namespace }, } = context_;
const enableDebugger = context_.config.debugNodeAlias && status !== NodeStatusCodes.FREEZE_COMPLETE;
const debugNodeAlias = context_.config.debugNodeAlias;
const subTasks = nodeAliases.map((nodeAlias) => {
const isDebugNode = debugNodeAlias === nodeAlias && status !== NodeStatusCodes.FREEZE_COMPLETE;
const reminder = isDebugNode ? 'Please attach JVM debugger now.' : '';
const title = `Check network pod: ${chalk.yellow(nodeAlias)} ${chalk.red(reminder)}`;
const context = helpers.extractContextFromConsensusNodes(nodeAlias, this.remoteConfig.getConsensusNodes());
return {
title,
task: async (context_, task) => {
if (enableDebugger && isDebugNode) {
await task.prompt(ListrInquirerPromptAdapter).run(confirmPrompt, {
message: `JVM debugger setup for ${nodeAlias}. Continue when debugging is complete?`,
default: false,
});
}
context_.config.podRefs[nodeAlias] = await this.checkNetworkNodeActiveness(namespace, nodeAlias, task, title, status, undefined, undefined, undefined, context);
},
};
});
return task.newListr(subTasks, {
concurrent: !enableDebugger, // Run sequentially when debugging to avoid multiple prompts
rendererOptions: {
collapseSubtasks: false,
},
});
}
async checkNetworkNodeActiveness(namespace, nodeAlias, task, title, status = NodeStatusCodes.ACTIVE, maxAttempts = constants.NETWORK_NODE_ACTIVE_MAX_ATTEMPTS, delay = constants.NETWORK_NODE_ACTIVE_DELAY, timeout = constants.NETWORK_NODE_ACTIVE_TIMEOUT, context) {
const podName = Templates.renderNetworkPodName(nodeAlias);
const podReference = PodReference.of(namespace, podName);
task.title = `${title} - status ${chalk.yellow('STARTING')}, attempt ${chalk.blueBright(`0/${maxAttempts}`)}`;
const consensusNodes = this.remoteConfig.getConsensusNodes();
if (typeof context !== 'string' || context.trim().length === 0) {
context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
}
let attempt = 0;
let success = false;
while (attempt < maxAttempts) {
const controller = new AbortController();
const timeoutId = setTimeout(() => {
task.title = `${title} - status ${chalk.yellow('TIMEOUT')}, attempt ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
controller.abort();
}, timeout);
try {
const response = await container
.resolve(InjectTokens.NetworkNodes)
.getNetworkNodePodStatus(podReference, context);
if (!response) {
task.title = `${title} - status ${chalk.yellow('UNKNOWN')}, attempt ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
clearTimeout(timeoutId);
throw new SoloError('empty response'); // Guard
}
const statusLine = response
.split('\n')
.find((line) => line.startsWith('platform_PlatformStatus'));
if (!statusLine) {
task.title = `${title} - status ${chalk.yellow('STARTING')}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
clearTimeout(timeoutId);
throw new SoloError('missing status line'); // Guard
}
const statusNumber = Number.parseInt(statusLine.split(' ').pop());
if (statusNumber === status) {
task.title = `${title} - status ${chalk.green(NodeStatusEnums[status])}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
success = true;
clearTimeout(timeoutId);
break;
}
else if (statusNumber === NodeStatusCodes.CATASTROPHIC_FAILURE) {
task.title = `${title} - status ${chalk.red('CATASTROPHIC_FAILURE')}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
break;
}
else if (statusNumber) {
task.title = `${title} - status ${chalk.yellow(NodeStatusEnums[statusNumber])}, attempt: ${chalk.blueBright(`${attempt}/${maxAttempts}`)}`;
}
clearTimeout(timeoutId);
}
catch (error) {
this.logger.debug(`${title} : Error in checking node activeness: attempt: ${attempt}/${maxAttempts}: ${JSON.stringify(error)}`);
}
attempt++;
clearTimeout(timeoutId);
await sleep(Duration.ofMillis(delay));
}
if (!success) {
throw new SoloError(`node '${nodeAlias}' is not ${NodeStatusEnums[status]}` +
`[ attempt = ${chalk.blueBright(`${attempt}/${maxAttempts}`)} ]`);
}
if (constants.NETWORK_NODE_ACTIVE_EXTRA_DELAY_MS > 0) {
await sleep(Duration.ofMillis(constants.NETWORK_NODE_ACTIVE_EXTRA_DELAY_MS)); // delaying prevents - gRPC service error
}
return podReference;
}
/** Return task for check if node proxies are ready */
_checkNodesProxiesTask(task, nodeAliases) {
const subTasks = [];
for (const nodeAlias of nodeAliases) {
subTasks.push({
title: `Check proxy for node: ${chalk.yellow(nodeAlias)}`,
task: async (context_) => {
const context = helpers.extractContextFromConsensusNodes(nodeAlias, context_.config.consensusNodes);
const k8 = this.k8Factory.getK8(context);
await k8
.pods()
.waitForReadyStatus(context_.config.namespace, [`app=haproxy-${nodeAlias}`, 'solo.hedera.com/type=haproxy'], constants.NETWORK_PROXY_MAX_ATTEMPTS, constants.NETWORK_PROXY_DELAY);
},
});
}
// set up the sub-tasks
return task.newListr(subTasks, {
concurrent: true,
rendererOptions: {
collapseSubtasks: false,
},
});
}
/**
* When generating multiple all aliases are read from config.nodeAliases,
* When generating a single key the alias in config.nodeAlias is used
*/
_generateGossipKeys(generateMultiple) {
return {
title: 'Generate gossip keys',
task: ({ config }, task) => {
const nodeAliases = generateMultiple
? config.nodeAliases
: [config.nodeAlias];
const subTasks = this.keyManager.taskGenerateGossipKeys(nodeAliases, config.keysDir, config.curDate);
// set up the sub-tasks
return task.newListr(subTasks, constants.LISTR_DEFAULT_OPTIONS.DEFAULT);
},
skip: (context_) => !context_.config.generateGossipKeys,
};
}
/**
* When generating multiple all aliases are read from config.nodeAliases,
* When generating a single key the alias in config.nodeAlias is used
*/
_generateGrpcTlsKeys(generateMultiple) {
return {
title: 'Generate gRPC TLS Keys',
task: (context_, task) => {
const config = context_.config;
const nodeAliases = generateMultiple
? config.nodeAliases
: [config.nodeAlias];
const subTasks = this.keyManager.taskGenerateTLSKeys(nodeAliases, config.keysDir, config.curDate);
// set up the sub-tasks
return task.newListr(subTasks, constants.LISTR_DEFAULT_OPTIONS.WITH_CONCURRENCY);
},
skip: (context_) => !context_.config.generateTlsKeys,
};
}
copyGrpcTlsCertificates() {
return {
title: 'Copy gRPC TLS Certificates',
task: ({ config }, task) => this.certificateManager.buildCopyTlsCertificatesTasks(task, config.grpcTlsCertificatePath, config.grpcWebTlsCertificatePath, config.grpcTlsKeyPath, config.grpcWebTlsKeyPath),
skip: (context_) => !context_.config.grpcTlsCertificatePath && !context_.config.grpcWebTlsCertificatePath,
};
}
async _addStake(namespace, accountId, nodeAlias, stakeAmount = HEDERA_NODE_DEFAULT_STAKE_AMOUNT) {
try {
const deploymentName = this.configManager.getFlag(flags.deployment);
await this.accountManager.loadNodeClient(namespace, this.remoteConfig.getClusterRefs(), deploymentName, this.configManager.getFlag(flags.forcePortForward));
const client = this.accountManager._nodeClient;
const treasuryKey = await this.accountManager.getTreasuryAccountKeys(namespace, deploymentName);
const treasuryPrivateKey = PrivateKey.fromStringED25519(treasuryKey.privateKey);
const treasuryAccountId = this.accountManager.getTreasuryAccountId(deploymentName);
client.setOperator(treasuryAccountId, treasuryPrivateKey);
// check balance
const treasuryBalance = await new AccountBalanceQuery()
.setAccountId(treasuryAccountId)
.execute(client);
this.logger.debug(`Account ${treasuryAccountId} balance: ${treasuryBalance.hbars}`);
// get some initial balance
await this.accountManager.transferAmount(treasuryAccountId, accountId, stakeAmount);
// check balance
const balance = await new AccountBalanceQuery().setAccountId(accountId).execute(client);
this.logger.debug(`Account ${accountId} balance: ${balance.hbars}`);
// Create the transaction
const transaction = new AccountUpdateTransaction()
.setAccountId(accountId)
.setStakedNodeId(Templates.nodeIdFromNodeAlias(nodeAlias))
.freezeWith(client);
// Sign the transaction with the account's private key
const signTransaction = await transaction.sign(treasuryPrivateKey);
const transactionResponse = await signTransaction.execute(client);
const receipt = await transactionResponse.getReceipt(client);
this.logger.debug(`The transaction consensus status is ${receipt.status}`);
}
catch (error) {
throw new SoloError(`Error in adding stake: ${error.message}`, error);
}
}
prepareUpgradeZip() {
return {
title: 'Prepare upgrade zip file for node upgrade process',
task: async (context_) => {
const config = context_.config;
const { upgradeZipFile, deployment } = context_.config;
if (upgradeZipFile) {
context_.upgradeZipFile = upgradeZipFile;
this.logger.debug(`Using upgrade zip file: ${context_.upgradeZipFile}`);
}
else {
// download application.properties from the first node in the deployment
const nodeAlias = config.existingNodeAliases[0];
const nodeFullyQualifiedPodName = Templates.renderNetworkPodName(nodeAlias);
const podReference = PodReference.of(config.namespace, nodeFullyQualifiedPodName);
const containerReference = ContainerReference.of(podReference, constants.ROOT_CONTAINER);
const context = helpers.extractContextFromConsensusNodes(context_.config.nodeAlias, context_.config.consensusNodes);
const templatesDirectory = PathEx.join(config.stagingDir, 'templates');
fs.mkdirSync(templatesDirectory, { recursive: true });
await this.k8Factory
.getK8(context)
.containers()
.readByRef(containerReference)
.copyFrom(`${constants.HEDERA_HAPI_PATH}/data/config/${constants.APPLICATION_PROPERTIES}`, templatesDirectory);
const upgradeVersion = 'upgradeVersion' in config ? config.upgradeVersion : undefined;
context_.upgradeZipFile = await this._prepareUpgradeZip(config.stagingDir, upgradeVersion);
}
context_.upgradeZipHash = await this._uploadUpgradeZip(context_.upgradeZipFile, config.nodeClient, deployment);
},
};
}
loadAdminKey() {
return {
title: 'Load node admin key',
task: async (context_) => {
const config = context_.config;
if (context_.config.nodeAlias) {
try {
const context = helpers.extractContextFromConsensusNodes(context_.config.nodeAlias, context_.config.consensusNodes);
// load nodeAdminKey from k8s if exist
const keyFromK8 = await this.k8Factory
.getK8(context)
.secrets()
.read(config.namespace, Templates.renderNodeAdminKeyName(context_.config.nodeAlias));
const privateKey = Base64.decode(keyFromK8.data.privateKey);
config.adminKey = PrivateKey.fromStringED25519(privateKey);
}
catch (error) {
this.logger.debug(`Error in loading node admin key: ${error.message}, use default key`);
config.adminKey = PrivateKey.fromStringED25519(constants.GENESIS_KEY);
}
}
else {
config.adminKey = PrivateKey.fromStringED25519(constants.GENESIS_KEY);
}
},
};
}
checkExistingNodesStakedAmount() {
return {
title: 'Check existing nodes staked amount',
task: async ({ config }) => {
// Transfer some hbar to the node for staking purpose
const deploymentName = this.configManager.getFlag(flags.deployment);
const accountMap = this.accountManager.getNodeAccountMap(config.existingNodeAliases, deploymentName);
const treasuryAccountId = this.accountManager.getTreasuryAccountId(deploymentName);
for (const nodeAlias of config.existingNodeAliases) {
const accountId = accountMap.get(nodeAlias);
await this.accountManager.transferAmount(treasuryAccountId, accountId, 1);
}
},
};
}
sendPrepareUpgradeTransaction() {
return {
title: 'Send prepare upgrade transaction',
task: async (context_) => {
const { upgradeZipHash } = context_;
const { nodeClient, freezeAdminPrivateKey, deployment } = context_.config;
try {
const freezeAccountId = this.accountManager.getFreezeAccountId(deployment);
const treasuryAccountId = this.accountManager.getTreasuryAccountId(deployment);
// query the balance
const balance = await new AccountBalanceQuery()
.setAccountId(freezeAccountId)
.execute(nodeClient);
this.logger.debug(`Freeze admin account balance: ${balance.hbars}`);
// transfer some tiny amount to the freeze admin account
await this.accountManager.transferAmount(treasuryAccountId, freezeAccountId, 100_000);
// set operator of freeze transaction as freeze admin account
nodeClient.setOperator(freezeAccountId, freezeAdminPrivateKey);
const prepareUpgradeTransaction = await new FreezeTransaction()
.setFreezeType(FreezeType.PrepareUpgrade)
.setFileId(this.getFileUpgradeId(deployment))
.setFileHash(upgradeZipHash)
.freezeWith(nodeClient)
.execute(nodeClient);
const prepareUpgradeReceipt = await prepareUpgradeTransaction.getReceipt(nodeClient);
this.logger.debug(`sent prepare upgrade transaction [id: ${prepareUpgradeTransaction.transactionId.toString()}]`, prepareUpgradeReceipt.status.toString());
if (prepareUpgradeReceipt.status !== Status.Success) {
throw new SoloError(`Prepare upgrade transaction failed: ${prepareUpgradeReceipt.status}`);
}
}
catch (error) {
throw new SoloError(`Error in prepare upgrade: ${error.message}`, error);
}
},
};
}
sendFreezeUpgradeTransaction() {
return {
title: 'Send freeze upgrade transaction',
task: async (context_) => {
const { upgradeZipHash } = context_;
const { freezeAdminPrivateKey, nodeClient, deployment } = context_.config;
try {
const futureDate = new Date();
this.logger.debug(`Current time: ${futureDate}`);
futureDate.setTime(futureDate.getTime() + 5000); // 5 seconds in the future
this.logger.debug(`Freeze time: ${futureDate}`);
const freezeAdminAccountId = this.accountManager.getFreezeAccountId(deployment);
// query the balance
const balance = await new AccountBalanceQuery()
.setAccountId(freezeAdminAccountId)
.execute(nodeClient);
this.logger.debug(`Freeze admin account balance: ${balance.hbars}`);
nodeClient.setOperator(freezeAdminAccountId, freezeAdminPrivateKey);
const freezeUpgradeTx = await new FreezeTransaction()
.setFreezeType(FreezeType.FreezeUpgrade)
.setStartTimestamp(Timestamp.fromDate(futureDate))
.setFileId(this.getFileUpgradeId(deployment))
.setFileHash(upgradeZipHash)
.freezeWith(nodeClient)
.execute(nodeClient);
const freezeUpgradeReceipt = await freezeUpgradeTx.getReceipt(nodeClient);
this.logger.debug(`Upgrade frozen with transaction id: ${freezeUpgradeTx.transactionId.toString()}`, freezeUpgradeReceipt.status.toString());
}
catch (error) {
throw new SoloError(`Error in freeze upgrade: ${error.message}`, error);
}
},
};
}
sendFreezeTransaction() {
return {
title: 'Send freeze only transaction',
task: async (context_) => {
const { freezeAdminPrivateKey, deployment, namespace } = context_.config;
try {
const nodeClient = await this.accountManager.loadNodeClient(namespace, this.remoteConfig.getClusterRefs(), deployment);
const futureDate = new Date();
this.logger.debug(`Current time: ${futureDate}`);
futureDate.setTime(futureDate.getTime() + 5000); // 5 seconds in the future
this.logger.debug(`Freeze time: ${futureDate}`);
const freezeAdminAccountId = this.accountManager.getFreezeAccountId(deployment);
nodeClient.setOperator(freezeAdminAccountId, freezeAdminPrivateKey);
const freezeOnlyTransaction = await new FreezeTransaction()
.setFreezeType(FreezeType.FreezeOnly)
.setStartTimestamp(Timestamp.fromDate(futureDate))
.freezeWith(nodeClient)
.execute(nodeClient);
const freezeOnlyReceipt = await freezeOnlyTransaction.getReceipt(nodeClient);
this.logger.debug(`sent prepare transaction [id: ${freezeOnlyTransaction.transactionId.toString()}]`, freezeOnlyReceipt.status.toString());
}
catch (error) {
throw new SoloError(`Error in sending freeze transaction: ${error.message}`, error);
}
},
};
}
/** Download generated config files and key files from the network node,
* This function should only be called when updating or destroying a node
* */
downloadNodeGeneratedFilesForDynamicAddressBook() {
return {
title: 'Download generated files from an existing node',
task: async ({ config: { nodeAlias, existingNodeAliases, consensusNodes, stagingDir, keysDir, namespace }, }) => {
// don't try to download from the same node we are deleting, it won't work
const targetNodeAlias = nodeAlias === existingNodeAliases[0] && existingNodeAliases.length > 1
? existingNodeAliases[1]
: existingNodeAliases[0];
const nodeFullyQualifiedPodName = Templates.renderNetworkPodName(targetNodeAlias);
const podReference = PodReference.of(namespace, nodeFullyQualifiedPodName);
const containerReference = ContainerReference.of(podReference, constants.ROOT_CONTAINER);
const context = helpers.extractContextFromConsensusNodes(targetNodeAlias, consensusNodes);
const k8Container = this.k8Factory.getK8(context).containers().readByRef(containerReference);
const consensusVersion = this.remoteConfig.configuration?.versions?.consensusNode;
const releaseTag = consensusVersion?.toString() || HEDERA_PLATFORM_VERSION;
const needsConfigTxt = needsConfigTxtForConsensusVersion(releaseTag);
const configSource = `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/config.txt`;
if (needsConfigTxt && (await k8Container.hasFile(configSource))) {
// copy the config.txt file from the node1 upgrade directory if it exists
await k8Container.copyFrom(configSource, stagingDir);
}
// if directory data/upgrade/current/data/keys does not exist, then use data/upgrade/current
let keyDirectory = `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/data/keys`;
if (!(await k8Container.hasDir(keyDirectory))) {
keyDirectory = `${constants.HEDERA_HAPI_PATH}/data/upgrade/current`;
}
const signedKeyFiles = await k8Container
.listDir(keyDirectory)
.then((files) => files.filter((file) => file.name.startsWith(constants.SIGNING_KEY_PREFIX)));
await k8Container.execContainer([
'bash',
'-c',
`mkdir -p ${constants.HEDERA_HAPI_PATH}/data/keys_backup && cp -r ${keyDirectory} ${constants.HEDERA_HAPI_PATH}/data/keys_backup/`,
]);
for (const signedKeyFile of signedKeyFiles) {
await k8Container.copyFrom(`${keyDirectory}/${signedKeyFile.name}`, `${keysDir}`);
}
const applicationPropertiesSourceDirectory = `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/data/config/${constants.APPLICATION_PROPERTIES}`;
await ((await k8Container.hasFile(applicationPropertiesSourceDirectory))
? k8Container.copyFrom(applicationPropertiesSourceDirectory, `${stagingDir}/templates`)
: k8Container.copyFrom(`${constants.HEDERA_HAPI_PATH}/data/upgrade/current/data/config/${constants.APPLICATION_PROPERTIES}`, `${stagingDir}/templates`));
},
};
}
downloadNodeUpgradeFiles() {
return {
title: 'Download upgrade files from an existing node',
task: async (context_) => {
const { consensusNodes, namespace, stagingDir, nodeAliases } = context_.config;
const nodeAlias = nodeAliases[0];
const context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
const container = await new K8Helper(context).getConsensusNodeRootContainer(namespace, nodeAlias);
// found all files under ${constants.HEDERA_HAPI_PATH}/data/upgrade/current/
const upgradeDirectories = [
`${constants.HEDERA_HAPI_PATH}/data/upgrade/current`,
`${constants.HEDERA_HAPI_PATH}/data/upgrade/current/data/apps`,
`${constants.HEDERA_HAPI_PATH}/data/upgrade/current/data/libs`,
];
for (const upgradeDirectory of upgradeDirectories) {
// check if directory upgradeDirectory exist in root container
if (!(await container.hasDir(upgradeDirectory))) {
continue;
}
const files = await container.listDir(upgradeDirectory);
// iterate all files and copy them to the staging directory
for (const file of files) {
if (file.name.endsWith('.mf')) {
continue;
}
if (file.directory) {
continue;
}
this.logger.debug(`Copying file: ${file.name}`);
await container.copyFrom(`${upgradeDirectory}/${file.name}`, `${stagingDir}`);
}
}
},
};
}
taskCheckNetworkNodePods(context_, task, nodeAliases, maxAttempts) {
context_.config.podRefs = {};
const consensusNodes = context_.config.consensusNodes;
const subTasks = [];
for (const nodeAlias of nodeAliases) {
subTasks.push({
title: `Check network pod: ${chalk.yellow(nodeAlias)}`,
task: async ({ config }) => {
try {
const context = helpers.extractContextFromConsensusNodes(nodeAlias, consensusNodes);
config.podRefs[nodeAlias] = await this.checkNetworkNodePod(config.namespace, nodeAlias, maxAttempts, undefined, context);
}
catch {
config.skipStop = true;
}
},
});
}
// setup the sub-tasks
return task.newListr(subTasks, {
concurrent: true,
rendererOptions: {
collapseSubtasks: false,
},
});
}
/** Check if the network node pod is running */
async checkNetworkNodePod(namespace, nodeAlias, maxAttempts = constants.PODS_RUNNING_MAX_ATTEMPTS, delay = constants.PODS_RUNNING_DELAY, context) {
nodeAlias = nodeAlias.trim();
const podName = Templates.renderNetworkPodName(nodeAlias);
const podReference = PodReference.of(namespace, podName);
if (typeof context !== 'string' || context.trim().length === 0) {
context = extractContextFromConsensusNodes(nodeAlias, this.remoteConfig.getConsensusNodes());
}
try {
await this.k8Factory
.getK8(context)
.pods()
.waitForRunningPhase(namespace, [`solo.hedera.com/node-name=${nodeAlias}`, 'solo.hedera.com/type=network-node'], maxAttempts, delay);
return podReference;
}
catch (error) {
throw new SoloError(`no pod found for nodeAlias: ${nodeAlias}`, error);
}
}
loadConfiguration(argv, leaseWrapper, leaseManager, validateRemoteConfig = true) {
return {
title: 'Load configuration',
task: async () => {
await this.localConfig.load();
await this.remoteConfig.loadAndValidate(argv, validateRemoteConfig);
if (!this.oneShotState.isActive()) {
leaseWrapper.lease = await leaseManager.create();
}
},
};
}
/**
* Resolve the active node aliases and their service map for the given namespace/deployment.
* Nodes whose accountId equals {@link constants.IGNORED_NODE_ACCOUNT_ID} are excluded.
*
* Shared by {@link getExistingNodeAliases} (non-task callers) and
* {@link identifyExistingNodes} (Listr task) to avoid duplicating the
* `getNodeServiceMap` + filter loop in both places.
*/
async resolveExistingNodes(namespace, deployment) {
const clusterReferences = this.remoteConfig.getClusterRefs();
const serviceMap = await this.accountManager.getNodeServiceMap(namespace, clusterReferences, deployment);
const existingNodeAliases = [];
for (const networkNodeServices of serviceMap.values()) {
if (networkNodeServices.accountId === constants.IGNORED_NODE_ACCOUNT_ID) {
continue;
}
existingNodeAliases.push(networkNodeServices.nodeAlias);
}
return { existingNodeAliases, serviceMap };
}
async getExistingNodeAliases(namespace, deployment) {
const { existingNodeAliases } = await this.resolveExistingNodes(namespace, deployment);
return existingNodeAliases;
}
identifyExistingNodes() {
return {
ti