@hashgraph/solo
Version:
An opinionated CLI tool to deploy and manage private Hedera Networks.
888 lines • 77.7 kB
JavaScript
// SPDX-License-Identifier: Apache-2.0
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var NetworkCommand_1;
import { ListrInquirerPromptAdapter } from '@listr2/prompt-adapter-inquirer';
import { confirm as confirmPrompt } from '@inquirer/prompts';
import chalk from 'chalk';
import { SoloError } from '../core/errors/solo-error.js';
import { UserBreak } from '../core/errors/user-break.js';
import { BaseCommand } from './base.js';
import { Flags as flags } from './flags.js';
import * as constants from '../core/constants.js';
import { getEnvironmentVariable } from '../core/constants.js';
import { Templates } from '../core/templates.js';
import { addRootImageValues, createAndCopyBlockNodeJsonFileForConsensusNode, parseNodeAliases, prepareValuesFilesMapMultipleCluster, resolveValidJsonFilePath, showVersionBanner, sleep, } from '../core/helpers.js';
import { helmValuesHelper } from '../core/helm-values-helper.js';
import { resolveNamespaceFromDeployment } from '../core/resolvers.js';
import fs from 'node:fs';
import path from 'node:path';
import { ListrLock } from '../core/lock/listr-lock.js';
import { v4 as uuidv4 } from 'uuid';
import { Base64 } from 'js-base64';
import { SecretType } from '../integration/kube/resources/secret/secret-type.js';
import { Duration } from '../core/time/duration.js';
import { PathEx } from '../business/utils/path-ex.js';
import { inject, injectable } from 'tsyringe-neo';
import { InjectTokens } from '../core/dependency-injection/inject-tokens.js';
import { patchInject } from '../core/dependency-injection/container-helper.js';
import { DeploymentPhase } from '../data/schema/model/remote/deployment-phase.js';
import { ComponentTypes } from '../core/config/remote/enumerations/component-types.js';
import { PvcName } from '../integration/kube/resources/pvc/pvc-name.js';
import { PvcReference } from '../integration/kube/resources/pvc/pvc-reference.js';
import { NamespaceName } from '../types/namespace/namespace-name.js';
import { SemanticVersion } from '../business/utils/semantic-version.js';
import * as versions from '../../version.js';
import { K8Helper } from '../business/utils/k8-helper.js';
import { PackageDownloader } from '../core/package-downloader.js';
import { Zippy } from '../core/zippy.js';
import { NetworkDeployedEvent } from '../core/events/event-types/network-deployed-event.js';
let NetworkCommand = class NetworkCommand extends BaseCommand {
static { NetworkCommand_1 = this; }
certificateManager;
keyManager;
platformInstaller;
profileManager;
zippy;
downloader;
eventBus;
profileValuesFile;
constructor(certificateManager, keyManager, platformInstaller, profileManager, zippy, downloader, eventBus) {
super();
this.certificateManager = certificateManager;
this.keyManager = keyManager;
this.platformInstaller = platformInstaller;
this.profileManager = profileManager;
this.zippy = zippy;
this.downloader = downloader;
this.eventBus = eventBus;
this.certificateManager = patchInject(certificateManager, InjectTokens.CertificateManager, this.constructor.name);
this.keyManager = patchInject(keyManager, InjectTokens.KeyManager, this.constructor.name);
this.platformInstaller = patchInject(platformInstaller, InjectTokens.PlatformInstaller, this.constructor.name);
this.profileManager = patchInject(profileManager, InjectTokens.ProfileManager, this.constructor.name);
this.zippy = patchInject(zippy, InjectTokens.Zippy, this.constructor.name);
this.downloader = patchInject(downloader, InjectTokens.PackageDownloader, this.constructor.name);
}
static DEPLOY_CONFIGS_NAME = 'deployConfigs';
static DESTROY_FLAGS_LIST = {
required: [flags.deployment],
optional: [flags.deletePvcs, flags.deleteSecrets, flags.enableTimeout, flags.force, flags.quiet],
};
static DEPLOY_FLAGS_LIST = {
required: [flags.deployment],
optional: [
flags.apiPermissionProperties,
flags.app,
flags.applicationEnv,
flags.applicationProperties,
flags.bootstrapProperties,
flags.genesisThrottlesFile,
flags.cacheDir,
flags.chainId,
flags.chartDirectory,
flags.soloChartVersion,
flags.debugNodeAlias,
flags.loadBalancerEnabled,
flags.log4j2Xml,
flags.persistentVolumeClaims,
flags.quiet,
flags.releaseTag,
flags.settingTxt,
flags.networkDeploymentValuesFile,
flags.nodeAliasesUnparsed,
flags.grpcTlsCertificatePath,
flags.grpcWebTlsCertificatePath,
flags.grpcTlsKeyPath,
flags.grpcWebTlsKeyPath,
flags.haproxyIps,
flags.envoyIps,
flags.storageType,
flags.gcsWriteAccessKey,
flags.gcsWriteSecrets,
flags.gcsEndpoint,
flags.gcsBucket,
flags.gcsBucketPrefix,
flags.awsWriteAccessKey,
flags.awsWriteSecrets,
flags.awsEndpoint,
flags.awsBucket,
flags.awsBucketRegion,
flags.awsBucketPrefix,
flags.backupBucket,
flags.backupWriteAccessKey,
flags.backupWriteSecrets,
flags.backupEndpoint,
flags.backupRegion,
flags.backupProvider,
flags.domainNames,
flags.serviceMonitor,
flags.podLog,
flags.enableMonitoringSupport,
flags.javaFlightRecorderConfiguration,
flags.wrapsEnabled,
flags.wrapsKeyPath,
flags.tssEnabled,
],
};
waitForNetworkPods() {
return {
title: 'Check node pods are running',
task: (context_, task) => {
const subTasks = [];
const config = context_.config;
for (const consensusNode of config.consensusNodes) {
subTasks.push({
title: `Check Node: ${chalk.yellow(consensusNode.name)}, Cluster: ${chalk.yellow(consensusNode.cluster)}`,
task: async () => {
await this.k8Factory
.getK8(consensusNode.context)
.pods()
.waitForRunningPhase(config.namespace, [`solo.hedera.com/node-name=${consensusNode.name}`, 'solo.hedera.com/type=network-node'], constants.PODS_RUNNING_MAX_ATTEMPTS, constants.PODS_RUNNING_DELAY);
},
});
}
// set up the sub-tasks
return task.newListr(subTasks, {
concurrent: true,
rendererOptions: {
collapseSubtasks: false,
},
});
},
};
}
async prepareMinioSecrets(config, minioAccessKey, minioSecretKey) {
// Generating new minio credentials
const minioData = {};
const namespace = config.namespace;
const environmentString = `MINIO_ROOT_USER=${minioAccessKey}\nMINIO_ROOT_PASSWORD=${minioSecretKey}`;
minioData['config.env'] = Base64.encode(environmentString);
// create minio secret in each cluster
for (const context of config.contexts) {
this.logger.debug(`creating minio secret using context: ${context}`);
const isMinioSecretCreated = await this.k8Factory
.getK8(context)
.secrets()
.createOrReplace(namespace, constants.MINIO_SECRET_NAME, SecretType.OPAQUE, minioData);
if (!isMinioSecretCreated) {
throw new SoloError(`failed to create new minio secret using context: ${context}`);
}
this.logger.debug(`created minio secret using context: ${context}`);
}
}
async prepareStreamUploaderSecrets(config) {
const namespace = config.namespace;
// Generating cloud storage secrets
const { gcsWriteAccessKey, gcsWriteSecrets, gcsEndpoint, awsWriteAccessKey, awsWriteSecrets, awsEndpoint } = config;
const cloudData = {};
if (config.storageType === constants.StorageType.AWS_ONLY ||
config.storageType === constants.StorageType.AWS_AND_GCS) {
cloudData['S3_ACCESS_KEY'] = Base64.encode(awsWriteAccessKey);
cloudData['S3_SECRET_KEY'] = Base64.encode(awsWriteSecrets);
cloudData['S3_ENDPOINT'] = Base64.encode(awsEndpoint);
}
if (config.storageType === constants.StorageType.GCS_ONLY ||
config.storageType === constants.StorageType.AWS_AND_GCS) {
cloudData['GCS_ACCESS_KEY'] = Base64.encode(gcsWriteAccessKey);
cloudData['GCS_SECRET_KEY'] = Base64.encode(gcsWriteSecrets);
cloudData['GCS_ENDPOINT'] = Base64.encode(gcsEndpoint);
}
// create secret in each cluster
for (const context of config.contexts) {
this.logger.debug(`creating secret for storage credential of type '${config.storageType}' using context: ${context}`);
const isCloudSecretCreated = await this.k8Factory
.getK8(context)
.secrets()
.createOrReplace(namespace, constants.UPLOADER_SECRET_NAME, SecretType.OPAQUE, cloudData);
if (!isCloudSecretCreated) {
throw new SoloError(`failed to create secret for storage credentials of type '${config.storageType}' using context: ${context}`);
}
this.logger.debug(`created secret for storage credential of type '${config.storageType}' using context: ${context}`);
}
}
async prepareBackupUploaderSecrets(config) {
const { backupWriteAccessKey, backupWriteSecrets, backupEndpoint, backupRegion, backupProvider } = config;
const backupData = {};
const namespace = config.namespace;
backupData['AWS_ACCESS_KEY_ID'] = Base64.encode(backupWriteAccessKey);
backupData['AWS_SECRET_ACCESS_KEY'] = Base64.encode(backupWriteSecrets);
backupData['RCLONE_CONFIG_BACKUPS_ENDPOINT'] = Base64.encode(backupEndpoint);
backupData['RCLONE_CONFIG_BACKUPS_REGION'] = Base64.encode(backupRegion);
backupData['RCLONE_CONFIG_BACKUPS_TYPE'] = Base64.encode('s3');
backupData['RCLONE_CONFIG_BACKUPS_PROVIDER'] = Base64.encode(backupProvider);
// create secret in each cluster
for (const context of config.contexts) {
this.logger.debug(`creating secret for backup uploader using context: ${context}`);
const k8client = this.k8Factory.getK8(context);
const isBackupSecretCreated = await k8client
.secrets()
.createOrReplace(namespace, constants.BACKUP_SECRET_NAME, SecretType.OPAQUE, backupData);
if (!isBackupSecretCreated) {
throw new SoloError(`failed to create secret for backup uploader using context: ${context}`);
}
this.logger.debug(`created secret for backup uploader using context: ${context}`);
}
}
async prepareStorageSecrets(config) {
try {
if (config.storageType !== constants.StorageType.MINIO_ONLY) {
const minioAccessKey = uuidv4();
const minioSecretKey = uuidv4();
await this.prepareMinioSecrets(config, minioAccessKey, minioSecretKey);
await this.prepareStreamUploaderSecrets(config);
}
if (config.backupBucket) {
await this.prepareBackupUploaderSecrets(config);
}
}
catch (error) {
throw new SoloError('Failed to create Kubernetes storage secret', error);
}
}
/**
* Prepare values args string for each cluster-ref
* @param config
*/
async prepareValuesArgMap(config) {
const valuesArguments = this.prepareValuesArg(config);
// prepare values files for each cluster
const valuesArgumentMap = {};
const deploymentName = this.configManager.getFlag(flags.deployment);
const applicationPropertiesPath = PathEx.joinWithRealPath(config.cacheDir, 'templates', constants.APPLICATION_PROPERTIES);
const jfrFilePath = config.javaFlightRecorderConfiguration;
const jfrFile = jfrFilePath === '' ? '' : jfrFilePath.slice(Math.max(0, jfrFilePath.lastIndexOf(path.sep) + 1));
this.profileValuesFile = await this.profileManager.prepareValuesForSoloChart(config.consensusNodes, deploymentName, applicationPropertiesPath, jfrFile, {
// Pass command-scoped values explicitly so profile/staging generation is isolated
// from mutable global flags when one-shot runs parallel subcommands.
cacheDir: config.cacheDir,
releaseTag: config.releaseTag,
appName: config.app,
chainId: config.chainId,
});
const valuesFiles = prepareValuesFilesMapMultipleCluster(config.clusterRefs, config.chartDirectory, this.profileValuesFile, config.valuesFile, [constants.SOLO_DEPLOYMENT_VALUES_FILE]);
// Generate per-cluster extraEnv values files to avoid passing the global node list to every
// cluster's Helm upgrade (in multi-cluster deployments each cluster has its own node subset).
// Each file carries only the nodes that belong to the target cluster, preventing Helm's
// array-replacement semantics from inserting nodes from other clusters.
const perClusterExtraEnvironmentValuesFiles = {};
const needsExtraEnvironment = config.wrapsEnabled || !!config.debugNodeAlias || config.app !== constants.HEDERA_APP_NAME; // JAVA_MAIN_CLASS for tools/local builds
if (needsExtraEnvironment) {
const realm = this.localConfig.configuration.realmForDeployment(config.deployment);
const shard = this.localConfig.configuration.shardForDeployment(config.deployment);
for (const clusterReference of Object.keys(valuesFiles)) {
// Only include nodes belonging to this cluster so the generated hedera.nodes array
// matches the cluster-specific node set and does not overwrite nodes in other clusters.
// Sort deterministically by nodeId so per-node Helm values align with the chart's
// expected node ordering regardless of upstream object iteration order.
const clusterConsensusNodes = config.consensusNodes
.filter((node) => node.cluster === clusterReference)
// eslint-disable-next-line unicorn/no-array-sort
.sort((left, right) => left.nodeId - right.nodeId);
if (clusterConsensusNodes.length === 0) {
continue;
}
const additionalNodeValues = {};
// Preserve blockNodesJson from the per-cluster profile values file so that it is not
// silently dropped when the extraEnv values file replaces the hedera.nodes array.
const clusterProfileValuesFile = this.profileValuesFile?.[clusterReference];
const nodeIdentityMap = clusterProfileValuesFile
? helmValuesHelper.extractPerNodeIdentityFromValuesFile(clusterProfileValuesFile, clusterConsensusNodes)
: {};
const blockNodesJsonMap = clusterProfileValuesFile
? helmValuesHelper.extractPerNodeBlockNodesJsonFromValuesFile(clusterProfileValuesFile, clusterConsensusNodes)
: {};
for (const consensusNode of clusterConsensusNodes) {
const identity = nodeIdentityMap[consensusNode.name] ?? {};
additionalNodeValues[consensusNode.name] = {
name: identity.name ?? consensusNode.name,
nodeId: identity.nodeId ?? consensusNode.nodeId,
// Prefer the accountId recorded in the profile values file (set by the account
// manager using the deployment's configured start account ID) over the computed
// default, so custom account IDs assigned via node transactions are preserved.
accountId: identity.accountId ?? `${shard}.${realm}.${constants.DEFAULT_START_ID_NUMBER + consensusNode.nodeId}`,
};
if (blockNodesJsonMap[consensusNode.name]) {
additionalNodeValues[consensusNode.name].blockNodesJson = blockNodesJsonMap[consensusNode.name];
}
}
// Collect extraEnv entries already present in this cluster's values files so that the
// generated file can include them and avoid Helm array replacement silently dropping
// env vars set by user-provided values files.
const existingValuesFilePaths = helmValuesHelper.parseValuesFilePaths(valuesFiles[clusterReference]);
const clusterExtraEnvironmentValuesFile = helmValuesHelper.generateExtraEnvironmentValuesFile(clusterConsensusNodes, {
wrapsEnabled: config.wrapsEnabled,
tss: this.soloConfig.tss,
debugNodeAlias: config.debugNodeAlias,
useJavaMainClass: config.app !== constants.HEDERA_APP_NAME,
additionalNodeValues,
baseExtraEnvironmentVariables: helmValuesHelper.extractExtraEnvironmentFromValuesFiles(existingValuesFilePaths, clusterConsensusNodes),
}, config.cacheDir);
perClusterExtraEnvironmentValuesFiles[clusterReference] = clusterExtraEnvironmentValuesFile;
this.logger.debug(`Created per-cluster extraEnv values file for ${clusterReference}: ${clusterExtraEnvironmentValuesFile}`);
}
}
for (const clusterReference of Object.keys(valuesFiles)) {
// Keep --set flags last so they override values files. This is critical when we also
// provide per-node extraEnv via a values file (e.g. --debug-node-alias), because a later
// values file can replace array elements and drop fields like node labels/account IDs.
let valuesArgument = valuesFiles[clusterReference];
// Add per-cluster extraEnv values file if any extraEnv customizations are needed
if (perClusterExtraEnvironmentValuesFiles[clusterReference]) {
valuesArgument += ` --values "${perClusterExtraEnvironmentValuesFiles[clusterReference]}"`;
}
valuesArgument += valuesArguments[clusterReference];
valuesArgumentMap[clusterReference] = valuesArgument;
this.logger.debug(`Prepared helm chart values for cluster-ref: ${clusterReference}`, {
valuesArgument: valuesArgumentMap[clusterReference],
});
}
return valuesArgumentMap;
}
/**
* Prepare the values argument for the helm chart for a given config
* @param config
*/
prepareValuesArg(config) {
const valuesArguments = {};
const clusterReferences = [];
// initialize the valueArgs
for (const consensusNode of config.consensusNodes) {
// add the cluster to the list of clusters
if (!clusterReferences.includes(consensusNode.cluster)) {
clusterReferences.push(consensusNode.cluster);
}
// Initialize empty valuesArg for each cluster
// All extraEnv logic (JAVA_MAIN_CLASS, TSS wraps, debug) is now handled via values files
if (!valuesArguments[consensusNode.cluster]) {
valuesArguments[consensusNode.cluster] = '';
}
}
// All extraEnv customizations (wraps, debug, JAVA_MAIN_CLASS) are handled
// via generateExtraEnvironmentValuesFile() in prepareValuesArgMap() to avoid Helm --set replacement issues
if (config.storageType === constants.StorageType.AWS_AND_GCS ||
config.storageType === constants.StorageType.GCS_ONLY) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] += ' --set cloud.gcs.enabled=true';
}
}
if (config.storageType === constants.StorageType.AWS_AND_GCS ||
config.storageType === constants.StorageType.AWS_ONLY) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] += ' --set cloud.s3.enabled=true';
}
}
if (config.storageType === constants.StorageType.GCS_ONLY ||
config.storageType === constants.StorageType.AWS_ONLY ||
config.storageType === constants.StorageType.AWS_AND_GCS) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] += ' --set cloud.minio.enabled=false';
}
}
if (config.storageType !== constants.StorageType.MINIO_ONLY) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] += ' --set cloud.generateNewSecrets=false';
}
}
if (config.gcsBucket) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] +=
` --set cloud.buckets.streamBucket=${config.gcsBucket}` +
` --set minio-server.tenant.buckets[0].name=${config.gcsBucket}`;
}
}
if (config.gcsBucketPrefix) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] += ` --set cloud.buckets.streamBucketPrefix=${config.gcsBucketPrefix}`;
}
}
if (config.awsBucket) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] +=
` --set cloud.buckets.streamBucket=${config.awsBucket}` +
` --set minio-server.tenant.buckets[0].name=${config.awsBucket}`;
}
}
if (config.awsBucketPrefix) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] += ` --set cloud.buckets.streamBucketPrefix=${config.awsBucketPrefix}`;
}
}
if (config.awsBucketRegion) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] += ` --set cloud.buckets.streamBucketRegion=${config.awsBucketRegion}`;
}
}
if (config.backupBucket) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] +=
' --set defaults.sidecars.backupUploader.enabled=true' +
` --set defaults.sidecars.backupUploader.config.backupBucket=${config.backupBucket}`;
}
}
const nodeIndexByClusterAndName = new Map();
const nextNodeIndexByCluster = new Map();
for (const consensusNode of config.consensusNodes) {
const nodeIndex = nextNodeIndexByCluster.get(consensusNode.cluster) ?? 0;
nextNodeIndexByCluster.set(consensusNode.cluster, nodeIndex + 1);
nodeIndexByClusterAndName.set(`${consensusNode.cluster}:${consensusNode.name}`, nodeIndex);
}
for (const consensusNode of config.consensusNodes) {
const nodeIndex = nodeIndexByClusterAndName.get(`${consensusNode.cluster}:${consensusNode.name}`);
if (nodeIndex === undefined) {
continue;
}
let valuesArgument = valuesArguments[consensusNode.cluster] ?? '';
valuesArgument += ` --set "hedera.nodes[${nodeIndex}].name=${consensusNode.name}"`;
valuesArgument = addRootImageValues(valuesArgument, `hedera.nodes[${nodeIndex}]`, constants.S6_NODE_IMAGE_REGISTRY, constants.S6_NODE_IMAGE_REPOSITORY, versions.S6_NODE_IMAGE_VERSION);
valuesArguments[consensusNode.cluster] = valuesArgument;
}
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] +=
' --install' +
' --set "telemetry.prometheus.svcMonitor.enabled=false"' + // remove after chart version is bumped
` --set "crds.serviceMonitor.enabled=${config.singleUseServiceMonitor}"` +
` --set "crds.podLog.enabled=${config.singleUsePodLog}"` +
` --set "defaults.volumeClaims.enabled=${config.persistentVolumeClaims}"`;
}
config.singleUseServiceMonitor = 'false';
config.singleUsePodLog = 'false';
// Iterate over each node and set static IPs for HAProxy
this.addArgForEachRecord(config.haproxyIpsParsed, config.consensusNodes, valuesArguments, ' --set "hedera.nodes[${nodeId}].haproxyStaticIP=${recordValue}"');
// Iterate over each node and set static IPs for Envoy Proxy
this.addArgForEachRecord(config.envoyIpsParsed, config.consensusNodes, valuesArguments, ' --set "hedera.nodes[${nodeId}].envoyProxyStaticIP=${recordValue}"');
if (config.resolvedThrottlesFile) {
// repairing the path, this avoid helm failing when running on windows
const throttlesFilePath = config.resolvedThrottlesFile.replaceAll('\\', '/');
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] +=
` --set-file "hedera.configMaps.genesisThrottlesJson=${throttlesFilePath}"`;
}
}
if (config.loadBalancerEnabled) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] +=
' --set "defaults.haproxy.service.type=LoadBalancer"' +
' --set "defaults.envoyProxy.service.type=LoadBalancer"' +
' --set "defaults.consensus.service.type=LoadBalancer"';
}
}
if (config.enableMonitoringSupport) {
for (const clusterReference of clusterReferences) {
valuesArguments[clusterReference] += ' --set "crs.podLog.enabled=true" --set "crs.serviceMonitor.enabled=true"';
}
}
return valuesArguments;
}
/**
* Adds the template string to the argument for each record
* @param records - the records to iterate over
* @param consensusNodes - the consensus nodes to iterate over
* @param valuesArguments - the values arguments to add to
* @param templateString - the template string to add
*/
addArgForEachRecord(records, consensusNodes, valuesArguments, templateString) {
if (records) {
for (const consensusNode of consensusNodes) {
if (records[consensusNode.name]) {
const newTemplateString = templateString.replace('{nodeId}', consensusNode.nodeId.toString());
valuesArguments[consensusNode.cluster] += newTemplateString.replace('{recordValue}', records[consensusNode.name]);
}
}
}
}
async prepareNamespaces(config) {
const namespace = config.namespace;
// check and create namespace in each cluster
for (const context of config.contexts) {
const k8client = this.k8Factory.getK8(context);
if (await k8client.namespaces().has(namespace)) {
this.logger.debug(`namespace '${namespace}' found using context: ${context}`);
}
else {
this.logger.debug(`creating namespace '${namespace}' using context: ${context}`);
await k8client.namespaces().create(namespace);
this.logger.debug(`created namespace '${namespace}' using context: ${context}`);
}
}
}
async prepareConfig(task, argv) {
const flagsWithDisabledPrompts = [
flags.apiPermissionProperties,
flags.app,
flags.applicationEnv,
flags.applicationProperties,
flags.bootstrapProperties,
flags.genesisThrottlesFile,
flags.cacheDir,
flags.chainId,
flags.chartDirectory,
flags.debugNodeAlias,
flags.loadBalancerEnabled,
flags.log4j2Xml,
flags.persistentVolumeClaims,
flags.settingTxt,
flags.grpcTlsCertificatePath,
flags.grpcWebTlsCertificatePath,
flags.grpcTlsKeyPath,
flags.grpcWebTlsKeyPath,
flags.haproxyIps,
flags.envoyIps,
flags.storageType,
flags.gcsWriteAccessKey,
flags.gcsWriteSecrets,
flags.gcsEndpoint,
flags.gcsBucket,
flags.gcsBucketPrefix,
flags.nodeAliasesUnparsed,
flags.domainNames,
];
// disable the prompts that we don't want to prompt the user for
flags.disablePrompts(flagsWithDisabledPrompts);
const allFlags = [
...NetworkCommand_1.DEPLOY_FLAGS_LIST.optional,
...NetworkCommand_1.DEPLOY_FLAGS_LIST.required,
];
await this.configManager.executePrompt(task, allFlags);
const namespace = (await resolveNamespaceFromDeployment(this.localConfig, this.configManager, task)) ??
NamespaceName.of(this.configManager.getFlag(flags.deployment));
this.configManager.setFlag(flags.namespace, namespace);
// create a config object for subsequent steps
const config = this.configManager.getConfig(NetworkCommand_1.DEPLOY_CONFIGS_NAME, allFlags, [
'keysDir',
'nodeAliases',
'stagingDir',
'stagingKeysDir',
'valuesArgMap',
'resolvedThrottlesFile',
'namespace',
'consensusNodes',
'contexts',
'clusterRefs',
'singleUsePodLog',
'singleUseServiceMonitor',
]);
const realm = this.localConfig.configuration.realmForDeployment(config.deployment);
const shard = this.localConfig.configuration.shardForDeployment(config.deployment);
const networkNodeVersion = new SemanticVersion(config.releaseTag);
const minimumVersionForNonZeroRealms = new SemanticVersion('0.60.0');
if ((realm !== 0 || shard !== 0) &&
new SemanticVersion(networkNodeVersion).lessThan(minimumVersionForNonZeroRealms)) {
throw new SoloError(`The realm and shard values must be 0 when using the ${minimumVersionForNonZeroRealms} version of the network node`);
}
if (config.haproxyIps) {
config.haproxyIpsParsed = Templates.parseNodeAliasToIpMapping(config.haproxyIps);
}
if (config.envoyIps) {
config.envoyIpsParsed = Templates.parseNodeAliasToIpMapping(config.envoyIps);
}
if (config.domainNames) {
config.domainNamesMapping = Templates.parseNodeAliasToDomainNameMapping(config.domainNames);
}
// compute other config parameters
config.keysDir = PathEx.join(config.cacheDir, 'keys');
config.stagingDir = Templates.renderStagingDir(config.cacheDir, config.releaseTag);
config.stagingKeysDir = PathEx.join(config.stagingDir, 'keys');
config.resolvedThrottlesFile = resolveValidJsonFilePath(config.genesisThrottlesFile, flags.genesisThrottlesFile.definition.defaultValue);
config.consensusNodes = this.remoteConfig.getConsensusNodes();
config.contexts = this.remoteConfig.getContexts();
config.clusterRefs = this.remoteConfig.getClusterRefs();
config.nodeAliases = parseNodeAliases(config.nodeAliasesUnparsed, config.consensusNodes, this.configManager);
argv[flags.nodeAliasesUnparsed.name] = config.nodeAliases.join(',');
config.blockNodeComponents = this.getBlockNodes();
config.javaFlightRecorderConfiguration = this.configManager.getFlag(flags.javaFlightRecorderConfiguration);
if (config.javaFlightRecorderConfiguration === '') {
config.javaFlightRecorderConfiguration = getEnvironmentVariable('JAVA_FLIGHT_RECORDER_CONFIGURATION') || '';
}
config.singleUseServiceMonitor = config.serviceMonitor;
config.singleUsePodLog = config.podLog;
config.valuesArgMap = await this.prepareValuesArgMap(config);
// need to prepare the namespaces before we can proceed
config.namespace = namespace;
await this.prepareNamespaces(config);
// prepare staging keys directory
if (!fs.existsSync(config.stagingKeysDir)) {
fs.mkdirSync(config.stagingKeysDir, { recursive: true });
}
// create cached keys dir if it does not exist yet
if (!fs.existsSync(config.keysDir)) {
fs.mkdirSync(config.keysDir);
}
this.logger.debug('Preparing storage secrets');
await this.prepareStorageSecrets(config);
return config;
}
async destroyTask(task, namespace, deletePvcs, deleteSecrets, contexts) {
task.title = `Uninstalling chart ${constants.SOLO_DEPLOYMENT_CHART}`;
// Uninstall all 'solo deployment' charts for each cluster using the contexts
await this.logDestroyResults('Uninstall solo-deployment chart', await Promise.allSettled(contexts.map(async (context) => {
await this.chartManager.uninstall(namespace, constants.SOLO_DEPLOYMENT_CHART, this.k8Factory.getK8(context).contexts().readCurrent());
})));
task.title = `Deleting the RemoteConfig configmap in namespace ${namespace}`;
await this.logDestroyResults('Delete remote config configmap', await Promise.allSettled(contexts.map(async (context) => {
await this.k8Factory.getK8(context).configMaps().delete(namespace, constants.SOLO_REMOTE_CONFIGMAP_NAME);
})));
if (deletePvcs) {
task.title = `Deleting PVCs in namespace ${namespace}`;
await this.logDestroyResults('Delete PVCs', await Promise.allSettled([this.deletePvcs(namespace, contexts)]));
}
if (deleteSecrets) {
task.title = `Deleting Secrets in namespace ${namespace}`;
await this.logDestroyResults('Delete secrets', await Promise.allSettled([this.deleteSecrets(namespace, contexts)]));
}
if (deleteSecrets && deletePvcs) {
task.title = `Deleting namespace ${namespace}`;
await this.logDestroyResults('Delete namespace', await Promise.allSettled(contexts.map(async (context) => {
await this.k8Factory.getK8(context).namespaces().delete(namespace);
})));
}
else {
task.title = `Deleting the RemoteConfig configmap in namespace ${namespace}`;
await Promise.all(contexts.map(async (context) => {
await this.k8Factory.getK8(context).configMaps().delete(namespace, constants.SOLO_REMOTE_CONFIGMAP_NAME);
}));
if (deletePvcs) {
task.title = `Deleting PVCs in namespace ${namespace}`;
await this.deletePvcs(namespace, contexts);
}
if (deleteSecrets) {
task.title = `Deleting Secrets in namespace ${namespace}`;
await this.deleteSecrets(namespace, contexts);
}
}
}
async logDestroyResults(title, results) {
const failures = results.filter((result) => result.status === 'rejected');
if (failures.length === 0) {
return;
}
for (const failure of failures) {
this.logger.warn(`${title} failed; continuing destroy`, failure.reason);
}
}
async deleteSecrets(namespace, contexts) {
const secretsData = [];
for (const context of contexts) {
const secrets = await this.k8Factory.getK8(context).secrets().list(namespace);
for (const secret of secrets) {
secretsData.push({ secret: secret.name, context: context });
}
}
const promises = secretsData.map(async ({ context, secret }) => {
await this.k8Factory.getK8(context).secrets().delete(namespace, secret);
});
await Promise.all(promises);
}
async deletePvcs(namespace, contexts) {
const pvcsData = [];
for (const context of contexts) {
const pvcs = await this.k8Factory.getK8(context).pvcs().list(namespace, []);
for (const pvc of pvcs) {
pvcsData.push({ pvc, context });
}
}
const promises = pvcsData.map(async ({ context, pvc }) => {
await this.k8Factory
.getK8(context)
.pvcs()
.delete(PvcReference.of(namespace, PvcName.of(pvc)))
.catch();
});
await Promise.all(promises);
}
async crdExists(context, crdName) {
return await this.k8Factory.getK8(context).crds().ifExists(crdName);
}
/**
* Ensure the PodLogs CRD from Grafana Alloy is installed
*/
async ensurePodLogsCrd({ contexts }) {
const PODLOGS_CRD = 'podlogs.monitoring.grafana.com';
const CRD_FILE_PATH = 'operations/helm/charts/alloy/charts/crds/crds/monitoring.grafana.com_podlogs.yaml';
// Use the GitHub Contents API (api.github.com) instead of raw.githubusercontent.com.
//
// Why: raw.githubusercontent.com is served by the Fastly CDN and its rate-limiting
// behaviour for unauthenticated requests is undocumented — adding a token there may
// have no effect. The Contents API, on the other hand, is part of the GitHub REST API
// (api.github.com) whose rate limits are well-documented: 60 req/hour unauthenticated
// vs 5 000 req/hour when a valid token is supplied. Since GITHUB_TOKEN is injected
// automatically into every GitHub Actions job, CI runs always get the higher limit,
// making 429s far less likely in the first place.
const CRD_URL = `https://api.github.com/repos/grafana/alloy/contents/${CRD_FILE_PATH}` +
`?ref=${versions.GRAFANA_PODLOGS_CRD_VERSION}`;
const CRD_RAW_URL = `https://raw.githubusercontent.com/grafana/alloy/${versions.GRAFANA_PODLOGS_CRD_VERSION}/${CRD_FILE_PATH}`;
const LOCAL_CRD_FILE = PathEx.join(constants.ROOT_DIR, 'resources', 'crds', `monitoring.grafana.com_podlogs-${versions.GRAFANA_PODLOGS_CRD_VERSION}.yaml`);
for (const context of contexts) {
const exists = await this.crdExists(context, PODLOGS_CRD);
if (exists) {
this.logger.debug(`CRD ${PODLOGS_CRD} already exists in context ${context}`);
continue;
}
this.logger.info(`Installing missing CRD ${PODLOGS_CRD} from ${CRD_URL} in context ${context}...`);
const temporaryFile = PathEx.join(constants.SOLO_CACHE_DIR, `podlogs-crd-${versions.GRAFANA_PODLOGS_CRD_VERSION}.yaml`);
// Download and cache the CRD YAML. The cache file is keyed by the CRD version so
// it is automatically invalidated when GRAFANA_PODLOGS_CRD_VERSION is bumped.
// SOLO_CACHE_DIR persists across job steps (unlike os.tmpdir() which is ephemeral),
// ensuring we only make one network request per job even if multiple contexts need
// the CRD installed.
if (!fs.existsSync(temporaryFile)) {
// Prefer a vendored CRD file to avoid external network/rate-limit failures in CI.
if (fs.existsSync(LOCAL_CRD_FILE)) {
fs.copyFileSync(LOCAL_CRD_FILE, temporaryFile);
this.logger.debug(`Using local PodLogs CRD file: ${LOCAL_CRD_FILE}`);
}
else {
const downloadErrors = [];
// Attempt #1: GitHub Contents API.
// The response is a JSON envelope with base64 content.
const apiHeaders = { Accept: 'application/vnd.github.v3+json' };
if (process.env.GITHUB_TOKEN) {
apiHeaders['Authorization'] = `Bearer ${process.env.GITHUB_TOKEN}`;
}
const apiResponse = await fetch(CRD_URL, { headers: apiHeaders });
if (apiResponse.ok) {
const json = (await apiResponse.json());
const yamlContent = Buffer.from(json.content.replaceAll(/\s/g, ''), 'base64').toString('utf8');
fs.writeFileSync(temporaryFile, yamlContent, 'utf8');
}
else {
const apiError = `${apiResponse.status} ${apiResponse.statusText}`.trim();
downloadErrors.push(`GitHub API: ${apiError}`);
this.logger.warn(`Failed to download PodLogs CRD from GitHub API (${apiError}), trying raw URL fallback.`);
// Attempt #2: raw.githubusercontent.com fallback.
const rawHeaders = {};
if (process.env.GITHUB_TOKEN) {
rawHeaders['Authorization'] = `Bearer ${process.env.GITHUB_TOKEN}`;
}
const rawResponse = await fetch(CRD_RAW_URL, { headers: rawHeaders });
if (!rawResponse.ok) {
const rawError = `${rawResponse.status} ${rawResponse.statusText}`.trim();
downloadErrors.push(`Raw URL: ${rawError}`);
throw new Error(`Failed to download CRD YAML (${downloadErrors.join('; ')})`);
}
const yamlContent = await rawResponse.text();
fs.writeFileSync(temporaryFile, yamlContent, 'utf8');
}
}
}
await this.k8Factory.getK8(context).manifests().applyManifest(temporaryFile);
}
}
/**
* Ensure all Prometheus Operator CRDs exist; install chart only if needed.
* If all CRDs are already present or monitoring support is disabled, skip installation.
*/
/** Ensure Prometheus Operator CRDs are present; install missing ones via the chart */
async ensurePrometheusOperatorCrds({ clusterRefs, namespace, deployment, }) {
const CRDS = [
{ key: 'alertmanagerconfigs', crd: 'alertmanagerconfigs.monitoring.coreos.com' },
{ key: 'alertmanagers', crd: 'alertmanagers.monitoring.coreos.com' },
{ key: 'podmonitors', crd: 'podmonitors.monitoring.coreos.com' },
{ key: 'probes', crd: 'probes.monitoring.coreos.com' },
{ key: 'prometheusagents', crd: 'prometheusagents.monitoring.coreos.com' },
{ key: 'prometheuses', crd: 'prometheuses.monitoring.coreos.com' },
{ key: 'prometheusrules', crd: 'prometheusrules.monitoring.coreos.com' },
{ key: 'scrapeconfigs', crd: 'scrapeconfigs.monitoring.coreos.com' },
{ key: 'servicemonitors', crd: 'servicemonitors.monitoring.coreos.com' },
{ key: 'thanosrulers', crd: 'thanosrulers.monitoring.coreos.com' },
];
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for (const [_, context] of clusterRefs) {
let valuesArgument = '';
let missingCount = 0;
for (const { key, crd } of CRDS) {
const exists = await this.crdExists(context, crd);
if (exists) {
valuesArgument += ` --set "${key}.enabled=false"`;
}
else {
missingCount++;
}
}
if (missingCount === 0) {
this.logger.info(`All Prometheus Operator CRDs already present in context ${context}; skipping installation.`);
continue;
}
const setupMap = new Map([
[constants.PROMETHEUS_OPERATOR_CRDS_RELEASE_NAME, constants.PROMETHEUS_OPERATOR_CRDS_CHART_URL],
]);
await this.chartManager.setup(setupMap);
await this.chartManager.install(namespace, constants.PROMETHEUS_OPERATOR_CRDS_RELEASE_NAME, constants.PROMETHEUS_OPERATOR_CRDS_CHART, constants.PROMETHEUS_OPERATOR_CRDS_CHART, versions.PROMETHEUS_OPERATOR_CRDS_VERSION, valuesArgument, context);
this.eventBus.emit(new NetworkDeployedEvent(deployment));
showVersionBanner(this.logger, constants.PROMETHEUS_OPERATOR_CRDS_CHART, versions.PROMETHEUS_OPERATOR_CRDS_VERSION);
}
}
/**
* Patch the ServiceMonitor created by the solo-deployment helm chart so that it is discovered
* by the kube-prometheus-stack Prometheus operator and targets the correct consensus node services.
*
* Two fixes are applied via a merge patch:
* 1. Adds the `release: <PROMETHEUS_RELEASE_NAME>` label so the Prometheus instance from
* kube-prometheus-stack (which selects ServiceMonitors by `release` label) can discover it.
* 2. Corrects `spec.selector.matchLabels` to `solo.hedera.com/type: network-node-svc` so the
* ServiceMonitor targets the non-headless consensus-node services (which expose the prometheus
* metrics port) rather than the hard-coded `network-node` value in the helm chart template.
*/
async patchServiceMonitorForPrometheus(namespace, context) {
const patch = {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
metadata: {
name: constants.SOLO_SERVICE_MONITOR_NAME,
namespace: namespace.name,
labels: {
release: constants.PROMETHEUS_RELEASE_NAME,
},
},
spec: {
selector: {
matchLabels: {
'solo.hedera.com/type': 'network-node-svc',
},
},
},
};
await this.k8Factory.getK8(context).manifests().patchObject(patch);
this.logger.debug(`Patched ServiceMonitor '${constants.SOLO_SERVICE_MONITOR_NAME}' in namespace '${namespace.name}': ` +
`added label release=${constants.PROMETHEUS_RELEASE_NAME} and fixed selector to network-node-svc`);
}
/** Run helm install and deploy network components */
async deploy(argv) {
let lease;
const tasks = this.taskList.newTaskList([
{
title: 'Initialize',
task: async (context_, task) => {
this.configManager.update(argv);
await this.localConfig.load();
await this.remoteConfig.loadAndValidate(argv, true, true);
if (!this.oneShotState.isActive()) {
lease = await this.leaseManager.create();
}
const releaseTag = new SemanticVersion(this.configManager.getFlag(flags.releaseTag));
if (this.remoteConfig.configuration.versions.consensusNode.toString() === '0.0.0' ||
!new SemanticVersion(this.remoteConfig.configuration.versions.consensusNode).equals(releaseTag)) {
// if is possible block node deployed before consensus node, then use release tag as fallback
this.remoteConfig.configuration.versions.consensusNode = releaseTag;
await this.remoteConfig.persist();
}
const currentVersion = new SemanticVersion(this.remoteConfig.configuration.versions.consensusNode.toString());
let tssEnabled = this.configManager.getFlag(flags.tssEnabled);
const minimumVersion = new SemanticVersion(versions.MINIMUM_HIERO_PLATFORM_VERSION_FOR_TSS);
// if platform version is insufficient for tss, disable it
if (tssEnabled && new SemanticVersion(currentVersion).lessThan(minimumVersion)) {
tssEnabled = false;
}
const wrapsEnabled = this.configManager.getFlag(flags.wrapsEnabled);
this.remoteConfig.configuration.state.wrapsEnabled = wrapsEnabled;
if (wrapsEnabled && new SemanticVersion(currentVersion).lessThan(minimumVersion)) {
this.logger.showUser(`Consensus node version ${currentVersion} does not support TSS or Wraps. Please upgrade to version ${minimumVersion} or la