@hashgraph/solo
Version:
An opinionated CLI tool to deploy and manage private Hedera Networks.
198 lines • 10.8 kB
JavaScript
// SPDX-License-Identifier: Apache-2.0
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
import { PodMetrics } from '../model/pod-metrics.js';
import { NamespaceName } from '../../../types/namespace/namespace-name.js';
import { ShellRunner } from '../../../core/shell-runner.js';
import { PodName } from '../../../integration/kube/resources/pod/pod-name.js';
import { inject, injectable } from 'tsyringe-neo';
import { patchInject } from '../../../core/dependency-injection/container-helper.js';
import { InjectTokens } from '../../../core/dependency-injection/inject-tokens.js';
import fs from 'node:fs';
import { AggregatedMetrics } from '../model/aggregated-metrics.js';
import { ClusterMetrics } from '../model/cluster-metrics.js';
import { ContainerReference } from '../../../integration/kube/resources/container/container-reference.js';
import { ContainerName } from '../../../integration/kube/resources/container/container-name.js';
import { PodReference } from '../../../integration/kube/resources/pod/pod-reference.js';
import { container } from 'tsyringe-neo';
import { Duration } from '../../../core/time/duration.js';
import path from 'node:path';
let MetricsServerImpl = class MetricsServerImpl {
logger;
k8Factory;
ignorePodMetrics;
installationDirectory;
constructor(logger, k8Factory, ignorePodMetrics, installationDirectory) {
this.logger = logger;
this.k8Factory = k8Factory;
this.ignorePodMetrics = ignorePodMetrics;
this.installationDirectory = installationDirectory;
this.logger = patchInject(logger, InjectTokens.SoloLogger, this.constructor.name);
this.k8Factory = patchInject(k8Factory, InjectTokens.K8Factory, this.constructor.name);
this.ignorePodMetrics = patchInject(ignorePodMetrics, InjectTokens.IgnorePodMetrics, this.constructor.name);
this.installationDirectory = patchInject(installationDirectory, InjectTokens.KubectlInstallationDirectory, this.constructor.name);
}
async getMetrics(snapshotName, namespaceLookup = undefined, labelSelector = undefined, contexts = undefined, events = []) {
const clusterMetrics = [];
if (!contexts || contexts?.length === 0) {
const clusterMetric = await this.getClusterMetrics(namespaceLookup, labelSelector);
if (clusterMetric) {
clusterMetrics.push(clusterMetric);
}
}
else {
for (const context of contexts) {
const clusterMetric = await this.getClusterMetrics(namespaceLookup, labelSelector, context);
if (clusterMetric) {
clusterMetrics.push(clusterMetric);
}
}
}
return this.createAggregatedMetrics(snapshotName, clusterMetrics, events);
}
async getClusterMetrics(namespaceLookup = undefined, labelSelector = undefined, context = undefined, attempt = 1) {
let podMetrics = [];
let clusterNamespace = '';
let mirrorNodePostgresPodName = undefined;
let mirrorNodePostgresNamespace = undefined;
try {
const podMetricItems = await this.k8Factory
.getK8(context && context !== 'default' ? context : undefined)
.pods()
.topPods(namespaceLookup, labelSelector);
for (const item of podMetricItems) {
const podName = item.podName.name;
const namespace = item.namespace.name;
podMetrics.push(new PodMetrics(item.namespace, item.podName, item.cpuInMillicores, item.memoryInMebibytes));
if (podName.startsWith('network-node1-0')) {
clusterNamespace = namespace;
}
// Capture both internal mirror node postgres and external postgres pods
if ((podName.startsWith('mirror-') && podName.includes('postgres')) || podName.startsWith('my-postgresql')) {
mirrorNodePostgresPodName = podName;
mirrorNodePostgresNamespace = namespace;
}
}
podMetrics = podMetrics.filter((podMetric) => {
for (const ignorePattern of this.ignorePodMetrics) {
if (podMetric.podName.name.includes(ignorePattern)) {
return false;
}
}
return true;
});
return this.createClusterMetrics(context ?? 'default', clusterNamespace ? NamespaceName.of(clusterNamespace) : undefined, podMetrics, mirrorNodePostgresPodName ? PodName.of(mirrorNodePostgresPodName) : undefined, mirrorNodePostgresNamespace ? NamespaceName.of(mirrorNodePostgresNamespace) : undefined);
}
catch (error) {
if (error.message.includes('Metrics API not available') ||
error.message.includes('service unavailable') ||
error.message.includes('Error occurred in metrics request')) {
if (attempt <= 3) {
const backOffSeconds = 5;
this.logger.debug(`Metrics API not available, retrying attempt ${attempt} after ${backOffSeconds} seconds...`, error);
await new Promise((resolve) => setTimeout(resolve, Duration.ofSeconds(backOffSeconds).toMillis()));
return this.getClusterMetrics(namespaceLookup, labelSelector, context, attempt + 1);
}
else {
this.logger.showUser('Metrics API not available for reporting metrics');
return undefined;
}
}
throw error;
}
}
async createAggregatedMetrics(snapshotName, clusterMetrics, events = []) {
let namespace = undefined;
if (!clusterMetrics || clusterMetrics?.length === 0) {
return undefined;
}
let cpuInMillicores = 0;
let memoryInMebibytes = 0;
let runtime = 0;
let transactions = 0;
for (const clusterMetric of clusterMetrics) {
cpuInMillicores += clusterMetric.cpuInMillicores;
memoryInMebibytes += clusterMetric.memoryInMebibytes;
runtime += await this.getNetworkNodeRuntime(clusterMetric.namespace, clusterMetric.context);
transactions += await this.getNetworkTransactions(clusterMetric.postgresNamespace, clusterMetric.context, clusterMetric.postgresPodName);
namespace = clusterMetric.namespace?.name ? clusterMetric.namespace : namespace;
const remoteConfigRuntimeState = container.resolve(InjectTokens.RemoteConfigRuntimeState);
if (namespace && namespace.name) {
await remoteConfigRuntimeState.load(namespace, clusterMetric.context);
}
}
return new AggregatedMetrics(snapshotName, clusterMetrics, cpuInMillicores, memoryInMebibytes, runtime, transactions, events);
}
createClusterMetrics(context, namespace, podMetrics, mirrorNodePostgresPodName, postgresNamespace) {
if (!podMetrics || podMetrics?.length === 0) {
return undefined;
}
let cpuInMillicores = 0;
let memoryInMebibytes = 0;
for (const podMetric of podMetrics) {
cpuInMillicores += podMetric.cpuInMillicores;
memoryInMebibytes += podMetric.memoryInMebibytes;
}
return new ClusterMetrics(context, namespace, podMetrics, mirrorNodePostgresPodName, postgresNamespace, cpuInMillicores, memoryInMebibytes);
}
async logMetrics(snapshotName, metricsLogFile, namespace, labelSelector, contexts, events = []) {
const aggregatedMetrics = await this.getMetrics(snapshotName, namespace, labelSelector, contexts, events);
fs.writeFileSync(`${metricsLogFile}.json`, aggregatedMetrics ? aggregatedMetrics.toString() : '');
}
async getNetworkNodeRuntime(namespace, context) {
if (!namespace) {
return 0;
}
const contextParameter = context && context !== 'default' ? `--context ${context}` : '';
const cmd = `kubectl get pod network-node1-0 -n ${namespace.name} --no-headers ${contextParameter} | awk '{print $5}'`;
const results = await new ShellRunner().run(cmd, [], true, false, {
PATH: `${this.installationDirectory}${path.delimiter}${process.env.PATH}`,
});
if (results?.length > 0) {
return Number.parseInt(results[0].split('m')[0]);
}
return 0;
}
async getNetworkTransactions(namespace, context, postgresPodName) {
if (!namespace || !postgresPodName) {
this.logger.debug(`getNetworkTransactions skipped: namespace=${namespace?.name}, postgresPodName=${postgresPodName}`);
return 0;
}
try {
const result = await this.k8Factory
.getK8(context && context !== 'default' ? context : undefined)
.containers()
.readByRef(ContainerReference.of(PodReference.of(namespace, postgresPodName), ContainerName.of('postgresql')))
.execContainer([
'bash',
'-c',
"PGPASSWORD=$(cat $POSTGRES_PASSWORD_FILE) psql -U postgres -d mirror_node -c 'select count(*) from transaction;' -t",
]);
return Number.parseInt(result.trim());
}
catch (error) {
this.logger.warn(`error looking up transactions: ${error.message}`, error);
}
return 0;
}
};
MetricsServerImpl = __decorate([
injectable(),
__param(0, inject(InjectTokens.SoloLogger)),
__param(1, inject(InjectTokens.K8Factory)),
__param(2, inject(InjectTokens.IgnorePodMetrics)),
__param(3, inject(InjectTokens.KubectlInstallationDirectory)),
__metadata("design:paramtypes", [Object, Object, Array, String])
], MetricsServerImpl);
export { MetricsServerImpl };
//# sourceMappingURL=metrics-server-impl.js.map