@hashgraph/solo
Version:
An opinionated CLI tool to deploy and manage private Hedera Networks.
235 lines • 13.5 kB
JavaScript
// SPDX-License-Identifier: Apache-2.0
import { describe } from 'mocha';
import { resetForTest } from '../../test-container.js';
import { container } from 'tsyringe-neo';
import { InjectTokens } from '../../../src/core/dependency-injection/inject-tokens.js';
import fs from 'node:fs';
import { DEFAULT_LOCAL_CONFIG_FILE, SOLO_CACHE_DIR } from '../../../src/core/constants.js';
import { Duration } from '../../../src/core/time/duration.js';
import { PathEx } from '../../../src/business/utils/path-ex.js';
import { EndToEndTestSuiteBuilder } from '../end-to-end-test-suite-builder.js';
import { main } from '../../../src/index.js';
import { BaseCommandTest } from './tests/base-command-test.js';
import { OneShotCommandDefinition } from '../../../src/commands/command-definitions/one-shot-command-definition.js';
import { MetricsServerImpl } from '../../../src/business/runtime-state/services/metrics-server-impl.js';
import * as constants from '../../../src/core/constants.js';
import { sleep } from '../../../src/core/helpers.js';
import { Flags } from '../../../src/commands/flags.js';
const testName = 'performance-tests';
const deploymentName = `${testName}-deployment`;
const testTitle = 'E2E Performance Tests';
const duration = Duration.ofMinutes(Number.parseInt(process.env.ONE_SHOT_METRICS_TEST_DURATION_IN_MINUTES) || 5).seconds;
const clients = 5;
const accounts = 1000;
const tokens = 50;
const associations = 50;
const nfts = 50;
const percent = 50;
const maxTps = 100;
const nftTransferLoadTestTimeoutMultiplier = 6;
let startTime;
let metricsInterval;
let events = [];
let peakMemoryInMebibytes = 0;
// When the workflow cancels this step (e.g. due to a new commit superseding the PR),
// go-task forwards SIGTERM to this process' process group before SIGKILL reaches task.
// Without this handler, mocha's graceful shutdown waits for the currently-running
// `await sleep(...)` to resolve AND for the setInterval to drain — which can take
// minutes. Force-exit immediately so the runner can move on without waiting.
process.on('SIGTERM', () => {
clearInterval(metricsInterval);
process.exit(143); // 128 + SIGTERM(15)
});
const defaultJFREnvironmentValue = process.env.JAVA_FLIGHT_RECORDER_CONFIGURATION;
const endToEndTestSuite = new EndToEndTestSuiteBuilder()
.withTestName(testName)
.withTestSuiteName(`${testTitle} Suite`)
.withNamespace(testName)
.withDeployment(deploymentName)
.withClusterCount(1)
.withJavaFlightRecorderConfiguration('test/data/java-flight-recorder/LowMem.jfc')
.withTestSuiteCallback((options, preDestroy) => {
describe(testTitle, () => {
const { testCacheDirectory, testLogger, namespace, contexts, deployment } = options;
// TODO the kube config context causes issues if it isn't one of the selected clusters we are deploying to
before(async () => {
fs.rmSync(testCacheDirectory, { recursive: true, force: true });
try {
fs.rmSync(PathEx.joinWithRealPath(testCacheDirectory, '..', DEFAULT_LOCAL_CONFIG_FILE), {
force: true,
});
}
catch {
// allowed to fail if the file doesn't exist
}
if (!fs.existsSync(testCacheDirectory)) {
fs.mkdirSync(testCacheDirectory, { recursive: true });
}
resetForTest(namespace.name, testCacheDirectory, false);
for (const item of contexts) {
const k8Client = container.resolve(InjectTokens.K8Factory).getK8(item);
await k8Client.namespaces().delete(namespace);
}
testLogger.info(`${testName}: starting ${testName} e2e test`);
testLogger.info(`${testName}: beginning ${testName}: deploy`);
process.env.JAVA_FLIGHT_RECORDER_CONFIGURATION = options.javaFlightRecorderConfiguration;
await main(soloOneShotDeploy(testName, deployment));
testLogger.info(`${testName}: finished ${testName}: deploy`);
startTime = new Date();
metricsInterval = setInterval(async () => {
logMetrics(startTime);
}, Duration.ofSeconds(5).toMillis());
}).timeout(Duration.ofMinutes(25).toMillis());
after(async () => {
clearInterval(metricsInterval);
// restore environment variable for other tests
process.env.JAVA_FLIGHT_RECORDER_CONFIGURATION = defaultJFREnvironmentValue;
// read all logged metrics and parse the JSON
const namespace = await getNamespaceFromDeployment();
const tartgetDirectory = PathEx.join(constants.SOLO_LOGS_DIR, `${namespace}`);
const files = fs.readdirSync(tartgetDirectory);
const allMetrics = {};
for (const file of files) {
const filePath = PathEx.join(tartgetDirectory, file);
const fileContents = fs.readFileSync(filePath, 'utf8');
const fileName = file.split('.')[0];
allMetrics[fileName] = JSON.parse(fileContents);
}
// save the aggregated metrics to a single file
const aggregatedMetricsFileName = 'timeline-metrics.json';
const aggregatedMetricsPath = PathEx.join(tartgetDirectory, aggregatedMetricsFileName);
fs.writeFileSync(aggregatedMetricsPath, JSON.stringify(allMetrics), 'utf8');
let maxCpuMetrics = 0;
let maxCpuFile = '';
let maxMemoryMetrics = 0;
let maxMemoryFile = '';
for (const [fileName, metrics] of Object.entries(allMetrics)) {
if (metrics.cpuInMillicores > maxCpuMetrics) {
maxCpuMetrics = metrics.cpuInMillicores;
maxCpuFile = fileName;
}
if (metrics.memoryInMebibytes > maxMemoryMetrics) {
maxMemoryMetrics = metrics.memoryInMebibytes;
maxMemoryFile = fileName;
}
}
// Use the max-memory snapshot as the representative record since memory
// pressure reflects actual workload behavior, not startup CPU spikes
const representativeFileName = `${maxMemoryFile}.json`;
const { clusterMetrics: clusterMetricsData, ...summaryFields } = allMetrics[maxMemoryFile];
const namespaceJson = {
...summaryFields,
peakCpuInMillicores: maxCpuMetrics,
peakCpuSnapshot: allMetrics[maxCpuFile]?.snapshotName,
peakMemoryInMebibytes: maxMemoryMetrics,
peakMemorySnapshot: allMetrics[maxMemoryFile]?.snapshotName,
clusterMetrics: clusterMetricsData,
};
fs.writeFileSync(PathEx.join(tartgetDirectory, `${namespace}.json`), JSON.stringify(namespaceJson), 'utf8');
// remove all snapshot files except the representative one
const filesToKeep = new Set([representativeFileName, aggregatedMetricsFileName]);
for (const file of files) {
if (!filesToKeep.has(file)) {
fs.rmSync(PathEx.join(tartgetDirectory, file));
}
}
// copy the summary to the main solo logs directory to be accessible by existing scripts
fs.copyFileSync(PathEx.join(tartgetDirectory, `${namespace}.json`), PathEx.join(constants.SOLO_LOGS_DIR, `${namespace}.json`));
await preDestroy(endToEndTestSuite);
testLogger.info(`${testName}: beginning ${testName}: destroy`);
await main(soloOneShotDestroy(testName));
testLogger.info(`${testName}: finished ${testName}: destroy`);
}).timeout(Duration.ofMinutes(8).toMillis());
it('NftTransferLoadTest', async () => {
logEvent('Starting NftTransferLoadTest');
await main(soloRapidFire(testName, 'NftTransferLoadTest', `-c ${clients} -a ${accounts} -T ${nfts} -n ${accounts} -S flat -p ${percent} -R -t ${duration}`, maxTps));
}).timeout(Duration.ofSeconds(duration * nftTransferLoadTestTimeoutMultiplier).toMillis());
it('TokenTransferLoadTest', async () => {
logEvent('Starting TokenTransferLoadTest');
await main(soloRapidFire(testName, 'TokenTransferLoadTest', `-c ${clients} -a ${accounts} -T ${tokens} -A ${associations} -R -t ${duration}`, maxTps));
}).timeout(Duration.ofSeconds(duration * 2).toMillis());
it('CryptoTransferLoadTest', async () => {
logEvent('Starting CryptoTransferLoadTest');
await main(soloRapidFire(testName, 'CryptoTransferLoadTest', `-c ${clients} -a ${accounts} -R -t ${duration}`, maxTps));
}).timeout(Duration.ofSeconds(duration * 2).toMillis());
it('HCSLoadTest', async () => {
logEvent('Starting HCSLoadTest');
await main(soloRapidFire(testName, 'HCSLoadTest', `-c ${clients} -a ${accounts} -R -t ${duration}`, maxTps));
}).timeout(Duration.ofSeconds(duration * 2).toMillis());
it('SmartContractLoadTest', async () => {
logEvent('Starting SmartContractLoadTest');
await main(soloRapidFire(testName, 'SmartContractLoadTest', `-c ${clients} -a ${accounts} -R -t ${duration}`, maxTps));
}).timeout(Duration.ofSeconds(duration * 2).toMillis());
it('Should write log metrics after NLG tests have completed', async () => {
logEvent('Completed all performance tests');
if (process.env.ONE_SHOT_METRICS_TEST_DURATION_IN_MINUTES) {
const sleepTimeInMinutes = Number.parseInt(process.env.ONE_SHOT_METRICS_TEST_DURATION_IN_MINUTES, 10);
if (Number.isNaN(sleepTimeInMinutes) || sleepTimeInMinutes <= 0) {
throw new Error(`${testName}: invalid ONE_SHOT_METRICS_TEST_DURATION_IN_MINUTES value: ${process.env.ONE_SHOT_METRICS_TEST_DURATION_IN_MINUTES}`);
}
for (let index = 0; index < sleepTimeInMinutes; index++) {
console.log(`${testName}: sleeping for metrics collection, ${index + 1} of ${sleepTimeInMinutes} minutes`);
await sleep(Duration.ofMinutes(1));
}
}
await logMetrics(startTime);
}).timeout(Duration.ofMinutes(60).toMillis());
});
})
.build();
endToEndTestSuite.runTestSuite();
async function getNamespaceFromDeployment() {
const deploymentName = fs.readFileSync(PathEx.join(SOLO_CACHE_DIR, 'last-one-shot-deployment.txt'), 'utf8');
const localConfig = container.resolve(InjectTokens.LocalConfigRuntimeState);
await localConfig.load();
const deployment = localConfig.configuration.deploymentByName(deploymentName);
return deployment.namespace;
}
export async function logMetrics(startTime) {
const elapsedMilliseconds = startTime ? Date.now() - startTime.getTime() : 0;
const namespace = await getNamespaceFromDeployment();
const tartgetDirectory = PathEx.join(constants.SOLO_LOGS_DIR, `${namespace}`);
fs.mkdirSync(tartgetDirectory, { recursive: true });
await new MetricsServerImpl().logMetrics(`${testName}-${elapsedMilliseconds}`, PathEx.join(tartgetDirectory, `${elapsedMilliseconds}`), undefined, undefined, undefined, events);
// Track running peak memory and inject it into the snapshot file
const snapshotPath = PathEx.join(tartgetDirectory, `${elapsedMilliseconds}.json`);
if (fs.existsSync(snapshotPath)) {
const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf8'));
if (snapshot.memoryInMebibytes > peakMemoryInMebibytes) {
peakMemoryInMebibytes = snapshot.memoryInMebibytes;
}
snapshot.peakMemoryInMebibytes = peakMemoryInMebibytes;
fs.writeFileSync(snapshotPath, JSON.stringify(snapshot), 'utf8');
}
flushEvents();
}
export function soloOneShotDeploy(testName, deployment) {
const { newArgv, argvPushGlobalFlags, optionFromFlag } = BaseCommandTest;
const argv = newArgv();
argv.push(OneShotCommandDefinition.COMMAND_NAME, OneShotCommandDefinition.SINGLE_SUBCOMMAND_NAME, OneShotCommandDefinition.SINGLE_DEPLOY);
argvPushGlobalFlags(argv, testName);
argv.push(optionFromFlag(Flags.deployment), deployment, optionFromFlag(Flags.edgeEnabled));
return argv;
}
export function soloOneShotDestroy(testName) {
const { newArgv, argvPushGlobalFlags } = BaseCommandTest;
const argv = newArgv();
argv.push(OneShotCommandDefinition.COMMAND_NAME, OneShotCommandDefinition.SINGLE_SUBCOMMAND_NAME, OneShotCommandDefinition.SINGLE_DESTROY);
argvPushGlobalFlags(argv, testName);
return argv;
}
function logEvent(event) {
events.push(event);
}
function flushEvents() {
events = [];
}
export function soloRapidFire(testName, performanceTest, argumentsString, maxTps) {
const { newArgv, argvPushGlobalFlags, optionFromFlag } = BaseCommandTest;
const deploymentName = fs.readFileSync(PathEx.join(SOLO_CACHE_DIR, 'last-one-shot-deployment.txt'), 'utf8');
const argv = newArgv();
argv.push('rapid-fire', 'load', 'start', optionFromFlag(Flags.deployment), deploymentName, optionFromFlag(Flags.performanceTest), performanceTest, optionFromFlag(Flags.maxTps), maxTps.toString(), optionFromFlag(Flags.nlgArguments), `'"${argumentsString}"'`);
argvPushGlobalFlags(argv, testName);
return argv;
}
//# sourceMappingURL=performance.test.js.map