@hashgraph/solo
Version:
An opinionated CLI tool to deploy and manage private Hedera Networks.
309 lines • 13.5 kB
JavaScript
// SPDX-License-Identifier: Apache-2.0
/**
* Persistently port-forward a local port to a port on a Kubernetes pod.
* This solves an issue where a detached port-forward can be terminated by network issues.
* Usage: persist-port-forward <namespace> <pod> <context> <port_map> [kubectl_executable] [kubectl_installation_dir]
* Note: <port_map> needs to be in the format <local>:<remote>.
*/
import { spawn } from 'node:child_process';
import os from 'node:os';
import path from 'node:path';
// eslint-disable-next-line unicorn/no-unreadable-array-destructuring
const [, , NAMESPACE, POD, CONTEXT, PORT_MAP, KUBECTL_EXECUTABLE, KUBECTL_INSTALLATION_DIRECTORY, EXTERNAL_ADDRESS] = process.argv;
if (!NAMESPACE || !POD || !CONTEXT || !PORT_MAP) {
console.error('Usage: persist-port-forward <namespace> <pod> <context> <port_map> [kubectl_executable] [kubectl_installation_dir]');
// eslint-disable-next-line unicorn/no-process-exit,n/no-process-exit
process.exit(2);
}
const MIN_BACKOFF = 1; // seconds
const MAX_BACKOFF = 60; // seconds
const POD_EXISTENCE_POLL_INTERVAL_SECONDS = 5;
const POD_MISSING_EXIT_THRESHOLD = 3;
const CLUSTER_UNAVAILABLE_EXIT_THRESHOLD = 3;
let backoff = MIN_BACKOFF;
let child;
let stopping = false;
let exitForMissingTarget = false;
let consecutivePodMissingChecks = 0;
let consecutiveClusterUnavailableChecks = 0;
let targetResource = POD;
function isMissingContextOrNamespaceError(message) {
const errorText = message.toLowerCase();
const missingContext = (errorText.includes('context') && errorText.includes('does not exist')) ||
(errorText.includes('no context exists with the name') && errorText.includes(CONTEXT.toLowerCase()));
const missingNamespace = (errorText.includes('namespaces') && errorText.includes('not found')) ||
(errorText.includes('namespace') && errorText.includes('does not exist'));
return missingContext || missingNamespace;
}
function isMissingPodError(message) {
const errorText = message.toLowerCase();
return (errorText.includes('notfound') ||
(errorText.includes('not found') && (errorText.includes('pods') || errorText.includes('pod'))));
}
function extractPodName(resource) {
const podPrefix = 'pods/';
if (!resource.startsWith(podPrefix)) {
return undefined;
}
return resource.slice(podPrefix.length);
}
function derivePodWorkloadPrefix(podName) {
if (!podName.includes('-')) {
return podName;
}
const deploymentStyleMatch = podName.match(/^(.*)-[a-f0-9]{9,10}-[a-z0-9]{5}$/);
if (deploymentStyleMatch?.[1]) {
return deploymentStyleMatch[1];
}
const statefulSetStyleMatch = podName.match(/^(.*)-\d+$/);
if (statefulSetStyleMatch?.[1]) {
return statefulSetStyleMatch[1];
}
return podName;
}
async function findReplacementPodResource(kubectlInstallationDirectory) {
const currentPodName = extractPodName(targetResource);
if (!currentPodName) {
return undefined;
}
const workloadPrefix = derivePodWorkloadPrefix(currentPodName);
const currentPodSegmentCount = currentPodName.split('-').length;
const runningPodsResult = await executeKubectl(['--context', CONTEXT, '-n', NAMESPACE, 'get', 'pods', '--field-selector=status.phase=Running', '-o', 'name'], kubectlInstallationDirectory, { captureOutput: true, trackAsChild: false });
if (runningPodsResult.code !== 0) {
return undefined;
}
const replacementPodName = runningPodsResult.stdout
.split('\n')
.map((line) => line.trim())
.filter((line) => line.startsWith('pod/'))
.map((line) => line.replace(/^pod\//, ''))
.find((podName) => podName !== currentPodName &&
podName.split('-').length === currentPodSegmentCount &&
(podName === workloadPrefix || podName.startsWith(`${workloadPrefix}-`)));
if (!replacementPodName) {
return undefined;
}
return `pods/${replacementPodName}`;
}
async function hasReplacementPodCandidate(kubectlInstallationDirectory) {
const currentPodName = extractPodName(targetResource);
if (!currentPodName) {
return false;
}
const workloadPrefix = derivePodWorkloadPrefix(currentPodName);
const currentPodSegmentCount = currentPodName.split('-').length;
const podsResult = await executeKubectl(['--context', CONTEXT, '-n', NAMESPACE, 'get', 'pods', '-o', 'name'], kubectlInstallationDirectory, { captureOutput: true, trackAsChild: false });
if (podsResult.code !== 0) {
return false;
}
return podsResult.stdout
.split('\n')
.map((line) => line.trim())
.filter((line) => line.startsWith('pod/'))
.map((line) => line.replace(/^pod\//, ''))
.some((podName) => podName !== currentPodName &&
podName.split('-').length === currentPodSegmentCount &&
(podName === workloadPrefix || podName.startsWith(`${workloadPrefix}-`)));
}
function isClusterUnavailableError(message) {
const errorText = message.toLowerCase();
const genericConnectionFailure = errorText.includes('unable to connect to the server') ||
errorText.includes('the connection to the server') ||
errorText.includes('server has asked for the client to provide credentials');
const dialTcpFailure = errorText.includes('dial tcp') &&
(errorText.includes('connection refused') ||
errorText.includes('no such host') ||
errorText.includes('i/o timeout'));
return genericConnectionFailure || dialTcpFailure;
}
async function executeKubectl(commandArguments, kubectlInstallationDirectory, options) {
return await new Promise((resolve) => {
const stdoutChunks = [];
const stderrChunks = [];
const kubectlCommand = KUBECTL_EXECUTABLE || 'kubectl';
const kubectlProcess = spawn(kubectlCommand, commandArguments, {
env: { ...process.env, PATH: `${kubectlInstallationDirectory}${path.delimiter}${process.env.PATH}` },
stdio: options.captureOutput ? ['ignore', 'pipe', 'pipe'] : 'inherit',
windowsHide: os.platform() === 'win32',
});
if (options.trackAsChild) {
child = kubectlProcess;
}
kubectlProcess.stdout?.on('data', (chunk) => {
stdoutChunks.push(chunk.toString());
});
kubectlProcess.stderr?.on('data', (chunk) => {
stderrChunks.push(chunk.toString());
});
kubectlProcess.on('error', (error) => {
resolve({
code: 1,
stdout: stdoutChunks.join(''),
stderr: `${stderrChunks.join('')}\n${String(error)}`,
});
});
kubectlProcess.on('close', (code, signal) => {
if (options.trackAsChild && child?.pid === kubectlProcess.pid) {
child = undefined;
}
const stderrOutput = stderrChunks.join('');
const signalMessage = signal ? `\nProcess terminated by signal: ${signal}` : '';
let exitCode = 1;
if (typeof code === 'number') {
exitCode = code;
}
else if (stopping && (signal === 'SIGTERM' || signal === 'SIGINT')) {
exitCode = 0;
}
resolve({
code: exitCode,
stdout: stdoutChunks.join(''),
stderr: `${stderrOutput}${signalMessage}`,
});
});
});
}
/**
* Check whether the original pod target still exists.
* If the pod has been removed, this persistent process should stop and not auto-restart.
*/
async function shouldExitForMissingTarget(kubectlInstallationDirectory) {
// The pod argument is the original pod reference this process was started for.
const podResult = await executeKubectl(['--context', CONTEXT, '-n', NAMESPACE, 'get', targetResource, '-o', 'name'], kubectlInstallationDirectory, { captureOutput: true, trackAsChild: false });
if (podResult.code !== 0) {
const combinedPodError = `${podResult.stderr}\n${podResult.stdout}`.trim();
if (isMissingContextOrNamespaceError(combinedPodError)) {
console.error(`Stopping persistent port-forward: original target/context is no longer available (${combinedPodError || 'unknown kubectl error'})`);
return true;
}
if (isMissingPodError(combinedPodError)) {
const replacementResource = await findReplacementPodResource(kubectlInstallationDirectory);
if (replacementResource) {
console.error(`Switching persistent port-forward target from ${targetResource} to ${replacementResource}`);
targetResource = replacementResource;
consecutivePodMissingChecks = 0;
if (child) {
try {
child.kill('SIGTERM');
}
catch {
// ignore
}
}
return false;
}
if (await hasReplacementPodCandidate(kubectlInstallationDirectory)) {
consecutivePodMissingChecks = 0;
return false;
}
consecutivePodMissingChecks += 1;
if (consecutivePodMissingChecks >= POD_MISSING_EXIT_THRESHOLD) {
console.error(`Stopping persistent port-forward: target pod appears gone after ${consecutivePodMissingChecks} checks (${combinedPodError || 'unknown kubectl error'})`);
return true;
}
return false;
}
if (isClusterUnavailableError(combinedPodError)) {
consecutiveClusterUnavailableChecks += 1;
if (consecutiveClusterUnavailableChecks >= CLUSTER_UNAVAILABLE_EXIT_THRESHOLD) {
console.error(`Stopping persistent port-forward: cluster appears unavailable after ${consecutiveClusterUnavailableChecks} checks (${combinedPodError || 'unknown kubectl error'})`);
return true;
}
return false;
}
consecutivePodMissingChecks = 0;
consecutiveClusterUnavailableChecks = 0;
return false;
}
consecutivePodMissingChecks = 0;
consecutiveClusterUnavailableChecks = 0;
return false;
}
function runKubectl(kubectlInstallationDirectory) {
const arguments_ = ['port-forward', '-n', NAMESPACE];
if (EXTERNAL_ADDRESS) {
arguments_.push('--address', EXTERNAL_ADDRESS);
}
if (CONTEXT) {
arguments_.push('--context', CONTEXT);
}
const [LOCAL, REMOTE] = PORT_MAP.split(':');
arguments_.push(targetResource, `${LOCAL}:${REMOTE}`);
console.error(`Starting kubectl ${arguments_.join(' ')}`);
return executeKubectl(arguments_, kubectlInstallationDirectory, { captureOutput: false, trackAsChild: true }).then((result) => {
if (result.code !== 0) {
console.error('Failed to start kubectl:', result.stderr || `exit code ${result.code}`);
}
return result.code;
});
}
function sleepSeconds(s) {
// eslint-disable-next-line unicorn/prevent-abbreviations
return new Promise((res) => setTimeout(res, s * 1000));
}
async function runKubectlUntilPodMissing(kubectlInstallationDirectory) {
const TICK = Symbol('tick');
const kubectlRunPromise = runKubectl(kubectlInstallationDirectory);
while (!stopping && !exitForMissingTarget) {
const result = await Promise.race([
kubectlRunPromise,
sleepSeconds(POD_EXISTENCE_POLL_INTERVAL_SECONDS).then(() => TICK),
]);
if (result !== TICK) {
return result;
}
if (await shouldExitForMissingTarget(kubectlInstallationDirectory)) {
exitForMissingTarget = true;
if (child) {
try {
child.kill('SIGTERM');
}
catch {
// ignore
}
}
break;
}
}
return await kubectlRunPromise;
}
async function main() {
const kubectlInstallationDirectory = KUBECTL_INSTALLATION_DIRECTORY || '';
while (!stopping && !exitForMissingTarget) {
if (await shouldExitForMissingTarget(kubectlInstallationDirectory)) {
exitForMissingTarget = true;
break;
}
const rc = await runKubectlUntilPodMissing(kubectlInstallationDirectory);
if (stopping || exitForMissingTarget) {
break;
}
console.error(`kubectl exited with code ${rc}, restarting in ${backoff} seconds`);
await sleepSeconds(backoff);
backoff = Math.min(backoff * 2, MAX_BACKOFF);
}
}
function shutdown(signal) {
stopping = true;
console.error(`Received ${signal}, shutting down`);
if (child) {
try {
child.kill('SIGTERM');
}
catch {
// ignore
}
}
// give processes a moment to terminate gracefully
// eslint-disable-next-line unicorn/no-process-exit,n/no-process-exit
setTimeout(() => process.exit(0), 500);
}
process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));
// eslint-disable-next-line unicorn/prefer-top-level-await
main().catch((error) => {
console.error('Unhandled error in persist-port-forward:', error);
// eslint-disable-next-line unicorn/no-process-exit,n/no-process-exit
process.exit(1);
});
//# sourceMappingURL=persist-port-forward.js.map