@starship-ci/client
Version:
Starship CI Client
707 lines (706 loc) • 25.9 kB
JavaScript
import chalk from 'chalk';
import deepmerge from 'deepmerge';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import * as yaml from 'js-yaml';
import { mkdirp } from 'mkdirp';
import * as os from 'os';
import { dirname, resolve } from 'path';
import * as shell from 'shelljs';
import { dependencies as defaultDependencies } from './deps';
import { readAndParsePackageJson } from './package';
import { verify } from './verifiers';
export const defaultStarshipContext = {
name: '',
repo: 'starship',
repoUrl: 'https://hyperweb-io.github.io/starship/',
chart: 'starship/devnet',
namespace: '',
version: '',
timeout: '10m',
restartThreshold: 3
};
const defaultName = 'starship';
const defaultVersion = 'v1.8.0';
// TODO talk to Anmol about moving these into yaml, if not already possible?
const defaultPorts = {
explorer: {
rest: 8080
},
registry: {
grpc: 9090,
rest: 8080
},
chains: {
ethereum: {
rpc: 8551,
rest: 8545,
ws: 8546
},
solana: {
rpc: 8899,
ws: 8900,
exposer: 8081,
faucet: 9900
},
defaultPorts: {
rpc: 26657,
grpc: 9090,
'grpc-web': 9091,
rest: 1317,
exposer: 8081,
faucet: 8000,
cometmock: 22331
}
},
relayers: {
defaultPorts: {
rest: 3000,
exposer: 8081
}
}
};
const defaultExecOptions = {
log: true,
silent: false,
ignoreError: true
};
export const formatChainID = (input) => {
// Replace underscores with hyphens
let formattedName = input.replace(/_/g, '-');
// Truncate the string to a maximum length of 63 characters
if (formattedName.length > 63) {
formattedName = formattedName.substring(0, 63);
}
return formattedName;
};
export class StarshipClient {
ctx;
version;
dependencies = defaultDependencies;
depsChecked = false;
config;
podPorts = defaultPorts;
podStatuses = new Map(); // To keep track of pod statuses
constructor(ctx) {
this.ctx = deepmerge(defaultStarshipContext, ctx);
// TODO add semver check against net
this.version = readAndParsePackageJson().version;
}
exec(cmd, options = {}) {
const opts = { ...defaultExecOptions, ...options };
this.checkDependencies();
const str = cmd.join(' ');
if (opts.log)
this.log(str);
const result = shell.exec(str, { silent: opts.silent });
if (result.code !== 0 && !opts.ignoreError) {
this.log(chalk.red('Error: ') + result.stderr);
this.exit(result.code);
}
return result;
}
log(str) {
// add log level
console.log(str);
}
exit(code) {
shell.exit(code);
}
checkDependencies() {
if (this.depsChecked)
return;
// so CI/CD and local dev work nicely
const platform = process.env.NODE_ENV === 'test' ? 'linux' : os.platform();
const messages = [];
const depMessages = [];
const missingDependencies = this.dependencies.filter((dep) => !dep.installed);
if (!missingDependencies.length) {
this.depsChecked = true;
return;
}
this.dependencies.forEach((dep) => {
if (missingDependencies.find((d) => d.name === dep.name)) {
depMessages.push(`${chalk.red('x')}${dep.name}`);
}
else {
depMessages.push(`${chalk.green('✓')}${dep.name}`);
}
});
messages.push('\n'); // Adding a newline for better readability
missingDependencies.forEach((dep) => {
messages.push(chalk.bold.white(dep.name + ': ') + chalk.cyan(dep.url));
if (dep.name === 'helm' && platform === 'darwin') {
messages.push(chalk.gray('Alternatively, you can install using brew: ') +
chalk.white.bold('`brew install helm`'));
}
if (dep.name === 'kubectl' && platform === 'darwin') {
messages.push(chalk.gray('Alternatively, you can install Docker for Mac which includes Kubernetes: ') + chalk.white.bold(dep.macUrl));
}
if (dep.name === 'docker' && platform === 'darwin') {
messages.push(chalk.gray('For macOS, you may also consider Docker for Mac: ') +
chalk.white.bold(dep.macUrl));
}
else if (dep.name === 'docker') {
messages.push(chalk.gray('For advanced Docker usage and installation on other platforms, see: ') + chalk.white.bold(dep.url));
}
messages.push('\n'); // Adding a newline for separation between dependencies
});
this.log(depMessages.join('\n'));
this.log('\nPlease install the missing dependencies:');
this.log(messages.join('\n'));
this.exit(1);
}
setup() {
this.setupHelm();
}
loadYaml(filename) {
const path = filename.startsWith('/')
? filename
: resolve((process.cwd(), filename));
const fileContents = readFileSync(path, 'utf8');
return yaml.load(fileContents);
}
saveYaml(filename, obj) {
const path = filename.startsWith('/')
? filename
: resolve((process.cwd(), filename));
const yamlContent = yaml.dump(obj);
mkdirp.sync(dirname(path));
writeFileSync(path, yamlContent, 'utf8');
}
loadConfig() {
this.ensureFileExists(this.ctx.config);
this.config = this.loadYaml(this.ctx.config);
this.overrideNameAndVersion();
}
saveConfig() {
this.saveYaml(this.ctx.config, this.config);
}
savePodPorts(filename) {
this.saveYaml(filename, this.podPorts);
}
loadPodPorts(filename) {
this.ensureFileExists(filename);
this.podPorts = this.loadYaml(filename);
}
setConfig(config) {
this.config = config;
this.overrideNameAndVersion();
}
setContext(ctx) {
this.ctx = ctx;
}
setPodPorts(ports) {
this.podPorts = deepmerge(defaultPorts, ports);
}
overrideNameAndVersion() {
if (!this.config) {
throw new Error('no config!');
}
// Override config name and version if provided in context
if (this.ctx.name) {
this.config.name = this.ctx.name;
}
// Use default name and version if not provided
if (!this.config.name) {
this.log(chalk.yellow('No name specified, using default name: ' + defaultName));
this.config.name = defaultName;
}
if (!this.ctx.version) {
this.log(chalk.yellow('No version specified, using default version: ' + defaultVersion));
this.ctx.version = defaultVersion;
}
this.log('config again: ' + this.config);
}
getArgs() {
const args = [];
if (this.ctx.namespace) {
args.push('--namespace', this.ctx.namespace);
}
return args;
}
getDeployArgs() {
const args = this.getArgs();
if (this.ctx.namespace) {
args.push('--create-namespace');
}
return args;
}
// TODO do we need this here?
test() {
this.exec([
'yarn',
'run',
'jest',
`--testPathPattern=../${this.ctx.repo}`,
'--verbose',
'--bail'
]);
}
async stop() {
this.stopPortForward();
this.setPodStatues(); // set pod statues before deleting the helm
this.deleteHelm();
await this.waitForPodsTermination();
}
async start() {
this.checkConnection();
this.setup();
this.deploy();
await this.waitForPods(); // Ensure waitForPods completes before starting port forwarding
this.startPortForward();
}
setupHelm() {
this.exec(['helm', 'repo', 'add', this.ctx.repo, this.ctx.repoUrl], {
ignoreError: false
});
this.exec(['helm', 'repo', 'update'], { ignoreError: false });
this.exec(['helm', 'search', 'repo', this.ctx.chart, '--version', this.ctx.version], { ignoreError: false });
}
ensureFileExists(filename) {
const path = filename.startsWith('/')
? filename
: resolve((process.cwd(), filename));
if (!existsSync(path)) {
throw new Error(`Configuration file not found: ${filename}`);
}
}
deploy(options = []) {
this.ensureFileExists(this.ctx.config);
this.log('Installing the helm chart. This is going to take a while.....');
const cmd = [
'helm',
'install',
'-f',
this.ctx.config,
this.config.name,
this.ctx.chart,
'--version',
this.ctx.version,
'--timeout',
this.ctx.timeout,
...this.getDeployArgs(),
...options
];
// Determine the data directory of the config file
const datadir = resolve(dirname(this.ctx.config));
// Iterate through each chain to add script arguments
this.config.chains.forEach((chain, chainIndex) => {
if (chain.scripts) {
Object.keys(chain.scripts).forEach((scriptKey) => {
const script = chain.scripts?.[scriptKey];
if (script && script.file) {
const scriptPath = resolve(datadir, script.file);
cmd.push(`--set-file chains[${chainIndex}].scripts.${scriptKey}.data=${scriptPath}`);
}
});
}
});
this.exec(cmd, { ignoreError: false });
this.log('Run "starship get-pods" to check the status of the cluster');
}
debug() {
this.ensureFileExists(this.ctx.config);
this.deploy(['--dry-run', '--debug']);
}
deleteHelm() {
this.exec(['helm', 'delete', this.config.name, ...this.getArgs()]);
}
getPods() {
this.exec([
'kubectl',
'get pods',
...this.getArgs()
// "--all-namespaces"
]);
}
checkConnection() {
const result = this.exec(['kubectl', 'get', 'nodes'], {
log: false,
silent: true
});
if (result.code !== 0) {
this.log(chalk.red('Error: Unable to connect to the Kubernetes cluster.'));
this.log(chalk.red('Please ensure that the Kubernetes cluster is configured correctly.'));
this.exit(1);
}
else {
this.log(chalk.green('Kubernetes cluster connection is working correctly.'));
}
}
getPodNames() {
const result = this.exec([
'kubectl',
'get',
'pods',
'--no-headers',
'-o',
'custom-columns=:metadata.name',
...this.getArgs()
], { log: false, silent: true });
// Split the output by new lines and filter out any empty lines
const podNames = result.split('\n').filter((name) => name.trim() !== '');
return podNames;
}
areAllPodsRunning() {
let allRunning = true;
this.podStatuses.forEach((status) => {
if (status.phase !== 'Running' || !status.ready) {
allRunning = false;
}
});
return allRunning;
}
checkPodStatus(podName, exitEarly = true) {
const result = this.exec([
'kubectl',
'get',
'pods',
podName,
'--no-headers',
'-o',
'custom-columns=:status.phase,:status.containerStatuses[*].ready,:status.containerStatuses[*].restartCount,:status.containerStatuses[*].state.waiting.reason',
...this.getArgs()
], { log: false, silent: true }).trim();
// Ensure the output contains valid fields to split
const parts = result.split(/\s+/);
if (parts.length < 3) {
this.log(chalk.red(`Unexpected pod status output for ${podName}: ${result}`));
return;
}
const [phase, readyList, restartCountList, reason] = parts;
// Validate readyList and restartCountList before applying split
if (!readyList || !restartCountList) {
this.log(chalk.red(`Invalid ready or restart count for pod ${podName}: ${result}`));
return;
}
const ready = readyList.split(',').every((state) => state === 'true');
const restarts = restartCountList
.split(',')
.reduce((acc, count) => acc + parseInt(count, 10), 0);
// check for repeated image pull errors
this.checkImagePullFailures(podName, exitEarly);
this.podStatuses.set(podName, {
phase,
ready,
restartCount: restarts,
reason
});
if (restarts > this.ctx.restartThreshold) {
this.log(`${chalk.red('Error:')} Pod ${podName} has restarted more than ${this.ctx.restartThreshold} times.`);
if (exitEarly)
this.exit(1);
}
}
setPodStatues() {
const podNames = this.getPodNames();
podNames.forEach((podName) => {
this.checkPodStatus(podName, false); // set exitEarly to false, only set the this.PodStatuses
});
}
async waitForPods() {
const podNames = this.getPodNames();
// Remove pods that are no longer active from the podStatuses map
this.podStatuses.forEach((_value, podName) => {
if (!podNames.includes(podName)) {
this.podStatuses.delete(podName);
}
});
// Check the status of each pod retrieved
podNames.forEach((podName) => {
this.checkPodStatus(podName);
});
this.displayPodStatuses();
if (!this.areAllPodsRunning()) {
await new Promise((resolve) => setTimeout(resolve, 2500));
await this.waitForPods(); // Recursive call
}
else {
this.log(chalk.green('All pods are running!'));
// once the pods are in running state, wait for 10 more seconds
await new Promise((resolve) => setTimeout(resolve, 5000));
}
}
async waitForPodsTermination() {
const podNames = this.getPodNames();
// Remove pods that are no longer active from the podStatuses map
this.podStatuses.forEach((_value, podName) => {
if (!podNames.includes(podName)) {
this.podStatuses.delete(podName);
}
});
if (this.podStatuses.size === 0) {
this.log(chalk.green('All pods have been successfully terminated!'));
// once the pods are in done state, wait for 1 more seconds
await new Promise((resolve) => setTimeout(resolve, 1000));
return;
}
// Check the status of each pod to terminating
podNames.forEach((podName) => {
const podStatus = this.podStatuses.get(podName);
podStatus.phase = 'Terminating';
});
this.displayPodStatuses();
await new Promise((resolve) => setTimeout(resolve, 2500));
await this.waitForPodsTermination(); // Recursive call
}
displayPodStatuses() {
console.clear();
this.podStatuses.forEach((status, podName) => {
let statusColor;
if (status.phase === 'Running' && status.ready) {
statusColor = chalk.green(status.phase);
}
else if (status.phase === 'Running' && !status.ready) {
statusColor = chalk.yellow('RunningButNotReady');
}
else if (status.phase === 'Terminating') {
statusColor = chalk.red(status.phase);
}
else {
statusColor = chalk.red(status.phase);
}
console.log(`[${chalk.blue(podName)}]: ${statusColor}, ${chalk.gray(`Ready: ${status.ready}, Restarts: ${status.restartCount}`)}`);
});
}
checkImagePullFailures(podName, exitEarly = true) {
// Fetch events from kubectl describe for the given pod
const eventLines = this.getPodEventsFromDescribe(podName);
const errorPattern = /Failed to pull image/;
const imageErrors = {};
// Parse through event lines to identify image pull failures
eventLines.forEach((line) => {
const message = line || '';
if (errorPattern.test(message)) {
const imageMatch = message.match(/image "(.*?)"/);
if (imageMatch && imageMatch[1]) {
const imageName = imageMatch[1];
imageErrors[imageName] = (imageErrors[imageName] || 0) + 1;
}
}
});
// Log errors for images that have failed more than twice
Object.entries(imageErrors).forEach(([imageName, errorCount]) => {
if (errorCount >= 3) {
this.log(`${chalk.red(`
Error: Image '${imageName}' failed to pull ${errorCount} times for pod ${podName}.
Please check the image name and ensure it is correct.
Run "starship stop" to stop the deployment which would be in stuck state.
`)}`);
if (exitEarly)
this.exit(1);
}
});
}
getPodEventsFromDescribe(podName) {
// Execute the 'kubectl describe pod' command
const result = this.exec(['kubectl', 'describe', 'pod', podName, ...this.getArgs()], { log: false, silent: true });
// Check if the command was successful
if (result.code !== 0) {
this.log(chalk.red(`Failed to describe pod ${podName}: ${result.stderr}`));
return [];
}
const describeOutput = result.stdout;
// Extract the 'Events' section from the describe output
const eventsSection = describeOutput.split('Events:')[1];
if (!eventsSection) {
this.log(chalk.yellow(`No events found for pod ${podName}`));
return [];
}
// Split the events section into individual lines
const eventLines = eventsSection
.split('\n')
.filter((line) => line.trim() !== '');
return eventLines;
}
forwardPort(chain, localPort, externalPort) {
let podName;
if (typeof chain.id === 'string') {
podName = `${formatChainID(chain.id)}-genesis-0`;
}
else {
podName = `${chain.name}-${chain.id}-0`;
}
if (localPort !== undefined && externalPort !== undefined) {
this.exec([
'kubectl',
'port-forward',
`pods/${podName}`,
`${localPort}:${externalPort}`,
...this.getArgs(),
'>',
'/dev/null',
'2>&1',
'&'
]);
this.log(chalk.yellow(`Forwarded ${podName}: local ${localPort} -> target (host) ${externalPort}`));
}
}
forwardPortCometmock(chain, localPort, externalPort) {
let podName;
if (typeof chain.id === 'string') {
podName = `${formatChainID(chain.id)}-cometmock-0`;
}
else {
podName = `${chain.name}-${chain.id}-0`;
}
if (localPort !== undefined && externalPort !== undefined) {
this.exec([
'kubectl',
'port-forward',
`pods/${podName}`,
`${localPort}:${externalPort}`,
...this.getArgs(),
'>',
'/dev/null',
'2>&1',
'&'
]);
this.log(chalk.yellow(`Forwarded ${podName}: local ${localPort} -> target (host) ${externalPort}`));
}
}
forwardPortRelayer(relayer, localPort, externalPort) {
if (localPort !== undefined && externalPort !== undefined) {
this.exec([
'kubectl',
'port-forward',
`pods/${relayer.type}-${relayer.name}-0`,
`${localPort}:${externalPort}`,
...this.getArgs(),
'>',
'/dev/null',
'2>&1',
'&'
]);
this.log(chalk.yellow(`Forwarded ${relayer.name}: local ${localPort} -> target (host) ${externalPort}`));
}
}
forwardPortService(serviceName, localPort, externalPort) {
if (localPort !== undefined && externalPort !== undefined) {
this.exec([
'kubectl',
'port-forward',
`service/${serviceName}`,
`${localPort}:${externalPort}`,
...this.getArgs(),
'>',
'/dev/null',
'2>&1',
'&'
]);
this.log(chalk.yellow(`Forwarded ${serviceName}: local ${localPort} -> target (host) ${externalPort}`));
}
}
startPortForward() {
if (!this.config) {
throw new Error('no config!');
}
this.log('Attempting to stop any existing port-forwards...');
this.stopPortForward();
this.log('Starting new port forwarding...');
this.config.chains?.forEach((chain) => {
const chainPodPorts = this.podPorts.chains[chain.name] || this.podPorts.chains.defaultPorts;
Object.entries(chain.ports || {}).forEach(([portName, portValue]) => {
if (chainPodPorts[portName]) {
if (chain.cometmock?.enabled && portName === 'rpc') {
this.forwardPortCometmock(chain, portValue, chainPodPorts.cometmock);
}
else {
this.forwardPort(chain, portValue, chainPodPorts[portName]);
}
}
});
});
this.config.relayers?.forEach((relayer) => {
const relayerPodPorts = this.podPorts.relayers[relayer.name] ||
this.podPorts.relayers.defaultPorts;
if (relayer.ports?.rest)
this.forwardPortRelayer(relayer, relayer.ports.rest, relayerPodPorts.rest);
if (relayer.ports?.exposer)
this.forwardPortRelayer(relayer, relayer.ports.exposer, relayerPodPorts.exposer);
});
if (this.config.registry?.enabled) {
this.forwardPortService('registry', this.config.registry.ports.rest, this.podPorts.registry.rest);
this.forwardPortService('registry', this.config.registry.ports.grpc, this.podPorts.registry.grpc);
}
if (this.config.explorer?.enabled) {
this.forwardPortService('explorer', this.config.explorer.ports.rest, this.podPorts.explorer.rest);
}
// Forward ports for frontend services
this.config.frontends?.forEach((frontend) => {
if (frontend.ports) {
if (frontend.ports.rest) {
this.forwardPortService(frontend.name, frontend.ports.rest, frontend.ports.rest);
}
}
});
}
getForwardPids() {
const result = this.exec([
'ps',
'-ef',
'|',
'grep',
'-i',
"'kubectl port-forward'",
'|',
'grep',
'-v',
"'grep'",
'|',
'awk',
"'{print $2}'"
], { log: false, silent: true });
const pids = (result || '')
.split('\n')
.map((pid) => pid.trim())
.filter((a) => a !== '');
return pids;
}
stopPortForward() {
this.log(chalk.green('Trying to stop all port-forward, if any....'));
const pids = this.getForwardPids();
pids.forEach((pid) => {
this.exec(['kill', '-15', pid], { log: false, silent: true });
});
this.exec(['sleep', '2'], { log: false, silent: true });
}
printForwardPids() {
const pids = this.getForwardPids();
pids.forEach((pid) => {
console.log(pid);
});
}
async verify() {
this.log(chalk.blue('Verifying services...'));
const results = await verify(this.config);
let allSuccess = true;
for (const result of results) {
const statusColor = result.status === 'success'
? 'green'
: result.status === 'skipped'
? 'yellow'
: 'red';
const status = chalk[statusColor](result.status.toUpperCase());
const message = result.message || result.error || '';
const details = result.details
? JSON.stringify(result.details, null, 2)
: '';
this.log(`${status} ${result.service} (${result.endpoint}): ${message}`);
if (details && result.status === 'failure') {
this.log(chalk.gray(`Details: ${details}`));
}
if (result.status === 'failure') {
allSuccess = false;
}
}
if (!allSuccess) {
this.log(chalk.red('\nSome services failed verification. Please check the logs above.'));
this.exit(1);
}
else {
this.log(chalk.green('\nAll services verified successfully!'));
this.exit(0);
}
}
}