@underpostnet/underpost
Version:
Underpost Platform — end-to-end CI/CD and application-delivery toolchain CLI. Covers bare metal, Kubernetes, K3s, kubeadm, LXD, container/image orchestration, secrets, databases, cron jobs, monitoring, SSH, runners, PWA + Workbox delivery, and release orc
1,097 lines (995 loc) • 178 kB
JavaScript
/**
* Provides baremetal provisioning and configuration functionalities.
* @module src/cli/baremetal.js
* @namespace UnderpostBaremetal
*/
import { fileURLToPath } from 'url';
import { getNpmRootPath, getUnderpostRootPath } from '../server/conf.js';
import { pbcopy, shellExec } from '../server/process.js';
import dotenv from 'dotenv';
import { loggerFactory, loggerMiddleware } from '../server/logger.js';
import fs from 'fs-extra';
import path from 'path';
import Downloader from '../server/downloader.js';
import { newInstance, range, s4, timer } from '../client/components/core/CommonJs.js';
import { spawnSync } from 'child_process';
import Underpost from '../index.js';
import express from 'express';
const logger = loggerFactory(import.meta);
/**
* @class UnderpostBaremetal
* @description Manages baremetal provisioning and configuration tasks.
* This class provides a set of static methods to automate various
* infrastructure operations, including NFS management, control server setup,
* and system provisioning for different architectures.
*/
class UnderpostBaremetal {
// NFSv3 RPC ports. Single source of truth shared by the firewall/export setup
// (rebuildNfsServer) and the kernel `nfsroot=` mount options so the client mount and the
// opened firewall ports always agree.
// rpc.statd rejects identical listen and outgoing ports (exit 255 "Listening and outgoing ports cannot be the same!").
// statd=32765 (listen), statdOutgoing=32766 (SM_NOTIFY source port) is the standard split.
static NFS_V3_PORTS = { mountd: 20048, statd: 32765, statdOutgoing: 32766, lockd: 32803 };
// Lifecycle events POSTed by the ephemeral runtime to the bootstrap HTTP
// server, keyed by hostname. Populated by httpBootstrapServerRunnerFactory's
// POST handler and consumed by waitForBootstrapStage during orchestration.
static bootstrapStatusEvents = new Map();
static API = {
/**
* @method callback
* @description Initiates a baremetal provisioning workflow based on the provided options.
* This is the primary entry point for orchestrating baremetal operations.
* It handles NFS root filesystem building, control server installation/uninstallation,
* and system-level provisioning tasks like timezone and keyboard configuration.
* @param {string} [workflowId='rpi4mb'] - Identifier for the specific workflow configuration to use.
* @param {object} [options] - An object containing boolean flags for various operations.
* @param {string} [options.ipAddress=getLocalIPv4Address()] - The IP address of the control server or the local machine.
* @param {string} [options.hostname=workflowId] - The hostname of the target baremetal machine.
* @param {string} [options.ipFileServer=getLocalIPv4Address()] - The IP address of the file server (NFS/TFTP).
* @param {string} [options.ipConfig=''] - IP configuration string for the baremetal machine.
* @param {string} [options.netmask=''] - Netmask of network
* @param {string} [options.dnsServer=''] - DNS server IP address.
* @param {boolean} [options.dev=false] - Development mode flag.
* @param {boolean} [options.controlServerInstall=false] - Flag to install the control server (e.g., MAAS).
* @param {boolean} [options.controlServerUninstall=false] - Flag to uninstall the control server.
* @param {boolean} [options.controlServerRestart=false] - Flag to restart the control server.
* @param {boolean} [options.controlServerDbInstall=false] - Flag to install the control server's database.
* @param {boolean} [options.createMachine=false] - Flag to create a machine in MAAS.
* @param {boolean} [options.controlServerDbUninstall=false] - Flag to uninstall the control server's database.
* @param {string} [options.mac=''] - MAC address of the baremetal machine.
* @param {boolean} [options.ipxe=false] - Flag to use iPXE for booting.
* @param {boolean} [options.ipxeRebuild=false] - Flag to rebuild the iPXE binary with embedded script.
* @param {string} [options.ipxeBuildIso=''] - Builds a standalone iPXE ISO with embedded script for the specified workflow ID.
* @param {boolean} [options.installPacker=false] - Flag to install Packer CLI.
* @param {string} [options.packerMaasImageTemplate] - Template path from canonical/packer-maas to extract (requires workflow-id).
* @param {string} [options.packerWorkflowId] - Workflow ID for Packer MAAS image operations (used with --packer-maas-image-build or --packer-maas-image-upload).
* @param {boolean} [options.packerMaasImageBuild=false] - Flag to build a Packer MAAS image for the workflow specified by packerWorkflowId.
* @param {boolean} [options.packerMaasImageUpload=false] - Flag to upload a Packer MAAS image artifact without rebuilding for the workflow specified by packerWorkflowId.
* @param {boolean} [options.packerMaasImageCached=false] - Flag to use cached artifacts when building the Packer MAAS image.
* @param {string} [options.removeMachines=''] - Comma-separated list of machine system IDs or '*' to remove existing machines from MAAS before commissioning.
* @param {boolean} [options.clearDiscovered=false] - Flag to clear discovered machines from MAAS before commissioning.
* @param {boolean} [options.cloudInitUpdate=false] - Flag to update cloud-init configuration on the baremetal machine.
* @param {boolean} [options.commission=false] - Flag to commission the baremetal machine.
* @param {number} [options.bootstrapHttpServerPort=8888] - Port for the bootstrap HTTP server.
* @param {string} [options.bootstrapHttpServerPath='./public/localhost'] - Path for the bootstrap HTTP server files.
* @param {boolean} [options.bootstrapHttpServerRun=false] - Flag to start the bootstrap HTTP server.
* @param {string} [options.isoUrl=''] - Uses a custom ISO URL for baremetal machine commissioning.
* @param {boolean} [options.ubuntuToolsBuild=false] - Builds ubuntu tools for chroot environment.
* @param {boolean} [options.ubuntuToolsTest=false] - Tests ubuntu tools in chroot environment.
* @param {boolean} [options.rockyToolsBuild=false] - Builds rocky linux tools for chroot environment.
* @param {boolean} [options.rockyToolsTest=false] - Tests rocky linux tools in chroot environment.
* @param {string} [options.bootcmd=''] - Comma-separated list of boot commands to execute.
* @param {string} [options.runcmd=''] - Comma-separated list of run commands to execute.
* @param {boolean} [options.nfsBuild=false] - Flag to build the NFS root filesystem.
* @param {boolean} [options.nfsBuildServer=false] - Flag to build the NFS server components.
* @param {boolean} [options.nfsMount=false] - Flag to mount the NFS root filesystem.
* @param {boolean} [options.nfsReset=false] - Flag to reset the NFS environment by unmounting and cleaning the host path.
* @param {boolean} [options.nfsUnmount=false] - Flag to unmount the NFS root filesystem.
* @param {boolean} [options.nfsSh=false] - Flag to chroot into the NFS environment for shell access.
* @param {string} [options.logs=''] - Specifies which logs to display ('dhcp', 'cloud', 'machine', 'cloud-config').
* @param {string} [options.installDisk=''] - Specifies the disk to install the OS on (e.g., /dev/sda).
* @param {boolean} [options.autoInstall=true] - Flag to enable automatic installation of the OS on the baremetal machine.
* @param {boolean} [options.remoteInstall=true] - Flag to enable remote installation of the OS on the baremetal machine.
* @param {boolean} [options.worker=false] - Flag to designate the machine as a worker node.
* @param {string} [options.control=''] - Specifies the control node for the baremetal machine.
* @param {string} [options.sshKeyDir=''] - Specifies the directory containing SSH keys for the baremetal machine.
* @param {string} [options.deployId=''] - Specifies the deployment ID for SSH key resolution.
* @param {string} [options.engineRepo=''] - Specifies the custom engine repository URL.
* @param {string} [options.engineBranch=''] - Specifies the custom engine repository branch.
* @param {string} [options.enginePrivateRepo=''] - Specifies the custom private engine repository URL.
* @param {string} [options.enginePrivateBranch=''] - Specifies the custom private engine repository branch.
* @param {string} [options.user=''] - Specifies the SSH user for the baremetal machine.
* @param {boolean} [options.resumeInfraSetup=false] - Flag to skip commissioning and OS install, resuming SSH-based infra setup on an already installed node.
* @param {boolean} [options.resumeJoin=false] - Flag to skip everything except the kubeadm join command.
* @memberof UnderpostBaremetal
* @returns {void}
*/
async callback(
workflowId,
options = {
ipAddress: undefined,
hostname: undefined,
ipFileServer: undefined,
ipConfig: undefined,
netmask: undefined,
dnsServer: undefined,
dev: false,
controlServerInstall: false,
controlServerUninstall: false,
controlServerRestart: false,
controlServerDbInstall: false,
controlServerDbUninstall: false,
createMachine: false,
mac: '',
ipxe: false,
ipxeRebuild: false,
ipxeBuildIso: '',
installPacker: false,
packerMaasImageTemplate: false,
packerWorkflowId: '',
packerMaasImageBuild: false,
packerMaasImageUpload: false,
packerMaasImageCached: false,
removeMachines: '',
clearDiscovered: false,
cloudInitUpdate: false,
cloudInit: false,
commission: false,
bootstrapHttpServerPort: 8888,
bootstrapHttpServerPath: './public/localhost',
bootstrapHttpServerRun: false,
isoUrl: '',
ubuntuToolsBuild: false,
ubuntuToolsTest: false,
rockyToolsBuild: false,
rockyToolsTest: false,
bootcmd: '',
runcmd: '',
nfsBuild: false,
nfsBuildServer: false,
nfsMount: false,
nfsReset: false,
nfsUnmount: false,
nfsSh: false,
logs: '',
installDisk: '',
autoInstall: true,
remoteInstall: true,
worker: false,
control: '',
sshKeyDir: '',
user: '',
deployId: '',
engineRepo: '',
engineBranch: '',
enginePrivateRepo: '',
enginePrivateBranch: '',
resumeInfraSetup: false,
resumeJoin: false,
},
) {
let { ipAddress, hostname, ipFileServer, ipConfig, netmask, dnsServer } = options;
// Determine the root path for npm and underpost.
const npmRoot = getNpmRootPath();
const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
// Set default values if not provided.
workflowId = workflowId ? workflowId : 'rpi4mbarm64-iso-ram';
hostname = hostname ? hostname : workflowId;
ipAddress = ipAddress ? ipAddress : '192.168.1.191';
ipFileServer = ipFileServer ? ipFileServer : Underpost.dns.getLocalIPv4Address();
netmask = netmask ? netmask : '255.255.255.0';
dnsServer = dnsServer ? dnsServer : '8.8.8.8';
// IpConfig options:
// dhcp - DHCP configuration
// dhpc6 - DHCP IPv6 configuration
// auto6 - automatic IPv6 configuration
// on, any - any protocol available in the kernel (default)
// none, off - no autoconfiguration, static network configuration
ipConfig = ipConfig ? ipConfig : 'none';
// Set default MAC address
let macAddress = Underpost.baremetal.macAddressFactory(options).mac;
const workflowsConfig = Underpost.baremetal.loadWorkflowsConfig();
if (!workflowsConfig[workflowId]) {
throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
}
const tftpPrefix = workflowsConfig[workflowId].tftpPrefix || 'rpi4mb';
// Define the bootstrap architecture.
let bootstrapArch;
// Set bootstrap architecture.
if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
const { architecture } = workflowsConfig[workflowId].debootstrap.image;
bootstrapArch = architecture;
} else if (workflowsConfig[workflowId].type === 'chroot-container') {
const { architecture } = workflowsConfig[workflowId].container;
bootstrapArch = architecture;
}
// Define the database provider ID.
const dbProviderId = 'postgresql-17';
// Define the NFS host path based on the environment variable and hostname.
const nfsHostPath = `${process.env.NFS_EXPORT_PATH}/${hostname}`;
// Define the TFTP root prefix path based
const tftpRootPath = `${process.env.TFTP_ROOT}/${tftpPrefix}`;
// Define the iPXE cache directory to preserve builds across tftproot cleanups
const ipxeCacheDir = `/tmp/ipxe-cache/${tftpPrefix}`;
// Define the bootstrap HTTP server path.
const bootstrapHttpServerPath = options.bootstrapHttpServerPath
? options.bootstrapHttpServerPath
: `/tmp/bootstrap-http-server/${workflowId}`;
// Capture metadata for the callback execution, useful for logging and auditing.
const callbackMetaData = {
args: { workflowId, ipAddress, hostname, ipFileServer, ipConfig, netmask, dnsServer },
options,
runnerHost: { architecture: Underpost.baremetal.getHostArch().alias, ip: Underpost.dns.getLocalIPv4Address() },
nfsHostPath,
tftpRootPath,
bootstrapHttpServerPath,
};
// Log the initiation of the baremetal callback with relevant metadata.
logger.info('Baremetal callback', callbackMetaData);
// --resume-infra-setup: skip commissioning, OS install, and all PXE/TFTP/MAAS
// bootstrapping; directly resume the SSH-based infra setup on a node that
// already has the OS installed and is reachable via SSH.
if (options.resumeInfraSetup) {
const { privateKeyPath, user: resolvedUser } = Underpost.baremetal.resolveSshKeyPaths({
options,
workflowsConfig,
workflowId,
});
logger.info('--resume-infra-setup: skipping commission/bootstrapping; resuming SSH infra setup', {
hostname,
ipAddress,
keyPath: privateKeyPath,
user: resolvedUser,
workflowId,
infraSetup: workflowsConfig[workflowId]?.infraSetup || 'none',
});
return await Underpost.baremetal.postInstallDispatcher({
workflowId,
workflowsConfig,
hostname,
ipAddress,
options,
underpostRoot,
keyPath: privateKeyPath,
controlUser: resolvedUser,
});
}
// --resume-join: skip everything except the kubeadm join command.
// Even lighter than --resume-infra-setup: no engine setup, no npm install,
// no init-host, no config. Assumes all infra is already installed.
if (options.resumeJoin) {
const { privateKeyPath, user: resolvedUser } = Underpost.baremetal.resolveSshKeyPaths({
options,
workflowsConfig,
workflowId,
});
logger.info('--resume-join: skipping all bootstrapping; joining cluster directly with minimal SSH command', {
hostname,
ipAddress,
keyPath: privateKeyPath,
user: resolvedUser,
});
return await Underpost.baremetal.infraSetupKubeadm({
hostname,
ipAddress,
options,
underpostRoot,
keyPath: privateKeyPath,
controlUser: resolvedUser,
});
}
// Create a new machine in MAAS if the option is set.
let machine;
if (options.createMachine === true) {
const [searhMachine] = Underpost.baremetal.maasCliExec(`machines read hostname=${hostname}`);
if (searhMachine) {
// Check if existing machine's MAC matches the specified MAC
const existingMac = searhMachine.boot_interface?.mac_address || searhMachine.mac_address;
// If using hardware MAC (macAddress is null), skip MAC validation and use existing machine
if (macAddress === null) {
logger.info(`Using hardware MAC mode - keeping existing machine ${hostname} with MAC ${existingMac}`);
machine = searhMachine;
} else if (existingMac && existingMac !== macAddress) {
logger.warn(`⚠ Machine ${hostname} exists with MAC ${existingMac}, but --mac specified ${macAddress}`);
logger.info(`Deleting existing machine ${searhMachine.system_id} to recreate with correct MAC...`);
// Delete the existing machine
Underpost.baremetal.maasCliExec(`machine delete ${searhMachine.system_id}`);
// Create new machine with correct MAC
machine = Underpost.baremetal.machineFactory({
hostname,
ipAddress,
macAddress,
architecture: workflowsConfig[workflowId].architecture,
}).machine;
logger.info(`✓ Machine recreated with MAC ${macAddress}`);
} else {
logger.info(`Using existing machine ${hostname} with MAC ${existingMac}`);
machine = searhMachine;
}
} else {
// No existing machine found, create new one
// For hardware MAC mode (macAddress is null), we'll create machine after discovery
if (macAddress === null) {
logger.info(`Hardware MAC mode - machine will be created after discovery`);
machine = null;
} else {
machine = Underpost.baremetal.machineFactory({
hostname,
ipAddress,
macAddress,
architecture: workflowsConfig[workflowId].architecture,
}).machine;
}
}
}
if (options.ipxeBuildIso)
return await Underpost.baremetal.ipxeBuildIso({
workflowId,
isoOutputPath: options.ipxeBuildIso,
tftpPrefix,
ipFileServer,
ipAddress,
ipConfig,
netmask,
dnsServer,
macAddress,
cloudInit: options.cloudInit,
dev: options.dev,
forceRebuild: options.ipxeRebuild,
bootstrapHttpServerPort: Underpost.baremetal.bootstrapHttpServerPortFactory({
port: options.bootstrapHttpServerPort,
workflowId,
workflowsConfig,
}),
});
if (options.installPacker) {
await Underpost.baremetal.installPacker(underpostRoot);
return;
}
if (options.packerMaasImageTemplate) {
workflowId = options.packerWorkflowId;
if (!workflowId) {
throw new Error('--packer-workflow-id is required when using --packer-maas-image-template');
}
const templatePath = options.packerMaasImageTemplate;
const targetDir = `${underpostRoot}/packer/images/${workflowId}`;
logger.info(`Creating new Packer MAAS image template for workflow: ${workflowId}`);
logger.info(`Template path: ${templatePath}`);
logger.info(`Target directory: ${targetDir}`);
try {
// Use Underpost.repo to copy files from GitHub
const result = await Underpost.repo.copyGitUrlDirectoryRecursive({
gitUrl: 'https://github.com/canonical/packer-maas',
directoryPath: templatePath,
targetPath: targetDir,
branch: 'main',
overwrite: false,
});
logger.info(`\nSuccessfully copied ${result.filesCount} files`);
// Create empty workflow configuration entry
const workflowConfig = {
dir: `packer/images/${workflowId}`,
maas: {
name: `custom/${workflowId.toLowerCase()}`,
title: `${workflowId} Custom`,
architecture: 'amd64/generic',
base_image: 'ubuntu/22.04',
filetype: 'tgz',
content: `${workflowId.toLowerCase()}.tar.gz`,
},
};
const workflows = Underpost.baremetal.loadPackerMaasImageBuildWorkflows();
workflows[workflowId] = workflowConfig;
Underpost.baremetal.writePackerMaasImageBuildWorkflows(workflows);
logger.info('Template extracted successfully!');
logger.info(`Added configuration for ${workflowId} to engine/baremetal/packer-workflows.json`);
logger.info('Next steps');
logger.info(`1. Review and customize the Packer template files in: ${targetDir}`);
logger.info(`2. Review the workflow configuration in engine/baremetal/packer-workflows.json`);
logger.info(
`3. Build the image with: underpost baremetal --packer-workflow-id ${workflowId} --packer-maas-image-build`,
);
} catch (error) {
throw new Error(`Failed to extract template: ${error.message}`);
}
return;
}
if (options.packerMaasImageBuild || options.packerMaasImageUpload) {
// Use the workflow ID from --packer-workflow-id option
if (!options.packerWorkflowId) {
throw new Error('Workflow ID is required. Please specify using --packer-workflow-id <workflow-id>');
}
workflowId = options.packerWorkflowId;
const workflow = Underpost.baremetal.loadPackerMaasImageBuildWorkflows()[workflowId];
if (!workflow) {
throw new Error(`Packer MAAS image build workflow not found: ${workflowId}`);
}
const packerDir = `${underpostRoot}/${workflow.dir}`;
const tarballPath = `${packerDir}/${workflow.maas.content}`;
// Build phase (skip if upload-only mode)
if (options.packerMaasImageBuild) {
if (shellExec('packer version', { silentOnError: true }).code !== 0) {
throw new Error('Packer is not installed. Please install Packer to proceed.');
}
// Check for QEMU support if building for a different architecture (validator bots case)
Underpost.baremetal.checkQemuCrossArchSupport(workflow);
logger.info(`Building Packer image for ${workflowId} in ${packerDir}...`);
// Only remove artifacts if not using cached mode
if (!options.packerMaasImageCached) {
const artifacts = [
'output-rocky9',
'packer_cache',
'x86_64_VARS.fd',
'aarch64_VARS.fd',
workflow.maas.content,
];
shellExec(`cd packer/images/${workflowId}
rm -rf ${artifacts.join(' ')}`);
logger.info('Removed previous build artifacts');
} else {
logger.info('Cached mode: Keeping existing artifacts for incremental build');
}
shellExec(`chmod +x ${underpostRoot}/scripts/packer-init-vars-file.sh`);
shellExec(`${underpostRoot}/scripts/packer-init-vars-file.sh`);
const init = spawnSync('packer', ['init', '.'], { stdio: 'inherit', cwd: packerDir });
if (init.status !== 0) {
throw new Error('Packer init failed');
}
const isArm = process.arch === 'arm64';
// Add /usr/local/bin to PATH so Packer can find compiled QEMU binaries
const packerEnv = {
...process.env,
PACKER_LOG: '1',
PATH: `/usr/local/bin:${process.env.PATH || '/usr/bin:/bin'}`,
};
const build = spawnSync('packer', ['build', '-var', `host_is_arm=${isArm}`, '.'], {
stdio: 'inherit',
cwd: packerDir,
env: packerEnv,
});
if (build.status !== 0) {
throw new Error('Packer build failed');
}
} else {
// Upload-only mode: verify tarball exists
logger.info(`Upload-only mode: checking for existing build artifact...`);
if (!fs.existsSync(tarballPath)) {
throw new Error(
`Build artifact not found: ${tarballPath}\n` +
`Please build first with: --packer-workflow-id ${workflowId} --packer-maas-image-build`,
);
}
const stats = fs.statSync(tarballPath);
logger.info(`Found existing artifact: ${tarballPath} (${(stats.size / 1024 / 1024 / 1024).toFixed(2)} GB)`);
}
logger.info(`Uploading image to MAAS...`);
// Use the upload script to avoid MAAS CLI bugs
const uploadScript = `${underpostRoot}/scripts/maas-upload-boot-resource.sh`;
const uploadCmd = `${uploadScript} ${process.env.MAAS_ADMIN_USERNAME} "${workflow.maas.name}" "${workflow.maas.title}" "${workflow.maas.architecture}" "${workflow.maas.base_image}" "${workflow.maas.filetype}" "${tarballPath}"`;
logger.info(`Uploading to MAAS using: ${uploadScript}`);
// silentOnError: caller logs stdout/stderr structure on failure
// before throwing its own, more informative error.
const uploadResult = shellExec(uploadCmd, { silentOnError: true });
if (uploadResult.code !== 0) {
logger.error(`Upload failed with exit code: ${uploadResult.code}`);
if (uploadResult.stdout) {
logger.error(`Upload output:\n${uploadResult.stdout}`);
}
if (uploadResult.stderr) {
logger.error(`Upload error output:\n${uploadResult.stderr}`);
}
throw new Error('MAAS upload failed - see output above for details');
}
logger.info(`Successfully uploaded ${workflow.maas.name} to MAAS!`);
return;
}
// Handle various log display options.
if (options.logs === 'dhcp') {
shellExec(`journalctl -f -t dhcpd -u snap.maas.pebble.service`);
return;
}
if (options.logs === 'dhcp-lease') {
shellExec(`cat /var/snap/maas/common/maas/dhcp/dhcpd.leases`);
shellExec(`cat /var/snap/maas/common/maas/dhcp/dhcpd.pid`);
return;
}
if (options.logs === 'dhcp-lan') {
shellExec(`sudo tcpdump -l -n -i any -s0 -vv 'udp and (port 67 or 68)'`);
return;
}
if (options.logs === 'cloud-init') {
shellExec(`tail -f -n 900 ${nfsHostPath}/var/log/cloud-init.log`);
return;
}
if (options.logs === 'cloud-init-machine') {
shellExec(`tail -f -n 900 ${nfsHostPath}/var/log/cloud-init-output.log`);
return;
}
if (options.logs === 'cloud-init-config') {
shellExec(`cat ${bootstrapHttpServerPath}/${hostname}/cloud-init/user-data`);
shellExec(`cat ${bootstrapHttpServerPath}/${hostname}/cloud-init/meta-data`);
shellExec(`cat ${bootstrapHttpServerPath}/${hostname}/cloud-init/vendor-data`);
return;
}
// Handle NFS shell access option.
if (options.nfsSh === true) {
// Copy the chroot command to the clipboard for easy execution.
if (bootstrapArch && bootstrapArch !== callbackMetaData.runnerHost.architecture)
switch (bootstrapArch) {
case 'arm64':
pbcopy(`sudo chroot ${nfsHostPath} /usr/bin/qemu-aarch64-static /bin/bash`);
break;
case 'amd64':
pbcopy(`sudo chroot ${nfsHostPath} /usr/bin/qemu-x86_64-static /bin/bash`);
break;
default:
break;
}
else pbcopy(`sudo chroot ${nfsHostPath} /bin/bash`);
return; // Exit early as this is a specific interactive operation.
}
// Handle control server installation.
if (options.controlServerInstall === true) {
// Ensure the MAAS setup script is executable and then run it.
shellExec(`chmod +x ${underpostRoot}/scripts/maas-setup.sh`);
if (!fs.existsSync(`${process.env.HOME}/.ssh/id_rsa.pub`)) shellExec(`node bin ssh --generate`);
shellExec(`${underpostRoot}/scripts/maas-setup.sh`);
// Install GRUB modules into the NFS root filesystem to
// ensure the necessary files are present for bootloader installation later.
Underpost.baremetal.installGrubModules();
return;
}
// Handle control server uninstallation.
if (options.controlServerUninstall === true) {
// Stop and remove MAAS services, handling potential errors gracefully.
shellExec(`sudo snap stop maas.pebble`);
shellExec(`sudo snap stop maas`);
shellExec(`sudo snap remove maas --purge`);
// Remove residual snap data to ensure a clean uninstall.
shellExec(`sudo rm -rf /var/snap/maas`);
shellExec(`sudo rm -rf ~/snap/maas`);
// Remove MAAS configuration and data directories.
shellExec(`sudo rm -rf /etc/maas`);
shellExec(`sudo rm -rf /var/lib/maas`);
shellExec(`sudo rm -rf /var/log/maas`);
return;
}
// Handle control server restart.
if (options.controlServerRestart === true) {
shellExec(`sudo snap restart maas`);
return;
}
// Handle control server database installation.
if (options.controlServerDbInstall === true) {
// Deploy the database provider and manage MAAS database.
shellExec(`node ${underpostRoot}/bin/deploy ${dbProviderId} install`);
shellExec(`node ${underpostRoot}/bin/deploy maas-db`);
return;
}
// Handle control server database uninstallation.
if (options.controlServerDbUninstall === true) {
shellExec(`node ${underpostRoot}/bin/deploy ${dbProviderId} uninstall`);
return;
}
// Handle NFS mount operation.
if (options.nfsMount === true) {
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
mount: true,
});
return;
}
// Handle NFS unmount operation.
if (options.nfsUnmount === true) {
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
unmount: true,
});
return;
}
// Handle NFS root filesystem build operation.
if (options.nfsBuild === true) {
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
unmount: true,
});
// Clean and create the NFS host path.
shellExec(`sudo rm -rf ${nfsHostPath}/*`);
shellExec(`mkdir -p ${nfsHostPath}`);
// Perform the first stage of debootstrap.
if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
const { architecture, name } = workflowsConfig[workflowId].debootstrap.image;
shellExec(
[
`sudo debootstrap`,
`--arch=${architecture}`,
`--variant=minbase`,
`--foreign`, // Indicates a two-stage debootstrap.
name,
nfsHostPath,
`http://ports.ubuntu.com/ubuntu-ports/`,
].join(' '),
);
} else if (workflowsConfig[workflowId].type === 'chroot-container') {
const { image } = workflowsConfig[workflowId].container;
shellExec(`sudo podman pull --arch=${bootstrapArch} ${image}`);
shellExec(`sudo podman create --arch=${bootstrapArch} --name chroot-source ${image}`);
shellExec(`sudo podman export chroot-source | sudo tar -x -C ${nfsHostPath}`);
shellExec(`sudo podman rm chroot-source`);
}
// Create a podman container to extract QEMU static binaries.
shellExec(`sudo podman create --name extract docker.io/multiarch/qemu-user-static`);
shellExec(`podman ps -a`); // List all podman containers for verification.
// If cross-architecture, copy the QEMU static binary into the chroot.
if (bootstrapArch !== callbackMetaData.runnerHost.architecture)
Underpost.baremetal.crossArchBinFactory({
nfsHostPath,
bootstrapArch,
});
// Clean up the temporary podman container.
shellExec(`sudo podman rm extract`);
shellExec(`podman ps -a`);
shellExec(`file ${nfsHostPath}/bin/bash`); // Verify the bash executable in the chroot.
// Mount necessary filesystems and register binfmt for the second stage.
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
mount: true,
});
// Perform the second stage of debootstrap within the chroot environment.
if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [`/debootstrap/debootstrap --second-stage`],
});
} else if (
workflowsConfig[workflowId].type === 'chroot-container' &&
workflowsConfig[workflowId].osIdLike.match('rhel')
) {
// Copy resolv.conf to allow network access inside chroot
shellExec(`sudo cp /etc/resolv.conf ${nfsHostPath}/etc/resolv.conf`);
// Consolidate all package installations into one step to avoid redundancy
const { packages } = workflowsConfig[workflowId].container;
const basePackages = [
'findutils',
'systemd',
'sudo',
'dracut',
'dracut-network',
'dracut-config-generic',
'nfs-utils',
'file',
'binutils',
'kernel-modules-core',
'NetworkManager',
'dhclient',
'iputils',
];
const allPackages = packages && packages.length > 0 ? [...basePackages, ...packages] : basePackages;
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
`dnf install -y --allowerasing ${allPackages.join(
' ',
)} 2>/dev/null || yum install -y --allowerasing ${allPackages.join(
' ',
)} 2>/dev/null || echo "Package install completed"`,
`dnf clean all`,
`echo "=== Installed packages verification ==="`,
`rpm -qa | grep -E "dracut|kernel|nfs" | sort`,
`echo "=== Boot directory contents ==="`,
`ls -la /boot /lib/modules/*/`,
// Search for bootable kernel in order of preference:
// 1. Raw ARM64 Image file (preferred for GRUB)
// 2. vmlinuz or vmlinux (may be PE32+ on Rocky Linux)
`echo "Searching for bootable kernel..."`,
`KERNEL_FILE=""`,
// First try to find raw Image file
`if [ -f /boot/Image ]; then KERNEL_FILE=/boot/Image; echo "Found raw ARM64 Image: $KERNEL_FILE"; fi`,
`if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "Image" -o -name "Image.gz" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found kernel Image in modules: $KERNEL_FILE"; fi`,
// Fallback to vmlinuz
`if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /boot -name "vmlinuz-*" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinuz: $KERNEL_FILE"; fi`,
`if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "vmlinuz" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinuz in modules: $KERNEL_FILE"; fi`,
// Last resort: any vmlinux
`if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "vmlinux" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinux: $KERNEL_FILE"; fi`,
`if [ -z "$KERNEL_FILE" ]; then echo "ERROR: No kernel found!"; exit 1; fi`,
// Copy and check kernel type
`cp "$KERNEL_FILE" /boot/vmlinuz-efi.tmp`,
// Decompress if gzipped
`if file /boot/vmlinuz-efi.tmp | grep -q gzip; then echo "Decompressing gzipped kernel..."; gunzip -c /boot/vmlinuz-efi.tmp > /boot/vmlinuz-efi && rm /boot/vmlinuz-efi.tmp; else mv /boot/vmlinuz-efi.tmp /boot/vmlinuz-efi; fi`,
`KERNEL_TYPE=$(file /boot/vmlinuz-efi 2>/dev/null)`,
`echo "Final kernel file type: $KERNEL_TYPE"`,
// Handle PE32+ if still present - use kernel directly without extraction since iPXE can boot it
`case "$KERNEL_TYPE" in *PE32+*|*EFI*application*) echo "WARNING: Kernel is PE32+ EFI executable"; echo "GRUB may fail to boot this - recommend using iPXE chainload or installing kernel-core package"; echo "Keeping PE32+ kernel as-is for now..."; ;; *ARM64*|*aarch64*|*Image*|*data*) echo "Kernel appears to be raw ARM64 format - suitable for GRUB"; ;; *) echo "Unknown kernel format - attempting to use anyway"; ;; esac`,
// Get kernel version for initramfs rebuild
`KVER=$(basename $(dirname "$KERNEL_FILE"))`,
`echo "Kernel version: $KVER"`,
// Rebuild initramfs with NFS and network support
`echo "Rebuilding initramfs with NFS and network support..."`,
`echo "Available dracut modules:"`,
`dracut --list-modules 2>/dev/null | grep -E "network|nfs" || echo "No network modules listed"`,
// Use network-manager module (it's available in Rocky 9) for better compatibility
`dracut --force --add "nfs network base" --add-drivers "nfs sunrpc" --kver "$KVER" /boot/initrd.img "$KVER" 2>&1 || echo "Initramfs rebuild failed"`,
// Fallback: if rebuild fails, use existing initramfs
`if [ ! -f /boot/initrd.img ]; then echo "Initramfs rebuild failed, using existing..."; INITRD=$(find /boot -name "initramfs-$KVER.img" 2>/dev/null | head -n 1); if [ -z "$INITRD" ]; then INITRD=$(find /boot -name "initramfs*.img" 2>/dev/null | grep -v kdump | head -n 1); fi; if [ -n "$INITRD" ]; then cp "$INITRD" /boot/initrd.img; echo "Copied existing initramfs: $INITRD"; else echo "ERROR: No initramfs found!"; fi; fi`,
`echo "=== Final boot files ==="`,
`ls -lh /boot/vmlinuz-efi /boot/initrd.img`,
`file /boot/vmlinuz-efi`,
`file /boot/initrd.img`,
`echo "=== Setting root password ==="`,
`echo "root:root" | chpasswd`,
],
});
} else {
throw new Error(
`Unsupported workflow type for NFS build: ${workflowsConfig[workflowId].type} and like os ID ${workflowsConfig[workflowId].osIdLike}`,
);
}
}
// Fetch boot resources and machines if commissioning or listing.
let resources = Underpost.baremetal.maasCliExec(`boot-resources read`).map((o) => ({
id: o.id,
name: o.name,
architecture: o.architecture,
}));
if (options.ls === true) {
console.table(resources);
}
let machines = Underpost.baremetal.maasCliExec(`machines read`).map((m) => ({
system_id: m.interface_set[0].system_id,
mac_address: m.interface_set[0].mac_address,
hostname: m.hostname,
status_name: m.status_name,
}));
if (options.ls === true) {
console.table(machines);
}
if (options.clearDiscovered) Underpost.baremetal.removeDiscoveredMachines();
// Handle remove existing machines from MAAS.
if (options.removeMachines)
machines = Underpost.baremetal.removeMachines({
machines: options.removeMachines === 'all' ? machines : options.removeMachines.split(','),
ignore: machine ? [machine.system_id] : [],
});
if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
if (options.ubuntuToolsBuild) {
Underpost.cloudInit.buildTools({
workflowId,
nfsHostPath,
hostname,
callbackMetaData,
dev: options.dev,
});
const { chronyc, keyboard } = workflowsConfig[workflowId];
const { timezone, chronyConfPath } = chronyc;
const systemProvisioning = 'ubuntu';
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
...Underpost.system.factory[systemProvisioning].base(),
...Underpost.system.factory[systemProvisioning].user(),
...Underpost.system.factory[systemProvisioning].timezone({
timezone,
chronyConfPath,
}),
...Underpost.system.factory[systemProvisioning].keyboard(keyboard.layout),
],
});
}
if (options.ubuntuToolsTest)
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
`chmod +x /underpost/date.sh`,
`chmod +x /underpost/keyboard.sh`,
`chmod +x /underpost/dns.sh`,
`chmod +x /underpost/help.sh`,
`chmod +x /underpost/host.sh`,
`chmod +x /underpost/test.sh`,
`chmod +x /underpost/start.sh`,
`chmod +x /underpost/reset.sh`,
`chmod +x /underpost/shutdown.sh`,
`chmod +x /underpost/device_scan.sh`,
`chmod +x /underpost/mac.sh`,
`sudo chmod 700 ~/.ssh/`, // Set secure permissions for .ssh directory.
`sudo chmod 600 ~/.ssh/authorized_keys`, // Set secure permissions for authorized_keys.
`sudo chmod 644 ~/.ssh/known_hosts`, // Set permissions for known_hosts.
`sudo chmod 600 ~/.ssh/id_rsa`, // Set secure permissions for private key.
`sudo chmod 600 /etc/ssh/ssh_host_ed25519_key`, // Set secure permissions for host key.
`chown -R root:root ~/.ssh`, // Ensure root owns the .ssh directory.
`/underpost/test.sh`,
],
});
}
if (
workflowsConfig[workflowId].type === 'chroot-container' &&
workflowsConfig[workflowId].osIdLike.match('rhel')
) {
if (options.rockyToolsBuild) {
const { chronyc, keyboard } = workflowsConfig[workflowId];
const { timezone } = chronyc;
const systemProvisioning = 'rocky';
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
...Underpost.system.factory[systemProvisioning].base(),
...Underpost.system.factory[systemProvisioning].user(),
...Underpost.system.factory[systemProvisioning].timezone({
timezone,
chronyConfPath: chronyc.chronyConfPath,
}),
...Underpost.system.factory[systemProvisioning].keyboard(keyboard.layout),
],
});
}
if (options.rockyToolsTest)
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
`node --version`,
`npm --version`,
`underpost --version`,
`timedatectl status`,
`localectl status`,
`id root`,
`ls -la /home/root/.ssh/`,
`cat /home/root/.ssh/authorized_keys`,
'underpost test',
],
});
}
// Generate MAAS authentication credentials
const authCredentials =
options.commission || options.cloudInit || options.cloudInitUpdate
? Underpost.baremetal.maasAuthCredentialsFactory()
: { consumer_key: '', consumer_secret: '', token_key: '', token_secret: '' };
// Generate cloud-init configuration if needed for commissioning or cloud-init update workflows.
let cloudConfigSrc = '';
if (options.cloudInit || options.cloudInitUpdate) {
const { chronyc, networkInterfaceName } = workflowsConfig[workflowId];
const { timezone, chronyConfPath } = chronyc;
let write_files = [];
let runcmd = options.runcmd;
if (machine && options.commission) {
write_files = Underpost.baremetal.commissioningWriteFilesFactory({
machine,
authCredentials,
runnerHostIp: callbackMetaData.runnerHost.ip,
});
runcmd = '/usr/local/bin/underpost-enlist.sh';
}
cloudConfigSrc = Underpost.cloudInit.configFactory(
{
controlServerIp: callbackMetaData.runnerHost.ip,
hostname,
commissioningDeviceIp: ipAddress,
gatewayip: callbackMetaData.runnerHost.ip,
mac: macAddress,
timezone,
chronyConfPath,
networkInterfaceName,
ubuntuToolsBuild: options.ubuntuToolsBuild,
bootcmd: options.bootcmd,
runcmd,
write_files,
},
authCredentials,
).cloudConfigSrc;
}
// Rocky/RHEL Kickstart generation
let kickstartSrc = '';
if (Underpost.baremetal.getFamilyBaseOs(workflowsConfig[workflowId].osIdLike).isRhelBased) {
const bootstrapPort = Underpost.baremetal.bootstrapHttpServerPortFactory({
port: options.bootstrapHttpServerPort,
workflowId,
workflowsConfig,
});
// Base URL the ephemeral runtime POSTs lifecycle events to:
// http://<controller-ip>:<port>/<hostname> -> .../status
const bootstrapUrl = `http://${callbackMetaData.runnerHost.ip}:${bootstrapPort}/${hostname}`;
const { publicKeyPath } = Underpost.baremetal.resolveSshKeyPaths({ options, workflowsConfig, workflowId });
if (!fs.existsSync(publicKeyPath)) {
throw new Error(
`SSH public key not found at ${publicKeyPath}. Set --ssh-key-dir <dir> (expects <dir>/id_rsa.pub) or the workflow "sshKeyDir".`,
);
}
const { rootPassword, adminUsername, adminPassword, deployUsername, deployPassword } =
Underpost.baremetal.resolveInstallCredentials({ options, workflowsConfig, workflowId });
const { netIp, netPrefix, netGateway, netDns } = Underpost.baremetal.resolveInstalledNetwork({
ipAddress,
netmask,
dnsServer,
});
logger.info('Resolved installed-OS login + network', {
adminUsername,
adminPasswordSet: Boolean(adminPassword),
deployUsername: deployUsername || '(none)',
rootPasswordSet: Boolean(rootPassword),
staticIp: `${netIp}/${netPrefix}`,
gateway: netGateway,
});
kickstartSrc = Underpost.kickstart.kickstartFactory({
lang: 'en_US.UTF-8',
keyboard: workflowsConfig[workflowId].keyboard?.layout,
timezone: workflowsConfig[workflowId].chronyc?.timezone,
chronyConfPath: workflowsConfig[workflowId].chronyc?.chronyConfPath,
rootPassword,
adminUsername,
adminPassword,
deployUsername,
deployPassword,
netIp,
netPrefix,
netGateway,
netDns,
authorizedKeys: fs.readFileSync(publicKeyPath, 'utf8').trim(),
bootstrapUrl,
workflowId,
systemId: machine?.system_id || '',
targetHostname: hostname,
sshPort: 22,
installDiskHint:
(typeof options.installDisk === 'string' ? options.installDisk : '') ||
workflowsConfig[workflowId].installDisk ||
'',
autoInstall: options.autoInstall !== false,
});
}
// Build and optionally run the HTTP bootstrap server to serve cloud-init, kickstart, and ISO resources for commissioning and provisioning.
if (cloudConfigSrc || kickstartSrc || workflowsConfig[workflowId].isoUrl)
Underpost.baremetal.httpBootstrapServerStaticFactory({
bootstrapHttpServerPath,
hostname,
cloudConfigSrc,
kickstartSrc,
isoUrl: workflowsConfig[workflowId].isoUrl,
});
// Start HTTP bootstrap server if commissioning or if ISO URL is used (for ISO-based workflows).
if (options.bootstrapHttpServerRun || options.commission) {
Underpost.baremetal.httpBootstrapServerRunnerFactory({
hostname,
bootstrapHttpServerPath,
bootstrapHttpServerPort: Underpost.baremetal.bootstrapHttpServerPortFactory({
port: options.bootstrapHttpServerPort,
workflowId,
workflowsConfig,
}),
});
}
// Rebuild NFS exports and the matching MAAS/firewalld host configuration.
if (
(options.nfsBuildServer === true || options.commission === true) &&
(workflowsConfig[workflowId].type === 'iso-nfs' ||
workflowsConfig[workflowId].type === 'chroot-debootstrap' ||
workflowsConfig[workflowId].type === 'chroot-container')
)
Underpost.baremetal.rebuildNfsServer({
nfsHostPath,
nfsReset: options.nfsReset,
underpostRoot,
});
// Handle commissioning tasks
if (options.commission === true) {
let { firmwares, networkInterfaceName, maas, menuentryStr, type } = workflowsConfig[workflowId];
// Use commissioning config (Ubuntu ephemeral) for PXE boot resources
const commissioningImage = maas?.commissioning || {
architectu