underpost
Version:
Underpost Platform — end-to-end CI/CD and application-delivery toolchain CLI. Covers bare metal, Kubernetes, K3s, kubeadm, LXD, container/image orchestration, secrets, databases, cron jobs, monitoring, SSH, runners, PWA + Workbox delivery, and release orc
3,947 lines • 178 kB
JavaScript
/**
* Provides baremetal provisioning and configuration functionalities.
* @module src/cli/baremetal.js
* @namespace UnderpostBaremetal
*/
import { fileURLToPath } from 'url';
import { getNpmRootPath } from '../server/environment.js';
import { pbcopy, shellExec } from '../server/process.js';
import { loggerFactory, loggerMiddleware } from '../server/logger.js';
import fs from 'fs-extra';
import path from 'path';
import Downloader from '../server/downloader.js';
import { newInstance, range, s4, timer } from '../client/components/core/CommonJs.js';
import { spawnSync } from 'child_process';
import Underpost from '../index.js';
import express from 'express';
const logger = loggerFactory(import.meta);
/**
* @class UnderpostBaremetal
* @description Manages baremetal provisioning and configuration tasks.
* This class provides a set of static methods to automate various
* infrastructure operations, including NFS management, control server setup,
* and system provisioning for different architectures.
*/
class UnderpostBaremetal {
// NFSv3 RPC ports. Single source of truth shared by the firewall/export setup
// (rebuildNfsServer) and the kernel `nfsroot=` mount options so the client mount and the
// opened firewall ports always agree.
// rpc.statd rejects identical listen and outgoing ports (exit 255 "Listening and outgoing ports cannot be the same!").
// statd=32765 (listen), statdOutgoing=32766 (SM_NOTIFY source port) is the standard split.
static NFS_V3_PORTS = { mountd: 20048, statd: 32765, statdOutgoing: 32766, lockd: 32803 };
// Lifecycle events POSTed by the ephemeral runtime to the bootstrap HTTP
// server, keyed by hostname. Populated by httpBootstrapServerRunnerFactory's
// POST handler and consumed by waitForBootstrapStage during orchestration.
static bootstrapStatusEvents = new Map();
static API = {
/**
* @method callback
* @description Initiates a baremetal provisioning workflow based on the provided options.
* This is the primary entry point for orchestrating baremetal operations.
* It handles NFS root filesystem building, control server installation/uninstallation,
* and system-level provisioning tasks like timezone and keyboard configuration.
* @param {string} [workflowId='rpi4mb'] - Identifier for the specific workflow configuration to use.
* @param {object} [options] - An object containing boolean flags for various operations.
* @param {string} [options.ipAddress=getLocalIPv4Address()] - The IP address of the control server or the local machine.
* @param {string} [options.hostname=workflowId] - The hostname of the target baremetal machine.
* @param {string} [options.ipFileServer=getLocalIPv4Address()] - The IP address of the file server (NFS/TFTP).
* @param {string} [options.ipConfig=''] - IP configuration string for the baremetal machine.
* @param {string} [options.netmask=''] - Netmask of network
* @param {string} [options.dnsServer=''] - DNS server IP address.
* @param {boolean} [options.dev=false] - Development mode flag.
* @param {boolean} [options.controlServerInstall=false] - Flag to install the control server (e.g., MAAS).
* @param {boolean} [options.controlServerUninstall=false] - Flag to uninstall the control server.
* @param {boolean} [options.controlServerRestart=false] - Flag to restart the control server.
* @param {boolean} [options.controlServerDbInstall=false] - Flag to install the control server's database.
* @param {boolean} [options.createMachine=false] - Flag to create a machine in MAAS.
* @param {boolean} [options.controlServerDbUninstall=false] - Flag to uninstall the control server's database.
* @param {string} [options.mac=''] - MAC address of the baremetal machine.
* @param {boolean} [options.ipxe=false] - Flag to use iPXE for booting.
* @param {boolean} [options.ipxeRebuild=false] - Flag to rebuild the iPXE binary with embedded script.
* @param {string} [options.ipxeBuildIso=''] - Builds a standalone iPXE ISO with embedded script for the specified workflow ID.
* @param {boolean} [options.installPacker=false] - Flag to install Packer CLI.
* @param {string} [options.packerMaasImageTemplate] - Template path from canonical/packer-maas to extract (requires workflow-id).
* @param {string} [options.packerWorkflowId] - Workflow ID for Packer MAAS image operations (used with --packer-maas-image-build or --packer-maas-image-upload).
* @param {boolean} [options.packerMaasImageBuild=false] - Flag to build a Packer MAAS image for the workflow specified by packerWorkflowId.
* @param {boolean} [options.packerMaasImageUpload=false] - Flag to upload a Packer MAAS image artifact without rebuilding for the workflow specified by packerWorkflowId.
* @param {boolean} [options.packerMaasImageCached=false] - Flag to use cached artifacts when building the Packer MAAS image.
* @param {string} [options.removeMachines=''] - Comma-separated list of machine system IDs or '*' to remove existing machines from MAAS before commissioning.
* @param {boolean} [options.clearDiscovered=false] - Flag to clear discovered machines from MAAS before commissioning.
* @param {boolean} [options.cloudInitUpdate=false] - Flag to update cloud-init configuration on the baremetal machine.
* @param {boolean} [options.commission=false] - Flag to commission the baremetal machine.
* @param {number} [options.bootstrapHttpServerPort=8888] - Port for the bootstrap HTTP server.
* @param {string} [options.bootstrapHttpServerPath='./public/localhost'] - Path for the bootstrap HTTP server files.
* @param {boolean} [options.bootstrapHttpServerRun=false] - Flag to start the bootstrap HTTP server.
* @param {string} [options.isoUrl=''] - Uses a custom ISO URL for baremetal machine commissioning.
* @param {boolean} [options.ubuntuToolsBuild=false] - Builds ubuntu tools for chroot environment.
* @param {boolean} [options.ubuntuToolsTest=false] - Tests ubuntu tools in chroot environment.
* @param {boolean} [options.rockyToolsBuild=false] - Builds rocky linux tools for chroot environment.
* @param {boolean} [options.rockyToolsTest=false] - Tests rocky linux tools in chroot environment.
* @param {string} [options.bootcmd=''] - Comma-separated list of boot commands to execute.
* @param {string} [options.runcmd=''] - Comma-separated list of run commands to execute.
* @param {boolean} [options.nfsBuild=false] - Flag to build the NFS root filesystem.
* @param {boolean} [options.nfsBuildServer=false] - Flag to build the NFS server components.
* @param {boolean} [options.nfsMount=false] - Flag to mount the NFS root filesystem.
* @param {boolean} [options.nfsReset=false] - Flag to reset the NFS environment by unmounting and cleaning the host path.
* @param {boolean} [options.nfsUnmount=false] - Flag to unmount the NFS root filesystem.
* @param {boolean} [options.nfsSh=false] - Flag to chroot into the NFS environment for shell access.
* @param {string} [options.logs=''] - Specifies which logs to display ('dhcp', 'cloud', 'machine', 'cloud-config').
* @param {string} [options.installDisk=''] - Specifies the disk to install the OS on (e.g., /dev/sda).
* @param {boolean} [options.autoInstall=true] - Flag to enable automatic installation of the OS on the baremetal machine.
* @param {boolean} [options.remoteInstall=true] - Flag to enable remote installation of the OS on the baremetal machine.
* @param {boolean} [options.worker=false] - Flag to designate the machine as a worker node.
* @param {string} [options.control=''] - Specifies the control node for the baremetal machine.
* @param {string} [options.sshKeyDir=''] - Specifies the directory containing SSH keys for the baremetal machine.
* @param {string} [options.deployId=''] - Specifies the deployment ID for SSH key resolution.
* @param {string} [options.engineRepo=''] - Specifies the custom engine repository URL.
* @param {string} [options.engineBranch=''] - Specifies the custom engine repository branch.
* @param {string} [options.enginePrivateRepo=''] - Specifies the custom private engine repository URL.
* @param {string} [options.enginePrivateBranch=''] - Specifies the custom private engine repository branch.
* @param {string} [options.user=''] - Specifies the SSH user for the baremetal machine.
* @param {boolean} [options.resumeInfraSetup=false] - Flag to skip commissioning and OS install, resuming SSH-based infra setup on an already installed node.
* @param {boolean} [options.resumeJoin=false] - Flag to skip everything except the kubeadm join command.
* @memberof UnderpostBaremetal
* @returns {void}
*/
async callback(
workflowId,
options = {
ipAddress: undefined,
hostname: undefined,
ipFileServer: undefined,
ipConfig: undefined,
netmask: undefined,
dnsServer: undefined,
dev: false,
controlServerInstall: false,
controlServerUninstall: false,
controlServerRestart: false,
controlServerDbInstall: false,
controlServerDbUninstall: false,
createMachine: false,
mac: '',
ipxe: false,
ipxeRebuild: false,
ipxeBuildIso: '',
installPacker: false,
packerMaasImageTemplate: false,
packerWorkflowId: '',
packerMaasImageBuild: false,
packerMaasImageUpload: false,
packerMaasImageCached: false,
removeMachines: '',
clearDiscovered: false,
cloudInitUpdate: false,
cloudInit: false,
commission: false,
bootstrapHttpServerPort: 8888,
bootstrapHttpServerPath: './public/localhost',
bootstrapHttpServerRun: false,
isoUrl: '',
ubuntuToolsBuild: false,
ubuntuToolsTest: false,
rockyToolsBuild: false,
rockyToolsTest: false,
bootcmd: '',
runcmd: '',
nfsBuild: false,
nfsBuildServer: false,
nfsMount: false,
nfsReset: false,
nfsUnmount: false,
nfsSh: false,
logs: '',
installDisk: '',
autoInstall: true,
remoteInstall: true,
worker: false,
control: '',
sshKeyDir: '',
user: '',
deployId: '',
engineRepo: '',
engineBranch: '',
enginePrivateRepo: '',
enginePrivateBranch: '',
resumeInfraSetup: false,
resumeJoin: false,
},
) {
let { ipAddress, hostname, ipFileServer, ipConfig, netmask, dnsServer } = options;
// Determine the root path for npm and underpost.
const npmRoot = getNpmRootPath();
const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
// Set default values if not provided.
workflowId = workflowId ? workflowId : 'rpi4mbarm64-iso-ram';
hostname = hostname ? hostname : workflowId;
ipAddress = ipAddress ? ipAddress : '192.168.1.191';
ipFileServer = ipFileServer ? ipFileServer : Underpost.dns.getLocalIPv4Address();
netmask = netmask ? netmask : '255.255.255.0';
dnsServer = dnsServer ? dnsServer : '8.8.8.8';
// IpConfig options:
// dhcp - DHCP configuration
// dhpc6 - DHCP IPv6 configuration
// auto6 - automatic IPv6 configuration
// on, any - any protocol available in the kernel (default)
// none, off - no autoconfiguration, static network configuration
ipConfig = ipConfig ? ipConfig : 'none';
// Set default MAC address
let macAddress = Underpost.baremetal.macAddressFactory(options).mac;
const workflowsConfig = Underpost.baremetal.loadWorkflowsConfig();
if (!workflowsConfig[workflowId]) {
throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
}
const tftpPrefix = workflowsConfig[workflowId].tftpPrefix || 'rpi4mb';
// Define the bootstrap architecture.
let bootstrapArch;
// Set bootstrap architecture.
if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
const { architecture } = workflowsConfig[workflowId].debootstrap.image;
bootstrapArch = architecture;
} else if (workflowsConfig[workflowId].type === 'chroot-container') {
const { architecture } = workflowsConfig[workflowId].container;
bootstrapArch = architecture;
}
// Define the database provider ID.
const dbProviderId = 'postgresql-17';
// Define the NFS host path based on the environment variable and hostname.
const nfsHostPath = `${process.env.NFS_EXPORT_PATH}/${hostname}`;
// Define the TFTP root prefix path based
const tftpRootPath = `${process.env.TFTP_ROOT}/${tftpPrefix}`;
// Define the iPXE cache directory to preserve builds across tftproot cleanups
const ipxeCacheDir = `/tmp/ipxe-cache/${tftpPrefix}`;
// Define the bootstrap HTTP server path.
const bootstrapHttpServerPath = options.bootstrapHttpServerPath
? options.bootstrapHttpServerPath
: `/tmp/bootstrap-http-server/${workflowId}`;
// Capture metadata for the callback execution, useful for logging and auditing.
const callbackMetaData = {
args: { workflowId, ipAddress, hostname, ipFileServer, ipConfig, netmask, dnsServer },
options,
runnerHost: { architecture: Underpost.baremetal.getHostArch().alias, ip: Underpost.dns.getLocalIPv4Address() },
nfsHostPath,
tftpRootPath,
bootstrapHttpServerPath,
};
// Log the initiation of the baremetal callback with relevant metadata.
logger.info('Baremetal callback', callbackMetaData);
// --resume-infra-setup: skip commissioning, OS install, and all PXE/TFTP/MAAS
// bootstrapping; directly resume the SSH-based infra setup on a node that
// already has the OS installed and is reachable via SSH.
if (options.resumeInfraSetup) {
const { privateKeyPath, user: resolvedUser } = Underpost.baremetal.resolveSshKeyPaths({
options,
workflowsConfig,
workflowId,
});
logger.info('--resume-infra-setup: skipping commission/bootstrapping; resuming SSH infra setup', {
hostname,
ipAddress,
keyPath: privateKeyPath,
user: resolvedUser,
workflowId,
infraSetup: workflowsConfig[workflowId]?.infraSetup || 'none',
});
return await Underpost.baremetal.postInstallDispatcher({
workflowId,
workflowsConfig,
hostname,
ipAddress,
options,
underpostRoot,
keyPath: privateKeyPath,
controlUser: resolvedUser,
});
}
// --resume-join: skip everything except the kubeadm join command.
// Even lighter than --resume-infra-setup: no engine setup, no npm install,
// no init-host, no config. Assumes all infra is already installed.
if (options.resumeJoin) {
const { privateKeyPath, user: resolvedUser } = Underpost.baremetal.resolveSshKeyPaths({
options,
workflowsConfig,
workflowId,
});
logger.info('--resume-join: skipping all bootstrapping; joining cluster directly with minimal SSH command', {
hostname,
ipAddress,
keyPath: privateKeyPath,
user: resolvedUser,
});
return await Underpost.baremetal.infraSetupKubeadm({
hostname,
ipAddress,
options,
underpostRoot,
keyPath: privateKeyPath,
controlUser: resolvedUser,
});
}
// Create a new machine in MAAS if the option is set.
let machine;
if (options.createMachine === true) {
const [searhMachine] = Underpost.baremetal.maasCliExec(`machines read hostname=${hostname}`);
if (searhMachine) {
// Check if existing machine's MAC matches the specified MAC
const existingMac = searhMachine.boot_interface?.mac_address || searhMachine.mac_address;
// If using hardware MAC (macAddress is null), skip MAC validation and use existing machine
if (macAddress === null) {
logger.info(`Using hardware MAC mode - keeping existing machine ${hostname} with MAC ${existingMac}`);
machine = searhMachine;
} else if (existingMac && existingMac !== macAddress) {
logger.warn(`⚠ Machine ${hostname} exists with MAC ${existingMac}, but --mac specified ${macAddress}`);
logger.info(`Deleting existing machine ${searhMachine.system_id} to recreate with correct MAC...`);
// Delete the existing machine
Underpost.baremetal.maasCliExec(`machine delete ${searhMachine.system_id}`);
// Create new machine with correct MAC
machine = Underpost.baremetal.machineFactory({
hostname,
ipAddress,
macAddress,
architecture: workflowsConfig[workflowId].architecture,
}).machine;
logger.info(`✓ Machine recreated with MAC ${macAddress}`);
} else {
logger.info(`Using existing machine ${hostname} with MAC ${existingMac}`);
machine = searhMachine;
}
} else {
// No existing machine found, create new one
// For hardware MAC mode (macAddress is null), we'll create machine after discovery
if (macAddress === null) {
logger.info(`Hardware MAC mode - machine will be created after discovery`);
machine = null;
} else {
machine = Underpost.baremetal.machineFactory({
hostname,
ipAddress,
macAddress,
architecture: workflowsConfig[workflowId].architecture,
}).machine;
}
}
}
if (options.ipxeBuildIso)
return await Underpost.baremetal.ipxeBuildIso({
workflowId,
isoOutputPath: options.ipxeBuildIso,
tftpPrefix,
ipFileServer,
ipAddress,
ipConfig,
netmask,
dnsServer,
macAddress,
cloudInit: options.cloudInit,
dev: options.dev,
forceRebuild: options.ipxeRebuild,
bootstrapHttpServerPort: Underpost.baremetal.bootstrapHttpServerPortFactory({
port: options.bootstrapHttpServerPort,
workflowId,
workflowsConfig,
}),
});
if (options.installPacker) {
await Underpost.baremetal.installPacker(underpostRoot);
return;
}
if (options.packerMaasImageTemplate) {
workflowId = options.packerWorkflowId;
if (!workflowId) {
throw new Error('--packer-workflow-id is required when using --packer-maas-image-template');
}
const templatePath = options.packerMaasImageTemplate;
const targetDir = `${underpostRoot}/packer/images/${workflowId}`;
logger.info(`Creating new Packer MAAS image template for workflow: ${workflowId}`);
logger.info(`Template path: ${templatePath}`);
logger.info(`Target directory: ${targetDir}`);
try {
// Use Underpost.repo to copy files from GitHub
const result = await Underpost.repo.copyGitUrlDirectoryRecursive({
gitUrl: 'https://github.com/canonical/packer-maas',
directoryPath: templatePath,
targetPath: targetDir,
branch: 'main',
overwrite: false,
});
logger.info(`\nSuccessfully copied ${result.filesCount} files`);
// Create empty workflow configuration entry
const workflowConfig = {
dir: `packer/images/${workflowId}`,
maas: {
name: `custom/${workflowId.toLowerCase()}`,
title: `${workflowId} Custom`,
architecture: 'amd64/generic',
base_image: 'ubuntu/22.04',
filetype: 'tgz',
content: `${workflowId.toLowerCase()}.tar.gz`,
},
};
const workflows = Underpost.baremetal.loadPackerMaasImageBuildWorkflows();
workflows[workflowId] = workflowConfig;
Underpost.baremetal.writePackerMaasImageBuildWorkflows(workflows);
logger.info('Template extracted successfully!');
logger.info(`Added configuration for ${workflowId} to engine/baremetal/packer-workflows.json`);
logger.info('Next steps');
logger.info(`1. Review and customize the Packer template files in: ${targetDir}`);
logger.info(`2. Review the workflow configuration in engine/baremetal/packer-workflows.json`);
logger.info(
`3. Build the image with: underpost baremetal --packer-workflow-id ${workflowId} --packer-maas-image-build`,
);
} catch (error) {
throw new Error(`Failed to extract template: ${error.message}`);
}
return;
}
if (options.packerMaasImageBuild || options.packerMaasImageUpload) {
// Use the workflow ID from --packer-workflow-id option
if (!options.packerWorkflowId) {
throw new Error('Workflow ID is required. Please specify using --packer-workflow-id <workflow-id>');
}
workflowId = options.packerWorkflowId;
const workflow = Underpost.baremetal.loadPackerMaasImageBuildWorkflows()[workflowId];
if (!workflow) {
throw new Error(`Packer MAAS image build workflow not found: ${workflowId}`);
}
const packerDir = `${underpostRoot}/${workflow.dir}`;
const tarballPath = `${packerDir}/${workflow.maas.content}`;
// Build phase (skip if upload-only mode)
if (options.packerMaasImageBuild) {
if (shellExec('packer version', { silentOnError: true }).code !== 0) {
throw new Error('Packer is not installed. Please install Packer to proceed.');
}
// Check for QEMU support if building for a different architecture (validator bots case)
Underpost.baremetal.checkQemuCrossArchSupport(workflow);
logger.info(`Building Packer image for ${workflowId} in ${packerDir}...`);
// Only remove artifacts if not using cached mode
if (!options.packerMaasImageCached) {
const artifacts = [
'output-rocky9',
'packer_cache',
'x86_64_VARS.fd',
'aarch64_VARS.fd',
workflow.maas.content,
];
shellExec(`cd packer/images/${workflowId}
rm -rf ${artifacts.join(' ')}`);
logger.info('Removed previous build artifacts');
} else {
logger.info('Cached mode: Keeping existing artifacts for incremental build');
}
shellExec(`chmod +x ${underpostRoot}/scripts/packer-init-vars-file.sh`);
shellExec(`${underpostRoot}/scripts/packer-init-vars-file.sh`);
const init = spawnSync('packer', ['init', '.'], { stdio: 'inherit', cwd: packerDir });
if (init.status !== 0) {
throw new Error('Packer init failed');
}
const isArm = process.arch === 'arm64';
// Add /usr/local/bin to PATH so Packer can find compiled QEMU binaries
const packerEnv = {
...process.env,
PACKER_LOG: '1',
PATH: `/usr/local/bin:${process.env.PATH || '/usr/bin:/bin'}`,
};
const build = spawnSync('packer', ['build', '-var', `host_is_arm=${isArm}`, '.'], {
stdio: 'inherit',
cwd: packerDir,
env: packerEnv,
});
if (build.status !== 0) {
throw new Error('Packer build failed');
}
} else {
// Upload-only mode: verify tarball exists
logger.info(`Upload-only mode: checking for existing build artifact...`);
if (!fs.existsSync(tarballPath)) {
throw new Error(
`Build artifact not found: ${tarballPath}\n` +
`Please build first with: --packer-workflow-id ${workflowId} --packer-maas-image-build`,
);
}
const stats = fs.statSync(tarballPath);
logger.info(`Found existing artifact: ${tarballPath} (${(stats.size / 1024 / 1024 / 1024).toFixed(2)} GB)`);
}
logger.info(`Uploading image to MAAS...`);
// Use the upload script to avoid MAAS CLI bugs
const uploadScript = `${underpostRoot}/scripts/maas-upload-boot-resource.sh`;
const uploadCmd = `${uploadScript} ${process.env.MAAS_ADMIN_USERNAME} "${workflow.maas.name}" "${workflow.maas.title}" "${workflow.maas.architecture}" "${workflow.maas.base_image}" "${workflow.maas.filetype}" "${tarballPath}"`;
logger.info(`Uploading to MAAS using: ${uploadScript}`);
// silentOnError: caller logs stdout/stderr structure on failure
// before throwing its own, more informative error.
const uploadResult = shellExec(uploadCmd, { silentOnError: true });
if (uploadResult.code !== 0) {
logger.error(`Upload failed with exit code: ${uploadResult.code}`);
if (uploadResult.stdout) {
logger.error(`Upload output:\n${uploadResult.stdout}`);
}
if (uploadResult.stderr) {
logger.error(`Upload error output:\n${uploadResult.stderr}`);
}
throw new Error('MAAS upload failed - see output above for details');
}
logger.info(`Successfully uploaded ${workflow.maas.name} to MAAS!`);
return;
}
// Handle various log display options.
if (options.logs === 'dhcp') {
shellExec(`journalctl -f -t dhcpd -u snap.maas.pebble.service`);
return;
}
if (options.logs === 'dhcp-lease') {
shellExec(`cat /var/snap/maas/common/maas/dhcp/dhcpd.leases`);
shellExec(`cat /var/snap/maas/common/maas/dhcp/dhcpd.pid`);
return;
}
if (options.logs === 'dhcp-lan') {
shellExec(`sudo tcpdump -l -n -i any -s0 -vv 'udp and (port 67 or 68)'`);
return;
}
if (options.logs === 'cloud-init') {
shellExec(`tail -f -n 900 ${nfsHostPath}/var/log/cloud-init.log`);
return;
}
if (options.logs === 'cloud-init-machine') {
shellExec(`tail -f -n 900 ${nfsHostPath}/var/log/cloud-init-output.log`);
return;
}
if (options.logs === 'cloud-init-config') {
shellExec(`cat ${bootstrapHttpServerPath}/${hostname}/cloud-init/user-data`);
shellExec(`cat ${bootstrapHttpServerPath}/${hostname}/cloud-init/meta-data`);
shellExec(`cat ${bootstrapHttpServerPath}/${hostname}/cloud-init/vendor-data`);
return;
}
// Handle NFS shell access option.
if (options.nfsSh === true) {
// Copy the chroot command to the clipboard for easy execution.
if (bootstrapArch && bootstrapArch !== callbackMetaData.runnerHost.architecture)
switch (bootstrapArch) {
case 'arm64':
pbcopy(`sudo chroot ${nfsHostPath} /usr/bin/qemu-aarch64-static /bin/bash`);
break;
case 'amd64':
pbcopy(`sudo chroot ${nfsHostPath} /usr/bin/qemu-x86_64-static /bin/bash`);
break;
default:
break;
}
else pbcopy(`sudo chroot ${nfsHostPath} /bin/bash`);
return; // Exit early as this is a specific interactive operation.
}
// Handle control server installation.
if (options.controlServerInstall === true) {
// Ensure the MAAS setup script is executable and then run it.
shellExec(`chmod +x ${underpostRoot}/scripts/maas-setup.sh`);
if (!fs.existsSync(`${process.env.HOME}/.ssh/id_rsa.pub`)) shellExec(`node bin ssh --generate`);
shellExec(`${underpostRoot}/scripts/maas-setup.sh`);
// Install GRUB modules into the NFS root filesystem to
// ensure the necessary files are present for bootloader installation later.
Underpost.baremetal.installGrubModules();
return;
}
// Handle control server uninstallation.
if (options.controlServerUninstall === true) {
// Stop and remove MAAS services, handling potential errors gracefully.
shellExec(`sudo snap stop maas.pebble`);
shellExec(`sudo snap stop maas`);
shellExec(`sudo snap remove maas --purge`);
// Remove residual snap data to ensure a clean uninstall.
shellExec(`sudo rm -rf /var/snap/maas`);
shellExec(`sudo rm -rf ~/snap/maas`);
// Remove MAAS configuration and data directories.
shellExec(`sudo rm -rf /etc/maas`);
shellExec(`sudo rm -rf /var/lib/maas`);
shellExec(`sudo rm -rf /var/log/maas`);
return;
}
// Handle control server restart.
if (options.controlServerRestart === true) {
shellExec(`sudo snap restart maas`);
return;
}
// Handle control server database installation.
if (options.controlServerDbInstall === true) {
// Deploy the database provider and manage MAAS database.
shellExec(`node ${underpostRoot}/bin/deploy ${dbProviderId} install`);
shellExec(`node ${underpostRoot}/bin/deploy maas-db`);
return;
}
// Handle control server database uninstallation.
if (options.controlServerDbUninstall === true) {
shellExec(`node ${underpostRoot}/bin/deploy ${dbProviderId} uninstall`);
return;
}
// Handle NFS mount operation.
if (options.nfsMount === true) {
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
mount: true,
});
return;
}
// Handle NFS unmount operation.
if (options.nfsUnmount === true) {
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
unmount: true,
});
return;
}
// Handle NFS root filesystem build operation.
if (options.nfsBuild === true) {
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
unmount: true,
});
// Clean and create the NFS host path.
shellExec(`sudo rm -rf ${nfsHostPath}/*`);
shellExec(`mkdir -p ${nfsHostPath}`);
// Perform the first stage of debootstrap.
if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
const { architecture, name } = workflowsConfig[workflowId].debootstrap.image;
shellExec(
[
`sudo debootstrap`,
`--arch=${architecture}`,
`--variant=minbase`,
`--foreign`, // Indicates a two-stage debootstrap.
name,
nfsHostPath,
`http://ports.ubuntu.com/ubuntu-ports/`,
].join(' '),
);
} else if (workflowsConfig[workflowId].type === 'chroot-container') {
const { image } = workflowsConfig[workflowId].container;
shellExec(`sudo podman pull --arch=${bootstrapArch} ${image}`);
shellExec(`sudo podman create --arch=${bootstrapArch} --name chroot-source ${image}`);
shellExec(`sudo podman export chroot-source | sudo tar -x -C ${nfsHostPath}`);
shellExec(`sudo podman rm chroot-source`);
}
// Create a podman container to extract QEMU static binaries.
shellExec(`sudo podman create --name extract docker.io/multiarch/qemu-user-static`);
shellExec(`podman ps -a`); // List all podman containers for verification.
// If cross-architecture, copy the QEMU static binary into the chroot.
if (bootstrapArch !== callbackMetaData.runnerHost.architecture)
Underpost.baremetal.crossArchBinFactory({
nfsHostPath,
bootstrapArch,
});
// Clean up the temporary podman container.
shellExec(`sudo podman rm extract`);
shellExec(`podman ps -a`);
shellExec(`file ${nfsHostPath}/bin/bash`); // Verify the bash executable in the chroot.
// Mount necessary filesystems and register binfmt for the second stage.
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
mount: true,
});
// Perform the second stage of debootstrap within the chroot environment.
if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [`/debootstrap/debootstrap --second-stage`],
});
} else if (
workflowsConfig[workflowId].type === 'chroot-container' &&
workflowsConfig[workflowId].osIdLike.match('rhel')
) {
// Copy resolv.conf to allow network access inside chroot
shellExec(`sudo cp /etc/resolv.conf ${nfsHostPath}/etc/resolv.conf`);
// Consolidate all package installations into one step to avoid redundancy
const { packages } = workflowsConfig[workflowId].container;
const basePackages = [
'findutils',
'systemd',
'sudo',
'dracut',
'dracut-network',
'dracut-config-generic',
'nfs-utils',
'file',
'binutils',
'kernel-modules-core',
'NetworkManager',
'dhclient',
'iputils',
];
const allPackages = packages && packages.length > 0 ? [...basePackages, ...packages] : basePackages;
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
`dnf install -y --allowerasing ${allPackages.join(
' ',
)} 2>/dev/null || yum install -y --allowerasing ${allPackages.join(
' ',
)} 2>/dev/null || echo "Package install completed"`,
`dnf clean all`,
`echo "=== Installed packages verification ==="`,
`rpm -qa | grep -E "dracut|kernel|nfs" | sort`,
`echo "=== Boot directory contents ==="`,
`ls -la /boot /lib/modules/*/`,
// Search for bootable kernel in order of preference:
// 1. Raw ARM64 Image file (preferred for GRUB)
// 2. vmlinuz or vmlinux (may be PE32+ on Rocky Linux)
`echo "Searching for bootable kernel..."`,
`KERNEL_FILE=""`,
// First try to find raw Image file
`if [ -f /boot/Image ]; then KERNEL_FILE=/boot/Image; echo "Found raw ARM64 Image: $KERNEL_FILE"; fi`,
`if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "Image" -o -name "Image.gz" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found kernel Image in modules: $KERNEL_FILE"; fi`,
// Fallback to vmlinuz
`if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /boot -name "vmlinuz-*" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinuz: $KERNEL_FILE"; fi`,
`if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "vmlinuz" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinuz in modules: $KERNEL_FILE"; fi`,
// Last resort: any vmlinux
`if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "vmlinux" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinux: $KERNEL_FILE"; fi`,
`if [ -z "$KERNEL_FILE" ]; then echo "ERROR: No kernel found!"; exit 1; fi`,
// Copy and check kernel type
`cp "$KERNEL_FILE" /boot/vmlinuz-efi.tmp`,
// Decompress if gzipped
`if file /boot/vmlinuz-efi.tmp | grep -q gzip; then echo "Decompressing gzipped kernel..."; gunzip -c /boot/vmlinuz-efi.tmp > /boot/vmlinuz-efi && rm /boot/vmlinuz-efi.tmp; else mv /boot/vmlinuz-efi.tmp /boot/vmlinuz-efi; fi`,
`KERNEL_TYPE=$(file /boot/vmlinuz-efi 2>/dev/null)`,
`echo "Final kernel file type: $KERNEL_TYPE"`,
// Handle PE32+ if still present - use kernel directly without extraction since iPXE can boot it
`case "$KERNEL_TYPE" in *PE32+*|*EFI*application*) echo "WARNING: Kernel is PE32+ EFI executable"; echo "GRUB may fail to boot this - recommend using iPXE chainload or installing kernel-core package"; echo "Keeping PE32+ kernel as-is for now..."; ;; *ARM64*|*aarch64*|*Image*|*data*) echo "Kernel appears to be raw ARM64 format - suitable for GRUB"; ;; *) echo "Unknown kernel format - attempting to use anyway"; ;; esac`,
// Get kernel version for initramfs rebuild
`KVER=$(basename $(dirname "$KERNEL_FILE"))`,
`echo "Kernel version: $KVER"`,
// Rebuild initramfs with NFS and network support
`echo "Rebuilding initramfs with NFS and network support..."`,
`echo "Available dracut modules:"`,
`dracut --list-modules 2>/dev/null | grep -E "network|nfs" || echo "No network modules listed"`,
// Use network-manager module (it's available in Rocky 9) for better compatibility
`dracut --force --add "nfs network base" --add-drivers "nfs sunrpc" --kver "$KVER" /boot/initrd.img "$KVER" 2>&1 || echo "Initramfs rebuild failed"`,
// Fallback: if rebuild fails, use existing initramfs
`if [ ! -f /boot/initrd.img ]; then echo "Initramfs rebuild failed, using existing..."; INITRD=$(find /boot -name "initramfs-$KVER.img" 2>/dev/null | head -n 1); if [ -z "$INITRD" ]; then INITRD=$(find /boot -name "initramfs*.img" 2>/dev/null | grep -v kdump | head -n 1); fi; if [ -n "$INITRD" ]; then cp "$INITRD" /boot/initrd.img; echo "Copied existing initramfs: $INITRD"; else echo "ERROR: No initramfs found!"; fi; fi`,
`echo "=== Final boot files ==="`,
`ls -lh /boot/vmlinuz-efi /boot/initrd.img`,
`file /boot/vmlinuz-efi`,
`file /boot/initrd.img`,
`echo "=== Setting root password ==="`,
`echo "root:root" | chpasswd`,
],
});
} else {
throw new Error(
`Unsupported workflow type for NFS build: ${workflowsConfig[workflowId].type} and like os ID ${workflowsConfig[workflowId].osIdLike}`,
);
}
}
// Fetch boot resources and machines if commissioning or listing.
let resources = Underpost.baremetal.maasCliExec(`boot-resources read`).map((o) => ({
id: o.id,
name: o.name,
architecture: o.architecture,
}));
if (options.ls === true) {
console.table(resources);
}
let machines = Underpost.baremetal.maasCliExec(`machines read`).map((m) => ({
system_id: m.interface_set[0].system_id,
mac_address: m.interface_set[0].mac_address,
hostname: m.hostname,
status_name: m.status_name,
}));
if (options.ls === true) {
console.table(machines);
}
if (options.clearDiscovered) Underpost.baremetal.removeDiscoveredMachines();
// Handle remove existing machines from MAAS.
if (options.removeMachines)
machines = Underpost.baremetal.removeMachines({
machines: options.removeMachines === 'all' ? machines : options.removeMachines.split(','),
ignore: machine ? [machine.system_id] : [],
});
if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
if (options.ubuntuToolsBuild) {
Underpost.cloudInit.buildTools({
workflowId,
nfsHostPath,
hostname,
callbackMetaData,
dev: options.dev,
});
const { chronyc, keyboard } = workflowsConfig[workflowId];
const { timezone, chronyConfPath } = chronyc;
const systemProvisioning = 'ubuntu';
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
...Underpost.system.factory[systemProvisioning].base(),
...Underpost.system.factory[systemProvisioning].user(),
...Underpost.system.factory[systemProvisioning].timezone({
timezone,
chronyConfPath,
}),
...Underpost.system.factory[systemProvisioning].keyboard(keyboard.layout),
],
});
}
if (options.ubuntuToolsTest)
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
`chmod +x /underpost/date.sh`,
`chmod +x /underpost/keyboard.sh`,
`chmod +x /underpost/dns.sh`,
`chmod +x /underpost/help.sh`,
`chmod +x /underpost/host.sh`,
`chmod +x /underpost/test.sh`,
`chmod +x /underpost/start.sh`,
`chmod +x /underpost/reset.sh`,
`chmod +x /underpost/shutdown.sh`,
`chmod +x /underpost/device_scan.sh`,
`chmod +x /underpost/mac.sh`,
`sudo chmod 700 ~/.ssh/`, // Set secure permissions for .ssh directory.
`sudo chmod 600 ~/.ssh/authorized_keys`, // Set secure permissions for authorized_keys.
`sudo chmod 644 ~/.ssh/known_hosts`, // Set permissions for known_hosts.
`sudo chmod 600 ~/.ssh/id_rsa`, // Set secure permissions for private key.
`sudo chmod 600 /etc/ssh/ssh_host_ed25519_key`, // Set secure permissions for host key.
`chown -R root:root ~/.ssh`, // Ensure root owns the .ssh directory.
`/underpost/test.sh`,
],
});
}
if (
workflowsConfig[workflowId].type === 'chroot-container' &&
workflowsConfig[workflowId].osIdLike.match('rhel')
) {
if (options.rockyToolsBuild) {
const { chronyc, keyboard } = workflowsConfig[workflowId];
const { timezone } = chronyc;
const systemProvisioning = 'rocky';
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
...Underpost.system.factory[systemProvisioning].base(),
...Underpost.system.factory[systemProvisioning].user(),
...Underpost.system.factory[systemProvisioning].timezone({
timezone,
chronyConfPath: chronyc.chronyConfPath,
}),
...Underpost.system.factory[systemProvisioning].keyboard(keyboard.layout),
],
});
}
if (options.rockyToolsTest)
Underpost.baremetal.crossArchRunner({
nfsHostPath,
bootstrapArch,
callbackMetaData,
steps: [
`node --version`,
`npm --version`,
`underpost --version`,
`timedatectl status`,
`localectl status`,
`id root`,
`ls -la /home/root/.ssh/`,
`cat /home/root/.ssh/authorized_keys`,
'underpost test',
],
});
}
// Generate MAAS authentication credentials
const authCredentials =
options.commission || options.cloudInit || options.cloudInitUpdate
? Underpost.baremetal.maasAuthCredentialsFactory()
: { consumer_key: '', consumer_secret: '', token_key: '', token_secret: '' };
// Generate cloud-init configuration if needed for commissioning or cloud-init update workflows.
let cloudConfigSrc = '';
if (options.cloudInit || options.cloudInitUpdate) {
const { chronyc, networkInterfaceName } = workflowsConfig[workflowId];
const { timezone, chronyConfPath } = chronyc;
let write_files = [];
let runcmd = options.runcmd;
if (machine && options.commission) {
write_files = Underpost.baremetal.commissioningWriteFilesFactory({
machine,
authCredentials,
runnerHostIp: callbackMetaData.runnerHost.ip,
});
runcmd = '/usr/local/bin/underpost-enlist.sh';
}
cloudConfigSrc = Underpost.cloudInit.configFactory(
{
controlServerIp: callbackMetaData.runnerHost.ip,
hostname,
commissioningDeviceIp: ipAddress,
gatewayip: callbackMetaData.runnerHost.ip,
mac: macAddress,
timezone,
chronyConfPath,
networkInterfaceName,
ubuntuToolsBuild: options.ubuntuToolsBuild,
bootcmd: options.bootcmd,
runcmd,
write_files,
},
authCredentials,
).cloudConfigSrc;
}
// Rocky/RHEL Kickstart generation
let kickstartSrc = '';
if (Underpost.baremetal.getFamilyBaseOs(workflowsConfig[workflowId].osIdLike).isRhelBased) {
const bootstrapPort = Underpost.baremetal.bootstrapHttpServerPortFactory({
port: options.bootstrapHttpServerPort,
workflowId,
workflowsConfig,
});
// Base URL the ephemeral runtime POSTs lifecycle events to:
// http://<controller-ip>:<port>/<hostname> -> .../status
const bootstrapUrl = `http://${callbackMetaData.runnerHost.ip}:${bootstrapPort}/${hostname}`;
const { publicKeyPath } = Underpost.baremetal.resolveSshKeyPaths({ options, workflowsConfig, workflowId });
if (!fs.existsSync(publicKeyPath)) {
throw new Error(
`SSH public key not found at ${publicKeyPath}. Set --ssh-key-dir <dir> (expects <dir>/id_rsa.pub) or the workflow "sshKeyDir".`,
);
}
const { rootPassword, adminUsername, adminPassword, deployUsername, deployPassword } =
Underpost.baremetal.resolveInstallCredentials({ options, workflowsConfig, workflowId });
const { netIp, netPrefix, netGateway, netDns } = Underpost.baremetal.resolveInstalledNetwork({
ipAddress,
netmask,
dnsServer,
});
logger.info('Resolved installed-OS login + network', {
adminUsername,
adminPasswordSet: Boolean(adminPassword),
deployUsername: deployUsername || '(none)',
rootPasswordSet: Boolean(rootPassword),
staticIp: `${netIp}/${netPrefix}`,
gateway: netGateway,
});
kickstartSrc = Underpost.kickstart.kickstartFactory({
lang: 'en_US.UTF-8',
keyboard: workflowsConfig[workflowId].keyboard?.layout,
timezone: workflowsConfig[workflowId].chronyc?.timezone,
chronyConfPath: workflowsConfig[workflowId].chronyc?.chronyConfPath,
rootPassword,
adminUsername,
adminPassword,
deployUsername,
deployPassword,
netIp,
netPrefix,
netGateway,
netDns,
authorizedKeys: fs.readFileSync(publicKeyPath, 'utf8').trim(),
bootstrapUrl,
workflowId,
systemId: machine?.system_id || '',
targetHostname: hostname,
sshPort: 22,
installDiskHint:
(typeof options.installDisk === 'string' ? options.installDisk : '') ||
workflowsConfig[workflowId].installDisk ||
'',
autoInstall: options.autoInstall !== false,
});
}
// Build and optionally run the HTTP bootstrap server to serve cloud-init, kickstart, and ISO resources for commissioning and provisioning.
if (cloudConfigSrc || kickstartSrc || workflowsConfig[workflowId].isoUrl)
Underpost.baremetal.httpBootstrapServerStaticFactory({
bootstrapHttpServerPath,
hostname,
cloudConfigSrc,
kickstartSrc,
isoUrl: workflowsConfig[workflowId].isoUrl,
});
// Start HTTP bootstrap server if commissioning or if ISO URL is used (for ISO-based workflows).
if (options.bootstrapHttpServerRun || options.commission) {
Underpost.baremetal.httpBootstrapServerRunnerFactory({
hostname,
bootstrapHttpServerPath,
bootstrapHttpServerPort: Underpost.baremetal.bootstrapHttpServerPortFactory({
port: options.bootstrapHttpServerPort,
workflowId,
workflowsConfig,
}),
});
}
// Rebuild NFS exports and the matching MAAS/firewalld host configuration.
if (
(options.nfsBuildServer === true || options.commission === true) &&
(workflowsConfig[workflowId].type === 'iso-nfs' ||
workflowsConfig[workflowId].type === 'chroot-debootstrap' ||
workflowsConfig[workflowId].type === 'chroot-container')
)
Underpost.baremetal.rebuildNfsServer({
nfsHostPath,
nfsReset: options.nfsReset,
underpostRoot,
});
// Handle commissioning tasks
if (options.commission === true) {
let { firmwares, networkInterfaceName, maas, menuentryStr, type } = workflowsConfig[workflowId];
// Use commissioning config (Ubuntu ephemeral) for PXE boot resources
const commissioningImage = maas?.commissioning || {
architecture: 'arm64/generic',
name: 'ubuntu/noble',
};
const resource = resources.find(
(o) => o.architecture === commissioningImage.architecture && o.name === commissioningImage.name,
);
logger.info('Commissioning resource', resource);
if (
Underpost.baremetal.getFamilyBaseOs(workflowsConfig[workflowId].osIdLike).isDebianBased &&
(type === 'iso-nfs' || type === 'chroot-debootstrap' || type === 'chroot-container')
) {
// Prepare NFS casper path if using NFS boot.
shellExec(`mkdir -p ${nfsHostPath}/casper`);
}
// Clean and create TFTP root path. The iPXE EFI binary is restored from
// cache (or rebuilt) later by the gated build block, so no early copy here.
shellExec(`sudo rm -rf ${tftpRootPath}`);
shellExec(`mkdir -p ${tftpRootPath}/pxe`);
// Process firmwares for TFTP.
for (const firmware of firmwares) {
const { url, gateway, subnet } = firmware;
if (url.match('.zip')) {
const name = url.split('/').pop().replace('.zip', '');
const path = `../${name}`;
if (!fs.existsSync(path)) {
await Downloader.downloadFile(url, `../${name}.zip`); // Download firmware if not exists.
shellExec(`cd .. && mkdir ${name} && cd ${name} && unzip ../${name}.zip`); // Unzip firmware.
}
shellExec(`sudo cp -a ${path}/* ${tftpRootPath}`); // Copy firmware files to TFTP root.
if (gateway && subnet) {
const bootConfSrc = Underpost.baremetal.bootConfFactory({
workflowId,
tftpIp: callbackMetaData.runnerHost.ip,
tftpPrefixStr: tftpPrefix,
macAddress,
clientIp: ipAddress,
subnet,
gateway,
});
if (bootConfSrc) fs.writeFileSync(`${tftpRootPath}/boot_${name}.conf`, bootConfSrc, 'utf8');
}
}
}
// Configure GRUB for PXE boot.
{
// Fetch kernel and initrd paths from MAAS boot resource.
// Both NFS and disk-based commissioning use MAAS boot resources.
let kernelFilesPaths, resourcesPath;
if (workflowsConfig[workflowId].type === 'chroot-container') {
const arch = commissioningImage.architecture.split('/')[0];
resourcesPath = `/var/snap/maas/common/maas/image-storage/bootloaders/uefi/${arch}`;
kernelFilesPaths = {
'vmlinuz-efi': `${nfsHostPath}/boot/vmlinuz-efi`,
'initrd.img': `${nfsHostPath}/boot/initrd.img`,
};
} else {
const kf = Underpost.baremetal.kernelFactory({
resource,
type,
nfsHostPath,
isoUrl: options.isoUrl || workflowsConfig[workflowId].isoUrl,
workflowId,
});
kernelFilesPaths = kf.kernelFilesPaths;
resourcesPath = kf.resourcesPath;
}
const { cmd } = Underpost.baremetal.kernelCmdBootParamsFactory({
ipClient: ipAddress,
ipDhcpServer: callbackMetaData.runnerHost.ip,
ipConfig,
ipFileServer,
netmask,
hostname,
dnsServer,
networkInterfaceName,
fileSystemUrl:
type === 'iso-ram'
? `http://${callbackMetaData.runnerHost.ip}:${Underpost.baremetal.bootstrapHttpServerPortFactory({
port: options.bootstrapHttpServerPort,
workflowId,
workflowsConfig,
})}/${hostname}/${kernelFilesPaths.isoUrl.split('/').pop()}`
: kernelFilesPaths.isoUrl,
bootstrapHttpServerPort: Underpost.baremetal.bootstrapHttpServerPortFactory({
port: options.bootstrapHttpServerPort,
workflowId,
workflowsConfig,
}),
type,
macAddress,
cloudInit: options.cloudInit,
dev: options.dev,
osIdLike: workflowsConfig[workflowId].osIdLike || '',
authCredentials,
architecture: workflowsConfig[workflowId].architecture,
});
// Check if iPXE mode is enabled AND the iPXE EFI binary exists
let useIpxe = options.ipxe;
if (options.ipxe) {
const arch = commissioningImage.architecture.split('/')[0];
const ipxeScript = Underpost.baremetal.ipxeScriptFactory({
maasIp: callbackMetaData.runnerHost.ip,
macAddress,
architecture: arch,
tftpPrefix,
kernelCmd: cmd,
});
fs.writeFileSync(`${tftpRootPath}/stable-id.ipxe`, ipxeScript, 'utf8');
// Create embedded boot script that does DHCP and chains to the main script
const embeddedScript = Underpost.baremetal.ipxeEmbeddedScriptFactory({
tftpServer: callbackMetaData.runnerHost.ip,
scriptPath: `/${tftpPrefix}/stable-id.ipxe`,
macAddress: macAddress,
});
fs.writeFileSync(`${tftpRootPath}/boot.ipxe`, embeddedScript, 'utf8');
logger.info('✓ iPXE script generated for MAAS commissioning', {
registeredMAC: macAddress,
path: `${tftpRootPath}/stable-id.ipxe`,
embeddedPath: `${tftpRootPath}/boot.ipxe`,
});
// iPXE EFI binary build policy:
// --ipxe-rebuild -> always rebuild.
// --ipxe (cache exists) -> reuse the cached binary, no build.
// --ipxe (no cache) -> warn and build, then cache for reuse.
const cachedIpxePath = `${ipxeCacheDir}/ipxe.efi`;
const tftpIpxePath = `${tftpRootPath}/ipxe.efi`;
if (options.ipxeRebuild || !fs.existsSync(cachedIpxePath)) {
if (!options.ipxeRebuild) {
logger.warn(
'⚠ No cached iPXE EFI binary found — building now. Later runs reuse the cache; use --ipxe-rebuild to force a rebuild.',
);
}
Underpost.baremetal.ipxeEfiFactory({
tftpRootPath,
ipxeCacheDir,
arch,
underpostRoot,
embeddedScriptPath: `${tftpRootPath}/boot.ipxe`,
forceRebuild: true,
});
} else {
shellExec(`cp ${cachedIpxePath} ${tftpIpxePath}`);
logger.info('✓ Using cached iPXE EFI binary (pass --ipxe-rebuild to force a rebuild)', {
path: cachedIpxePath,
});
}
}
const { grubCfgSrc } = Underpost.baremetal.grubFactory({
menuentryStr,
kernelPath: `/${tftpPrefix}/pxe/vmlinuz-efi`,
initrdPath: `/${tftpPrefix}/pxe/initrd.img`,
cmd,
tftpIp: callbackMetaData.runnerHost.ip,
ipxe: useIpxe,
ipxePath: `/${tftpPrefix}/ipxe.efi`,
});
Underpost.baremetal.writeGrubConfigToFile({
grubCfgSrc: machine ? grubCfgSrc.replaceAll('system-id', machine.system_id) : grubCfgSrc,
});
if (machine) {
logger.info('✓ GRUB config written with system_id', { system_id: machine.system_id });
}
Underpost.baremetal.updateKernelFiles({
commissioningImage,
resourcesPath,
tftpRootPath,
kernelFilesPaths,
});
}
// Pass architecture from commissioning or deployment config
const grubArch = commissioningImage.architecture;
Underpost.baremetal.efiGrubModulesFactory({ image: { architecture: grubArch } });
// Set ownership and permissions for TFTP root.
shellExec(`sudo chown -R $(whoami):$(whoami) ${process.env.TFTP_ROOT}`);
shellExec(`sudo sudo chmod 755 ${process.env.TFTP_ROOT}`);
if (type === 'chroot-debootstrap' || type === 'chroot-container')
await Underpost.baremetal.nfsMountCallback({
hostname,
nfsHostPath,
workflowId,
mount: true,
});
const commissionMonitorPayload = {
macAddress,
ipAddress,
hostname,
architecture: Underpost.baremetal.fallbackArchitecture(workflowsConfig[workflowId]),
machine,
};
logger.info('Waiting for commissioning...', {
...commissionMonitorPayload,
machine: machine ? machine.system_id : null,
});
const { discovery, machine: discoveredMachine } =
await Underpost.baremetal.commissionMonitor(commissionMonitorPayload);
if (discoveredMachine) machine = discoveredMachine;
// Rocky/RHEL disk-install orchestration:
// Once the ephemeral Kickstart/Anaconda runtime reports key-only SSH
// readiness over the bootstrap HTTP POST sink, drive the unattended
// install of Rocky onto the detected target disk via a key-only,
// non-interactive remote command. The runtime also has an AUTO_INSTALL
// fallback, so a missed handshake never blocks installation forever.
if (
Underpost.baremetal.getFamilyBaseOs(workflowsConfig[workflowId].osIdLike).isRhelBased &&
options.remoteInstall !== false
) {
const { privateKeyPath, user: resolvedUser } = Underpost.baremetal.resolveSshKeyPaths({
options,
workflowsConfig,
workflowId,
});
await Underpost.baremetal.remoteInstallOrchestrator({
hostname,
ipAddress,
sshPort: 22,
keyPath: privateKeyPath,
});
// After the OS is installed and the node reboots into the deployed
// disk, run the post-install dispatcher: it reads the workflow's
// infraSetup field and provisions the requested infrastructure (e.g.
// a kubeadm control-plane or worker) over key-only SSH. The resolved
// user is used when authenticating to an EXISTING control-plane (e.g.
// admin@dd-core); fresh nodes are always reached as root.
await Underpost.baremetal.postInstallDispatcher({
workflowId,
workflowsConfig,
hostname,
ipAddress,
options,
underpostRoot,
keyPath: privateKeyPath,
controlUser: resolvedUser,
});
}
}
},
/**
* @method postInstallDispatcher
* @description Generic first-boot post-install dispatcher. Reads the workflow's
* `infraSetup` field and runs the matching infrastructure setup on the freshly
* deployed node. Easy to extend: add a new case per supported infraSetup type.
* Fails with a clear error if the workflow requests an unsupported type.
* @param {object} params
* @param {string} params.workflowId - Active workflow id.
* @param {object} params.workflowsConfig - Loaded workflows config.
* @param {string} params.hostname - Node hostname.
* @param {string} params.ipAddress - Node IP (installed OS, key-only SSH).
* @param {object} params.options - CLI options (carries --worker / --control).
* @param {string} params.underpostRoot - Engine root for resolving scripts.
* @param {string} [params.keyPath] - Private key path for key-only SSH.
* @param {string} [params.controlUser='root'] - SSH user for an existing control-plane.
* @returns {Promise<void>}
* @memberof UnderpostBaremetal
*/
async postInstallDispatcher({
workflowId,
workflowsConfig,
hostname,
ipAddress,
options,
underpostRoot,
keyPath,
controlUser = 'root',
}) {
const infraSetup = workflowsConfig[workflowId]?.infraSetup;
if (!infraSetup) {
logger.info('No infraSetup configured for workflow; skipping post-install setup', { workflowId });
return;
}
logger.info('Post-install dispatcher selecting infra setup', { workflowId, infraSetup });
switch (infraSetup) {
case 'underpost-kubeadm-contour':
await Underpost.baremetal.infraSetupKubeadm({
hostname,
ipAddress,
options,
underpostRoot,
keyPath,
controlUser,
});
break;
default:
throw new Error(
`Unsupported infraSetup "${infraSetup}" for workflow "${workflowId}". Supported: underpost-kubeadm-contour`,
);
}
},
/**
* @method infraSetupKubeadm
* @description Provisions a kubeadm node on the freshly deployed machine over
* key-only SSH. Control mode (default) initializes a new control-plane. Worker
* mode (`--worker`) joins an existing cluster: the join token is retrieved
* dynamically over SSH from the control-plane node (`--control <ip>`) — never
* pasted manually — and passed to the node setup script.
* @param {object} params
* @param {string} params.hostname - Node hostname (logging).
* @param {string} params.ipAddress - Target node IP (installed OS).
* @param {object} params.options - CLI options; options.worker, options.control.
* @param {string} params.underpostRoot - Engine root for resolving the script.
* @param {string} [params.keyPath] - Private key path for key-only SSH.
* @param {string} [params.controlUser='root'] - SSH login user for an existing control-plane (e.g. admin).
* @returns {Promise<void>}
* @memberof UnderpostBaremetal
*/
async infraSetupKubeadm({
hostname,
ipAddress,
options,
underpostRoot,
keyPath = 'engine-private/deploy/id_rsa',
controlUser = 'root',
}) {
const role = options.worker ? 'worker' : 'control';
const scriptPath = `${underpostRoot}/scripts/kubeadm-node-setup.sh`;
const installedOsTimeoutMs = 15 * 60 * 1000;
logger.info(`Kubeadm post-install setup (${role}) on ${hostname} (${ipAddress})`);
if (!options.resumeInfraSetup && !options.resumeJoin) {
// The node is rebooting from the ephemeral installer into the deployed OS.
// First wait for the (ephemeral) SSH port to CLOSE so we don't latch onto
// the pre-reboot sshd, then wait for the installed OS sshd to come UP.
logger.info('Waiting for node to reboot into the deployed OS (port close → reopen)...', { ipAddress });
await Underpost.ssh.waitForSshPortClosed({ host: ipAddress, port: 22, timeoutMs: 4 * 60 * 1000 });
const reachable = await Underpost.ssh.waitForSshPort({
host: ipAddress,
port: 22,
timeoutMs: installedOsTimeoutMs,
});
if (!reachable) {
logger.error('Installed OS SSH not reachable after reboot; cannot run kubeadm setup', { ipAddress });
return;
}
// The first time the port opens the node is still early in boot and
// NetworkManager may re-apply the static profile, resetting in-flight TCP
// connections. Let the network settle before driving the long node setup.
logger.info('Deployed OS SSH is up; letting the network settle before node setup...', { ipAddress });
await timer(30000);
} else {
logger.info('--resume-infra-setup: SSH already up, skipping port close/reopen wait', { ipAddress });
}
let scriptArgs;
if (role === 'worker') {
const controlIp = options.control;
if (!controlIp || typeof controlIp !== 'string') {
throw new Error('Worker mode requires the control-plane IP via --control <ip>');
}
// Retrieve the join command dynamically from the control-plane node over
// SSH (reuses the same key-only SSH execution path). No manual token paste.
// The control-plane may be an existing deploy reached as a non-root user
// (e.g. admin@dd-core), selected via --user/--deploy-id.
logger.info('Retrieving kubeadm join command from control-plane over SSH', {
controlIp,
controlUser,
keyPath,
});
const joinResult = await Underpost.ssh.sshExecBatch({
host: controlIp,
port: 22,
user: controlUser,
keyPath,
retries: 4,
waitForPortMs: 5 * 60 * 1000,
command: 'sudo kubeadm token create --print-join-command',
});
let joinCommand = (joinResult.stdout || '')
.split('\n')
.map((l) => l.trim())
.find((l) => l.startsWith('kubeadm join'));
if (!joinResult.ok || !joinCommand) {
throw new Error(
`Failed to retrieve kubeadm join command from control-plane ${controlUser}@${controlIp} using key "${keyPath}" (code ${joinResult.code}). ` +
`Ensure that key authorizes ${controlUser}@${controlIp} — pass --deploy-id <id> --user <user> (key from engine-private/conf/<id>/users/<user>) ` +
`or --ssh-key-dir <dir> to use the key/user the control-plane accepts. stderr: ${joinResult.stderr.slice(-300)}`,
);
}
// kubeadm prints the API endpoint as the control-plane's hostname (often
// localhost.localdomain), which the worker cannot resolve/route. Capture
// that original endpoint host (so the worker can map it -> control IP in
// /etc/hosts; the cluster-info / kubeadm-config ConfigMaps reference it),
// then rewrite the join positional endpoint to the real control-plane IP.
const endpointMatch = joinCommand.match(/kubeadm join\s+([^\s:]+):\d+/);
const endpointHost = endpointMatch ? endpointMatch[1] : '';
joinCommand = joinCommand.replace(/kubeadm join\s+\S+:(\d+)/, `kubeadm join ${controlIp}:$1`);
logger.info('✓ Retrieved kubeadm join command from control-plane', { controlIp, endpointHost, joinCommand });
scriptArgs = `--worker --join-command="${joinCommand}" --control-ip=${controlIp}`;
if (endpointHost && endpointHost !== controlIp) {
scriptArgs += ` --control-endpoint-host=${endpointHost}`;
}
// --resume-join: skip all node prep and go straight to the kubeadm join.
if (options.resumeJoin) scriptArgs += ' --join-only';
} else {
scriptArgs = '--control';
}
// Secrets needed on the node to clone the private engine-private repo
// (engine-private/conf/.../.env.production for `node bin run secret`).
const githubUsername = Underpost.baremetal.readEngineConfig('GITHUB_USERNAME') || 'underpostnet';
const githubEnv = {
GITHUB_TOKEN: Underpost.baremetal.readEngineConfig('GITHUB_TOKEN'),
GITHUB_USERNAME: githubUsername,
};
if (!githubEnv.GITHUB_TOKEN) {
logger.warn('GITHUB_TOKEN not resolved on controller; engine-private clone on the node will be skipped');
}
// Resolve the engine + engine-private repos the node clones and normalizes
// to /home/dd/engine and /home/dd/engine/engine-private. CLI overrides win;
// otherwise default to the base engine and the deployId's private repo
// (engine-<id>-private), mirroring repository.privateEngineRepoFactory.
const deployIdSuffix = options.deployId ? options.deployId.split('-')[1] : '';
const engineRepo = options.engineRepo || `https://github.com/${githubUsername}/engine.git`;
const enginePrivateRepo =
options.enginePrivateRepo ||
(deployIdSuffix
? `https://github.com/${githubUsername}/engine-${deployIdSuffix}-private.git`
: `https://github.com/${githubUsername}/engine-private.git`);
const repoArgs = [
`--engine-repo=${engineRepo}`,
`--engine-private-repo=${enginePrivateRepo}`,
...(options.engineBranch ? [`--engine-branch=${options.engineBranch}`] : []),
...(options.enginePrivateBranch ? [`--engine-private-branch=${options.enginePrivateBranch}`] : []),
].join(' ');
scriptArgs = `${scriptArgs} ${repoArgs}`.trim();
logger.info('Node engine repos', { engineRepo, enginePrivateRepo });
logger.info(`Running kubeadm-node-setup.sh (${role}) on ${ipAddress}... (this can take several minutes)`);
// The node setup is a long, mostly-idempotent install. Keep retries low so
// a late failure doesn't re-run the whole thing many times; the script
// skips already-completed steps (Node, engine clone) on a re-run.
const result = await Underpost.ssh.sshRunScript({
host: ipAddress,
scriptPath,
args: scriptArgs,
env: githubEnv,
remotePath: '/tmp/kubeadm-node-setup.sh',
keyPath,
retries: 2,
waitForPortMs: 5 * 60 * 1000,
});
if (result.ok) {
logger.info(`✓ Kubeadm ${role} node setup completed on ${hostname}`, {
stdout: result.stdout.split('\n').slice(-12).join('\n'),
});
} else {
logger.error(`✗ Kubeadm ${role} node setup failed on ${hostname}`, {
code: result.code,
stderr: result.stderr.slice(-600),
});
}
},
/**
* @method remoteInstallOrchestrator
* @description Waits for the ephemeral runtime's 'ssh-ready' lifecycle event,
* then triggers the unattended Rocky install over a key-only SSH batch
* command. Returns structured success/failure information. Non-fatal: logs
* and returns on failure so the runtime's AUTO_INSTALL fallback can proceed.
* @param {object} params
* @param {string} params.hostname - Hostname key used by the bootstrap POST sink.
* @param {string} params.ipAddress - Fallback IP if the runtime did not report one.
* @param {number} [params.sshPort=22] - SSH port of the ephemeral runtime.
* @param {string} [params.keyPath] - Private key path for key-only auth.
* @returns {Promise<{ok: boolean, host: string, result?: object, reason?: string}>}
* @memberof UnderpostBaremetal
*/
async remoteInstallOrchestrator({ hostname, ipAddress, sshPort = 22, keyPath = 'engine-private/deploy/id_rsa' }) {
logger.info('Awaiting ephemeral runtime ssh-ready handshake...', { hostname, ipAddress });
const readyEvent = await Underpost.baremetal.waitForBootstrapStage({ hostname, stage: 'ssh-ready' });
const host = readyEvent?.ip && readyEvent.ip !== 'UNKNOWN' ? readyEvent.ip : ipAddress;
if (!readyEvent) {
logger.warn('No ssh-ready event received; relying on runtime AUTO_INSTALL fallback', { host });
return { ok: false, host, reason: 'no ssh-ready event' };
}
logger.info('SSH-ready event received', { host, metadata: readyEvent });
// Key-only, non-interactive trigger of the on-host installer. We only DROP
// the trigger file here; the persistent %pre lifecycle loop (a child of
// Anaconda, not of this sshd session) launches the installer fully
// detached. Running the installer directly as an sshd child causes it to
// be torn down when the SSH channel closes, before it can even start.
const result = await Underpost.ssh.sshExecBatch({
host,
port: sshPort,
user: 'root',
keyPath,
waitForPortMs: 5 * 60 * 1000,
retries: 4,
command: ['set -e', 'touch /tmp/.underpost-install-trigger', 'echo "install trigger dropped"'].join('\n'),
});
if (result.ok) {
logger.info('✓ Remote install trigger dropped', { host, stdout: result.stdout.trim() });
} else {
logger.error('Remote install trigger failed; runtime AUTO_INSTALL fallback will cover it', {
host,
code: result.code,
stderr: result.stderr.slice(-400),
});
}
// Follow the install lifecycle so the controller reflects real progress.
const started = await Underpost.baremetal.waitForBootstrapStage({
hostname,
stage: 'install-start',
timeoutMs: 3 * 60 * 1000,
});
if (!started) {
logger.warn('Installer did not report install-start within 3m; check /tmp/underpost-install.log on the node', {
host,
});
return { ok: false, host, reason: 'no install-start', result };
}
logger.info('Installer launched on node', { host });
// Wait for a terminal stage. The node reboots on success, so 'completed'
// is the last event we will receive.
const terminal = await Underpost.baremetal.waitForBootstrapStage({
hostname,
stage: 'completed',
timeoutMs: 40 * 60 * 1000,
});
if (terminal) {
logger.info('✓ Rocky install completed; node rebooting into deployed OS', { host, event: terminal });
return { ok: true, host, result, terminal };
}
const failed = (UnderpostBaremetal.bootstrapStatusEvents.get(hostname) || []).find((e) => e.stage === 'failed');
if (failed) {
logger.error('✗ Rocky install reported failure', { host, event: failed });
return { ok: false, host, reason: 'install failed', result, failed };
}
logger.warn('Install did not reach a terminal stage within timeout', { host });
return { ok: false, host, reason: 'no terminal stage', result };
},
/**
* @method installPacker
* @description Installs Packer CLI.
* @memberof UnderpostBaremetal
* @returns {Promise<void>}
*/
async installPacker(underpostRoot) {
const scriptPath = `${underpostRoot}/scripts/packer-setup.sh`;
logger.info(`Installing Packer using script: ${scriptPath}`);
shellExec(`sudo chmod +x ${scriptPath}`);
shellExec(`sudo ${scriptPath}`);
},
/**
* @method ipxeBuildIso
* @description Builds a UEFI-bootable iPXE ISO with an embedded bridge script.
* @param {object} params
* @param {string} params.workflowId - The workflow identifier (e.g., 'hp-envy-iso-ram').
* @param {string} params.isoOutputPath - Output path for the generated ISO file.
* @param {string} params.tftpPrefix - TFTP prefix directory (e.g., 'envy').
* @param {string} params.ipFileServer - IP address of the TFTP/file server to chain to.
* @param {string} [params.ipAddress='192.168.1.191'] - The IP address of the client machine.
* @param {string} [params.ipConfig='none'] - IP configuration method (e.g., 'dhcp', 'none').
* @param {string} [params.netmask='255.255.255.0'] - The network mask.
* @param {string} [params.dnsServer='8.8.8.8'] - The DNS server address.
* @param {string} [params.macAddress=''] - The MAC address of the client machine.
* @param {boolean} [params.cloudInit=false] - Flag to enable cloud-init.
* @param {boolean} [params.dev=false] - Development mode flag to determine paths.
* @param {boolean} [params.forceRebuild=false] - Force a complete iPXE rebuild. Without this, reuses existing ISO.
* @param {number} [params.bootstrapHttpServerPort=8888] - Port for the bootstrap HTTP server used in ISO RAM workflows.
* @memberof UnderpostBaremetal
* @returns {Promise<void>}
*/
async ipxeBuildIso({
workflowId,
isoOutputPath,
tftpPrefix,
ipFileServer,
ipAddress,
ipConfig,
netmask,
dnsServer,
macAddress,
cloudInit,
dev,
forceRebuild = false,
bootstrapHttpServerPort,
}) {
const outputPath = !isoOutputPath || isoOutputPath === '.' ? `./ipxe-${workflowId}.iso` : isoOutputPath;
shellExec(`mkdir -p $(dirname ${outputPath})`);
const workflowsConfig = Underpost.baremetal.loadWorkflowsConfig();
if (!workflowsConfig[workflowId]) {
throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
}
const authCredentials = cloudInit
? Underpost.baremetal.maasAuthCredentialsFactory()
: { consumer_key: '', consumer_secret: '', token_key: '', token_secret: '' };
const { cmd } = Underpost.baremetal.kernelCmdBootParamsFactory({
ipClient: ipAddress,
ipDhcpServer: ipFileServer,
ipFileServer,
ipConfig,
netmask,
hostname: workflowId,
dnsServer,
fileSystemUrl:
dev && workflowsConfig[workflowId].type === 'iso-ram'
? `http://${ipFileServer}:${Underpost.baremetal.bootstrapHttpServerPortFactory({
port: bootstrapHttpServerPort,
workflowId,
workflowsConfig,
})}/${workflowId}/${workflowsConfig[workflowId].isoUrl.split('/').pop()}`
: workflowsConfig[workflowId].isoUrl,
type: workflowsConfig[workflowId].type,
architecture: workflowsConfig[workflowId].architecture,
macAddress,
cloudInit,
osIdLike: workflowsConfig[workflowId].osIdLike,
networkInterfaceName: workflowsConfig[workflowId].networkInterfaceName,
authCredentials,
bootstrapHttpServerPort: Underpost.baremetal.bootstrapHttpServerPortFactory({
port: bootstrapHttpServerPort,
workflowId,
workflowsConfig,
}),
dev,
});
const ipxeSrcDir = '/home/dd/ipxe/src';
const embedScriptName = `embed_${workflowId}.ipxe`;
const embedScriptPath = path.join(ipxeSrcDir, embedScriptName);
const embedScriptContent = Underpost.baremetal.ipxeScriptFactory({
maasIp: ipFileServer,
tftpPrefix,
kernelCmd: cmd,
minimal: true,
});
fs.writeFileSync(embedScriptPath, embedScriptContent);
logger.info(`Created embedded script at ${embedScriptPath}`);
// Determine target architecture
let targetArch = 'x86_64';
if (
workflowsConfig[workflowId].architecture === 'arm64' ||
workflowsConfig[workflowId].architecture === 'aarch64'
) {
targetArch = 'arm64';
}
const platformDir = targetArch === 'arm64' ? 'bin-arm64-efi' : 'bin-x86_64-efi';
const makeTarget = `${platformDir}/ipxe.iso`;
const builtIsoPath = path.join(ipxeSrcDir, makeTarget);
if (!forceRebuild && fs.existsSync(builtIsoPath)) {
fs.copySync(builtIsoPath, outputPath);
logger.info(`Reusing existing iPXE ISO: ${builtIsoPath} -> ${outputPath}`);
} else {
logger.info(`Rebuild: cleaning iPXE build artifacts...`);
shellExec(`cd ${ipxeSrcDir} && make clean`);
const hostArch = process.arch === 'arm64' ? 'arm64' : 'x86_64';
let crossCompile = '';
if (hostArch === 'x86_64' && targetArch === 'arm64') {
crossCompile = 'CROSS_COMPILE=aarch64-linux-gnu-';
} else if (hostArch === 'arm64' && targetArch === 'x86_64') {
crossCompile = 'CROSS_COMPILE=x86_64-linux-gnu-';
}
logger.info(
`Building iPXE ISO for ${targetArch} on ${hostArch}: make ${makeTarget} ${crossCompile} EMBED=${embedScriptName}`,
);
const buildCmd = `cd ${ipxeSrcDir} && make ${makeTarget} ${crossCompile} EMBED=${embedScriptName}`;
shellExec(buildCmd);
if (fs.existsSync(builtIsoPath)) {
fs.copySync(builtIsoPath, outputPath);
logger.info(`ISO successfully built and copied to ${outputPath}`);
} else {
logger.error(`Failed to build ISO at ${builtIsoPath}`);
}
}
},
/**
* @method fallbackArchitecture
* @description Determines the architecture to use for boot resources, with a fallback mechanism.
* @param {object} workflowsConfig - The configuration object for the current workflow.
* @returns {string} The architecture string (e.g., 'arm64', 'amd64') to use for boot resources.
* @memberof UnderpostBaremetal
*/
fallbackArchitecture(workflowsConfig) {
return (
workflowsConfig.architecture ||
workflowsConfig.maas?.commissioning?.architecture ||
workflowsConfig.container?.architecture ||
workflowsConfig.debootstrap?.image?.architecture
);
},
/**
* @method macAddressFactory
* @description Generates or returns a MAC address based on options.
* @param {object} options - Options for MAC address generation.
* @param {string} options.mac - 'random' for random MAC, 'hardware' to use device's actual MAC, specific MAC string, or empty for default.
* @returns {object} Object with mac property - null for 'hardware', generated/specified MAC otherwise.
* @memberof UnderpostBaremetal
*/
macAddressFactory(options = { mac: '' }) {
const len = 6;
const defaultMac = range(1, len)
.map(() => '00')
.join(':');
if (options) {
if (!options.mac) options.mac = defaultMac;
if (options.mac === 'hardware') {
// Return null to indicate hardware MAC should be used (no spoofing)
options.mac = null;
} else if (options.mac === 'random') {
options.mac = range(1, len)
.map(() => s4().toLowerCase().substring(0, 2))
.join(':');
}
} else options = { mac: defaultMac };
return options;
},
/**
* @method downloadISO
* @description Downloads a generic ISO and extracts kernel boot files.
* @param {object} params - Parameters for the method.
* @param {object} params.resource - The MAAS boot resource object.
* @param {string} params.architecture - The architecture (arm64 or amd64).
* @param {string} params.nfsHostPath - The NFS host path to store the ISO and extracted files.
* @param {string} params.isoUrl - The full URL to the ISO file to download.
* @param {string} params.osIdLike - OS family identifier (e.g., 'debian ubuntu' or 'rhel centos fedora').
* @returns {object} An object containing paths to the extracted kernel, initrd, and optionally squashfs.
* @memberof UnderpostBaremetal
*/
downloadISO({ resource, architecture, nfsHostPath, isoUrl, osIdLike }) {
const arch = architecture || resource.architecture.split('/')[0];
// Validate that isoUrl is provided
if (!isoUrl) {
throw new Error('isoUrl parameter is required. Please specify the full ISO URL in the workflow configuration.');
}
// Extract ISO filename from URL
const isoFilename = isoUrl.split('/').pop();
// Determine OS family from osIdLike
const { isDebianBased, isRhelBased } = Underpost.baremetal.getFamilyBaseOs(osIdLike);
// Set extraction directory based on OS family
const extractDirName = isDebianBased ? 'casper' : 'iso-extract';
shellExec(`mkdir -p ${nfsHostPath}/${extractDirName}`);
const isoPath = `/var/tmp/live-iso/${isoFilename}`;
const extractDir = `${nfsHostPath}/${extractDirName}`;
if (!fs.existsSync(isoPath)) {
logger.info(`Downloading ISO for ${arch}...`);
logger.info(`URL: ${isoUrl}`);
shellExec(`mkdir -p /var/tmp/live-iso`);
shellExec(`wget --progress=bar:force -O ${isoPath} "${isoUrl}"`);
// Verify download by checking file existence and size (not exit code, which can be unreliable)
if (!fs.existsSync(isoPath)) {
throw new Error(`Failed to download ISO from ${isoUrl} - file not created`);
}
const stats = fs.statSync(isoPath);
if (stats.size < 100 * 1024 * 1024) {
shellExec(`rm -f ${isoPath}`);
throw new Error(`Downloaded ISO is too small (${stats.size} bytes), download may have failed`);
}
logger.info(`Downloaded ISO to ${isoPath} (${(stats.size / 1024 / 1024 / 1024).toFixed(2)} GB)`);
}
// ISO Logic
const mountPoint = `${nfsHostPath}/mnt-iso-${arch}`;
shellExec(`mkdir -p ${mountPoint}`);
// Ensure mount point is not already mounted
shellExec(`sudo umount ${mountPoint}`, {
silentOnError: true, // Ignore errors if not mounted
});
try {
// Mount the ISO
shellExec(`sudo mount -o loop,ro ${isoPath} ${mountPoint}`);
logger.info(`Mounted ISO at ${mountPoint}`);
// Distribution-specific extraction logic
if (isDebianBased) {
// Ubuntu/Debian: Extract from casper directory
if (!fs.existsSync(`${mountPoint}/casper`)) {
throw new Error(`Failed to mount ISO or casper directory not found: ${isoPath}`);
}
logger.info(`Checking casper directory contents...`);
shellExec(`ls -la ${mountPoint}/casper/ 2>/dev/null || echo "casper directory not found"`);
shellExec(`sudo cp -a ${mountPoint}/casper/* ${extractDir}/`);
} else if (isRhelBased) {
// RHEL/Rocky: Extract from images/pxeboot directory
const pxebootDir = `${mountPoint}/images/pxeboot`;
if (!fs.existsSync(pxebootDir)) {
throw new Error(`Failed to mount ISO or images/pxeboot directory not found: ${isoPath}`);
}
logger.info(`Extracting kernel and initrd from ${pxebootDir}...`);
shellExec(`sudo cp -a ${pxebootDir}/vmlinuz ${extractDir}/vmlinuz`);
shellExec(`sudo cp -a ${pxebootDir}/initrd.img ${extractDir}/initrd`);
}
} finally {
shellExec(`ls -la ${mountPoint}/`);
shellExec(`sudo chown -R $(whoami):$(whoami) ${extractDir}`);
// Unmount ISO
shellExec(`sudo umount ${mountPoint}`);
logger.info(`Unmounted ISO`);
// Clean up temporary mount point
shellExec(`rmdir ${mountPoint}`);
}
return {
'vmlinuz-efi': `${extractDir}/vmlinuz`,
'initrd.img': `${extractDir}/initrd`,
isoUrl,
};
},
/**
* @method machineFactory
* @description Creates a new machine in MAAS with specified options.
* @param {object} options - Options for creating the machine.
* @param {string} options.macAddress - The MAC address of the machine.
* @param {string} options.hostname - The hostname for the machine.
* @param {string} options.ipAddress - The IP address for the machine.
* @param {string} options.architecture - The architecture for the machine (default is 'arm64').
* @param {string} options.powerType - The power type for the machine (default is 'manual').
* @returns {object} An object containing the created machine details.
* @memberof UnderpostBaremetal
*/
machineFactory(
options = {
macAddress: '',
hostname: '',
ipAddress: '',
architecture: 'arm64',
powerType: 'manual',
},
) {
if (!options.powerType) options.powerType = 'manual';
const payload = {
architecture: options.architecture.match('arm') ? 'arm64/generic' : 'amd64/generic',
mac_address: options.macAddress,
mac_addresses: options.macAddress,
hostname: options.hostname,
power_type: options.powerType,
ip: options.ipAddress,
};
logger.info('Creating MAAS machine', payload);
const machine = Underpost.baremetal.maasCliExec(
`machines create ${Object.keys(payload)
.map((k) => `${k}="${payload[k]}"`)
.join(' ')}`,
);
try {
return { machine };
} catch (error) {
console.log(error);
logger.error(error);
throw new Error(`Failed to create MAAS machine. Output:\n${machine}`);
}
},
/**
* @method getFamilyBaseOs
* @description Determines if the OS belongs to Debian-based or RHEL-based family based on osIdLike string.
* @param {string} osIdLike - The os_id_like string from MAAS boot resource or workflow configuration.
* @returns {object} An object with boolean properties isDebianBased and isRhelBased indicating the OS family.
* @memberof UnderpostBaremetal
*/
getFamilyBaseOs(osIdLike = '') {
return {
isDebianBased: osIdLike.match(/debian|ubuntu/i),
isRhelBased: osIdLike.match(/rhel|centos|fedora|alma|rocky/i),
};
},
/**
* @method kernelFactory
* @description Retrieves kernel, initrd, and root filesystem paths from a MAAS boot resource.
* @param {object} params - Parameters for the method.
* @param {object} params.resource - The MAAS boot resource object.
* @param {string} params.type - The type of boot (e.g., 'iso-ram', 'iso-nfs', etc.).
* @param {string} params.nfsHostPath - The NFS host path (used for ISO types).
* @param {string} params.isoUrl - The ISO URL (used for ISO types).
* @param {string} params.workflowId - The workflow identifier.
* @returns {object} An object containing paths to the kernel, initrd, and root filesystem.
* @memberof UnderpostBaremetal
*/
kernelFactory({ resource, type, nfsHostPath, isoUrl, workflowId }) {
// For disk-based commissioning (casper/iso), use live ISO files
if (type === 'iso-ram' || type === 'iso-nfs') {
logger.info('Using live ISO for boot (disk-based commissioning)');
const workflowsConfig = Underpost.baremetal.loadWorkflowsConfig();
const arch = resource?.architecture
? resource.architecture.split('/')[0]
: workflowsConfig[workflowId].architecture;
const kernelFilesPaths = Underpost.baremetal.downloadISO({
resource,
architecture: arch,
nfsHostPath,
isoUrl,
osIdLike: workflowsConfig[workflowId].osIdLike || '',
});
const resourcesPath = `/var/snap/maas/common/maas/image-storage/bootloaders/uefi/${arch}`;
return { kernelFilesPaths, resourcesPath };
}
const resourceData = Underpost.baremetal.maasCliExec(`boot-resource read ${resource.id}`);
let kernelFilesPaths = {};
const bootFiles = resourceData.sets[Object.keys(resourceData.sets)[0]].files;
const arch = resource.architecture.split('/')[0];
const resourcesPath = `/var/snap/maas/common/maas/image-storage/bootloaders/uefi/${arch}`;
const kernelPath = `/var/snap/maas/common/maas/image-storage`;
logger.info('Available boot files', Object.keys(bootFiles));
logger.info('Boot files info', {
id: resourceData.id,
type: resourceData.type,
name: resourceData.name,
architecture: resourceData.architecture,
bootFiles,
arch,
resourcesPath,
kernelPath,
});
// Try standard synced image structure (Ubuntu, CentOS from MAAS repos)
const _suffix = resource.architecture.match('xgene') ? '.xgene' : '';
if (bootFiles['boot-kernel' + _suffix] && bootFiles['boot-initrd' + _suffix] && bootFiles['squashfs']) {
kernelFilesPaths = {
'vmlinuz-efi': `${kernelPath}/${bootFiles['boot-kernel' + _suffix].filename_on_disk}`,
'initrd.img': `${kernelPath}/${bootFiles['boot-initrd' + _suffix].filename_on_disk}`,
squashfs: `${kernelPath}/${bootFiles['squashfs'].filename_on_disk}`,
};
}
// Try uploaded image structure (Packer-built images, custom uploads)
else if (bootFiles['boot-kernel'] && bootFiles['boot-initrd'] && bootFiles['root-tgz']) {
kernelFilesPaths = {
'vmlinuz-efi': `${kernelPath}/${bootFiles['boot-kernel'].filename_on_disk}`,
'initrd.img': `${kernelPath}/${bootFiles['boot-initrd'].filename_on_disk}`,
squashfs: `${kernelPath}/${bootFiles['root-tgz'].filename_on_disk}`,
};
}
// Try alternative uploaded structure with root-image-xz
else if (bootFiles['boot-kernel'] && bootFiles['boot-initrd'] && bootFiles['root-image-xz']) {
kernelFilesPaths = {
'vmlinuz-efi': `${kernelPath}/${bootFiles['boot-kernel'].filename_on_disk}`,
'initrd.img': `${kernelPath}/${bootFiles['boot-initrd'].filename_on_disk}`,
squashfs: `${kernelPath}/${bootFiles['root-image-xz'].filename_on_disk}`,
};
}
// Fallback: try to find any kernel, initrd, and root image
else {
logger.warn('Non-standard boot file structure detected. Available files', Object.keys(bootFiles));
const rootArchiveKey = Object.keys(bootFiles).find(
(k) => k.includes('root') && (k.includes('tgz') || k.includes('tar.gz')),
);
const explicitKernel = Object.keys(bootFiles).find((k) => k.includes('kernel'));
const explicitInitrd = Object.keys(bootFiles).find((k) => k.includes('initrd'));
if (rootArchiveKey && (!explicitKernel || !explicitInitrd)) {
logger.info(`Root archive found (${rootArchiveKey}) and missing kernel/initrd. Attempting to extract.`);
const rootArchivePath = `${kernelPath}/${bootFiles[rootArchiveKey].filename_on_disk}`;
const tempExtractDir = `/tmp/maas-extract-${resource.id}`;
shellExec(`mkdir -p ${tempExtractDir}`);
// List files in archive to find kernel and initrd
const tarList = shellExec(`tar -tf ${rootArchivePath}`).stdout.split('\n');
// Look for boot/vmlinuz* and boot/initrd* (handling potential leading ./)
// Skip rescue, kdump, and other special images
const vmlinuzPaths = tarList.filter(
(f) => f.match(/(\.\/)?boot\/vmlinuz-[0-9]/) && !f.includes('rescue') && !f.includes('kdump'),
);
const initrdPaths = tarList.filter(
(f) =>
f.match(/(\.\/)?boot\/(initrd|initramfs)-[0-9]/) &&
!f.includes('rescue') &&
!f.includes('kdump') &&
f.includes('.img'),
);
logger.info(`Found kernel candidates:`, { vmlinuzPaths, initrdPaths });
// Try to match kernel and initrd by version number
let vmlinuzPath = null;
let initrdPath = null;
if (vmlinuzPaths.length > 0 && initrdPaths.length > 0) {
// Extract version from kernel filename (e.g., "5.14.0-611.11.1.el9_7.aarch64")
for (const kernelPath of vmlinuzPaths.sort().reverse()) {
const kernelVersion = kernelPath.match(/vmlinuz-(.+)$/)?.[1];
if (kernelVersion) {
// Look for matching initrd
const matchingInitrd = initrdPaths.find((p) => p.includes(kernelVersion));
if (matchingInitrd) {
vmlinuzPath = kernelPath;
initrdPath = matchingInitrd;
logger.info(`Matched kernel and initrd by version: ${kernelVersion}`);
break;
}
}
}
}
// Fallback: use newest versions if no match found
if (!vmlinuzPath && vmlinuzPaths.length > 0) {
vmlinuzPath = vmlinuzPaths.sort().pop();
}
if (!initrdPath && initrdPaths.length > 0) {
initrdPath = initrdPaths.sort().pop();
}
logger.info(`Selected kernel: ${vmlinuzPath}, initrd: ${initrdPath}`);
if (vmlinuzPath && initrdPath) {
// Extract specific files
// Extract all files in boot/ to ensure symlinks resolve
shellExec(`tar -xf ${rootArchivePath} -C ${tempExtractDir} --wildcards '*boot/*'`);
kernelFilesPaths = {
'vmlinuz-efi': `${tempExtractDir}/${vmlinuzPath}`,
'initrd.img': `${tempExtractDir}/${initrdPath}`,
squashfs: rootArchivePath,
};
logger.info('Extracted kernel and initrd from root archive.');
} else {
logger.error(
`Failed to find kernel/initrd in archive. Contents of boot/ directory:`,
tarList.filter((f) => f.includes('boot/')),
);
throw new Error(`Could not find kernel or initrd in ${rootArchiveKey}`);
}
} else {
const kernelFile = Object.keys(bootFiles).find((k) => k.includes('kernel')) || Object.keys(bootFiles)[0];
const initrdFile = Object.keys(bootFiles).find((k) => k.includes('initrd')) || Object.keys(bootFiles)[1];
const rootFile =
Object.keys(bootFiles).find(
(k) => k.includes('root') || k.includes('squashfs') || k.includes('tgz') || k.includes('xz'),
) || Object.keys(bootFiles)[2];
if (kernelFile && initrdFile && rootFile) {
kernelFilesPaths = {
'vmlinuz-efi': `${kernelPath}/${bootFiles[kernelFile].filename_on_disk}`,
'initrd.img': `${kernelPath}/${bootFiles[initrdFile].filename_on_disk}`,
squashfs: `${kernelPath}/${bootFiles[rootFile].filename_on_disk}`,
};
logger.info('Using detected files', { kernel: kernelFile, initrd: initrdFile, root: rootFile });
} else {
throw new Error(`Cannot identify boot files. Available: ${Object.keys(bootFiles).join(', ')}`);
}
}
}
return {
resource,
bootFiles,
arch,
resourcesPath,
kernelPath,
resourceData,
kernelFilesPaths,
};
},
/**
* @method writeGrubConfigToFile
* @description Writes the GRUB configuration content to the grub.cfg file in the TFTP root.
* @param {object} params - Parameters for the method.
* @param {string} params.grubCfgSrc - The GRUB configuration content to write.
* @memberof UnderpostBaremetal
* @returns {void}
*/
writeGrubConfigToFile({ grubCfgSrc = '' }) {
shellExec(`mkdir -p ${process.env.TFTP_ROOT}/grub`, {
disableLog: true,
});
return fs.writeFileSync(`${process.env.TFTP_ROOT}/grub/grub.cfg`, grubCfgSrc, 'utf8');
},
/**
* @method getGrubConfigFromFile
* @description Reads the GRUB configuration content from the grub.cfg file in the TFTP root.
* @memberof UnderpostBaremetal
* @returns {string} The GRUB configuration content.
*/
getGrubConfigFromFile() {
const grubCfgPath = `${process.env.TFTP_ROOT}/grub/grub.cfg`;
const grubCfgSrc = fs.readFileSync(grubCfgPath, 'utf8');
return { grubCfgPath, grubCfgSrc };
},
/**
* @method removeDiscoveredMachines
* @description Removes all machines in the 'discovered' status from MAAS.
* @memberof UnderpostBaremetal
* @returns {void}
*/
removeDiscoveredMachines() {
logger.info('Removing all discovered machines from MAAS...');
Underpost.baremetal.maasCliExec(`discoveries clear all=true`);
},
/**
* @method efiGrubModulesFactory
* @description Copies the appropriate EFI GRUB modules to the TFTP root based on the image architecture.
* @param {object} options - Options for determining which GRUB modules to copy.
* @param {object} options.image - Image configuration object.
* @param {string} options.image.architecture - The architecture of the image ('amd64' or 'arm64').
* @memberof UnderpostBaremetal
* @returns {void}
*/
efiGrubModulesFactory(options = { image: { architecture: 'amd64' } }) {
if (options.image.architecture.match('arm64')) {
// Copy ARM64 EFI GRUB modules.
const arm64EfiPath = `${process.env.TFTP_ROOT}/grub/arm64-efi`;
if (fs.existsSync(arm64EfiPath)) shellExec(`sudo rm -rf ${arm64EfiPath}`);
shellExec(`sudo cp -a /usr/lib/grub/arm64-efi ${arm64EfiPath}`);
} else {
// Copy AMD64 EFI GRUB modules.
const amd64EfiPath = `${process.env.TFTP_ROOT}/grub/x86_64-efi`;
if (fs.existsSync(amd64EfiPath)) shellExec(`sudo rm -rf ${amd64EfiPath}`);
shellExec(`sudo cp -a /usr/lib/grub/x86_64-efi ${amd64EfiPath}`);
}
},
/**
* @method ipxeEmbeddedScriptFactory
* @description Generates the embedded iPXE boot script that performs DHCP and chains to the main script.
* This script is embedded into the iPXE binary or loaded first by GRUB.
* Supports MAC address spoofing for baremetal commissioning workflows.
* @param {object} params - The parameters for generating the script.
* @param {string} params.tftpServer - The IP address of the TFTP server.
* @param {string} params.scriptPath - The path to the main iPXE script on TFTP server.
* @param {string} [params.macAddress] - Optional MAC address to spoof for MAAS registration.
* @returns {string} The embedded iPXE script content.
* @memberof UnderpostBaremetal
*/
ipxeEmbeddedScriptFactory({ tftpServer, scriptPath, macAddress = null }) {
const macSpoofingBlock =
macAddress && macAddress !== '00:00:00:00:00:00'
? `
# MAC Address Information
echo ========================================
echo Target MAC for MAAS: ${macAddress}
echo Hardware MAC: \${net0/mac}
echo ========================================
echo NOTE: iPXE MAC spoofing does NOT persist to kernel
echo The kernel will receive MAC via ifname= parameter
echo ========================================
`
: `
# Using hardware MAC address
echo ========================================
echo Using device hardware MAC address
echo Hardware MAC: \${net0/mac}
echo MAC spoofing disabled - device uses actual MAC
echo ========================================
`;
return `#!ipxe
# Embedded iPXE Boot Script
# This script performs DHCP configuration and chains to the main boot script
echo ========================================
echo iPXE Embedded Boot Loader
echo ========================================
echo TFTP Server: ${tftpServer}
echo Script Path: ${scriptPath}
echo ========================================
${macSpoofingBlock}
# Show network interface info before DHCP
echo Network Interface Info (before DHCP):
echo Interface: net0
echo MAC: \${net0/mac}
ifstat
# Perform DHCP to get network configuration
echo ========================================
echo Performing DHCP configuration...
echo ========================================
dhcp net0 || goto dhcp_retry
echo DHCP configuration successful
echo IP Address: \${net0/ip}
echo Netmask: \${net0/netmask}
echo Gateway: \${net0/gateway}
echo DNS: \${net0/dns}
echo TFTP Server: \${next-server}
echo MAC used by DHCP: \${net0/mac}
# Chain to the main iPXE script
echo ========================================
echo Chainloading main boot script...
echo Script: tftp://${tftpServer}${scriptPath}
echo ========================================
chain tftp://${tftpServer}${scriptPath} || goto chain_failed
:dhcp_retry
echo DHCP failed, retrying in 5 seconds...
sleep 5
dhcp net0 || goto dhcp_retry
goto dhcp_success
:dhcp_success
echo DHCP retry successful
echo IP Address: \${net0/ip}
echo MAC: \${net0/mac}
chain tftp://${tftpServer}${scriptPath} || goto chain_failed
:chain_failed
echo ========================================
echo ERROR: Failed to chain to main script
echo TFTP Server: ${tftpServer}
echo Script Path: ${scriptPath}
echo ========================================
echo Retrying in 10 seconds...
sleep 10
chain tftp://${tftpServer}${scriptPath} || goto shell_debug
:shell_debug
echo Dropping to iPXE shell for manual debugging
echo Try: chain tftp://${tftpServer}${scriptPath}
shell
`;
},
/**
* @method ipxeScriptFactory
* @description Generates the iPXE script content for stable identity.
* This iPXE script uses directly boots kernel/initrd via TFTP.
* When minimal mode is enabled, generates a simple dhcp+kernel+initrd+boot script.
* @param {object} params - The parameters for generating the script.
* @param {string} params.maasIp - The IP address of the MAAS/file server.
* @param {string} [params.macAddress] - The MAC address registered in MAAS (for display only).
* @param {string} [params.architecture] - The architecture (arm64/amd64).
* @param {string} params.tftpPrefix - The TFTP prefix path (e.g., 'rpi4mb').
* @param {string} params.kernelCmd - The kernel command line parameters.
* @param {boolean} [params.minimal=false] - Generate a minimal embedded script for ISO builds.
* @returns {string} The iPXE script content.
* @memberof UnderpostBaremetal
*/
ipxeScriptFactory({ maasIp, macAddress, architecture, tftpPrefix, kernelCmd, minimal = false }) {
if (minimal) {
return `#!ipxe
dhcp
set server_ip ${maasIp}
set tftp_prefix ${tftpPrefix}
kernel tftp://\${server_ip}/\${tftp_prefix}/pxe/vmlinuz-efi ${kernelCmd}
initrd tftp://\${server_ip}/\${tftp_prefix}/pxe/initrd.img
boot || shell
`;
}
const macInfo =
macAddress && macAddress !== '00:00:00:00:00:00'
? `echo Registered MAC: ${macAddress}`
: `echo Using hardware MAC address`;
// Construct the full TFTP paths for kernel and initrd
const kernelPath = `${tftpPrefix}/pxe/vmlinuz-efi`;
const initrdPath = `${tftpPrefix}/pxe/initrd.img`;
const grubBootloader = architecture === 'arm64' ? 'grubaa64.efi' : 'grubx64.efi';
const grubPath = `${tftpPrefix}/pxe/${grubBootloader}`;
return `#!ipxe
echo ========================================
echo iPXE Network Boot
echo ========================================
echo MAAS Server: ${maasIp}
echo Architecture: ${architecture}
${macInfo}
echo ========================================
# Show current network configuration
echo Current Network Configuration:
ifstat
# Display MAC address information
echo MAC Address: \${net0/mac}
echo IP Address: \${net0/ip}
echo Gateway: \${net0/gateway}
echo DNS: \${net0/dns}
${macAddress && macAddress !== '00:00:00:00:00:00' ? `echo Target MAC for kernel: ${macAddress}` : ''}
# Direct kernel/initrd boot via TFTP
# Modern simplified approach: Direct kernel/initrd boot via TFTP
echo ========================================
echo Loading kernel and initrd via TFTP...
echo Kernel: tftp://${maasIp}/${kernelPath}
echo Initrd: tftp://${maasIp}/${initrdPath}
${
macAddress && macAddress !== '00:00:00:00:00:00'
? `echo Kernel will use MAC: ${macAddress} (via ifname parameter)`
: 'echo Kernel will use hardware MAC'
}
echo ========================================
# Load kernel via TFTP
kernel tftp://${maasIp}/${kernelPath} ${kernelCmd || 'console=ttyS0,115200'} || goto grub_fallback
echo Kernel loaded successfully
# Load initrd via TFTP
initrd tftp://${maasIp}/${initrdPath} || goto grub_fallback
echo Initrd loaded successfully
# Boot the kernel
echo Booting kernel...
boot
:grub_fallback
echo ========================================
echo Direct kernel boot failed, falling back to GRUB chainload...
echo TFTP Path: tftp://${maasIp}/${grubPath}
echo ========================================
# Fallback: Chain to GRUB via TFTP (avoids malformed HTTP bootloader issues)
chain tftp://${maasIp}/${grubPath} || goto http_fallback
:http_fallback
echo TFTP GRUB chainload failed, trying HTTP fallback...
echo ========================================
# Fallback: Try MAAS HTTP bootloader (may have certificate issues)
set boot-url http://${maasIp}:5248/images/bootloader
echo Boot URL: \${boot-url}
chain \${boot-url}/uefi/${architecture}/${
grubBootloader === 'grubaa64.efi' ? 'bootaa64.efi' : 'bootx64.efi'
} || goto error
:error
echo ========================================
echo ERROR: All boot methods failed
echo ========================================
echo MAAS IP: ${maasIp}
echo Architecture: ${architecture}
echo MAC: \${net0/mac}
echo IP: \${net0/ip}
echo ========================================
echo Retrying GRUB TFTP in 10 seconds...
sleep 10
chain tftp://${maasIp}/${grubPath} || goto shell_debug
:shell_debug
echo Dropping to iPXE shell for manual intervention
shell
`;
},
/**
* @method ipxeEfiFactory
* @description Manages iPXE EFI binary build with cache support.
* Checks cache, builds only if needed, saves to cache after build.
* @param {object} params - The parameters for iPXE build.
* @param {string} params.tftpRootPath - TFTP root directory path.
* @param {string} params.ipxeCacheDir - iPXE cache directory path.
* @param {string} params.arch - Target architecture (arm64/amd64).
* @param {string} params.underpostRoot - Underpost root directory.
* @param {string} [params.embeddedScriptPath] - Path to embedded boot script.
* @param {boolean} [params.forceRebuild=false] - Force rebuild regardless of cache.
* @returns {void}
* @memberof UnderpostBaremetal
*/
ipxeEfiFactory({ tftpRootPath, ipxeCacheDir, arch, underpostRoot, embeddedScriptPath, forceRebuild = false }) {
const embedArg =
embeddedScriptPath && fs.existsSync(embeddedScriptPath) ? ` --embed-script ${embeddedScriptPath}` : '';
const rebuildArg = forceRebuild ? ' --rebuild' : '';
logger.info('Building iPXE EFI binary...', { embeddedScriptPath, forced: forceRebuild });
shellExec(`${underpostRoot}/scripts/ipxe-setup.sh ${tftpRootPath} --target-arch ${arch}${embedArg}${rebuildArg}`);
shellExec(`mkdir -p ${ipxeCacheDir}`);
shellExec(`cp ${tftpRootPath}/ipxe.efi ${ipxeCacheDir}/ipxe.efi`);
},
/**
* @method grubFactory
* @description Generates the GRUB configuration file content.
* @param {object} params - The parameters for generating the configuration.
* @param {string} params.menuentryStr - The title of the menu entry.
* @param {string} params.kernelPath - The path to the kernel file (relative to TFTP root).
* @param {string} params.initrdPath - The path to the initrd file (relative to TFTP root).
* @param {string} params.cmd - The kernel command line parameters.
* @param {string} params.tftpIp - The IP address of the TFTP server.
* @param {boolean} [params.ipxe] - Flag to enable iPXE chainloading.
* @param {string} [params.ipxePath] - The path to the iPXE binary.
* @returns {object} An object containing 'grubCfgSrc' the GRUB configuration source string.
* @memberof UnderpostBaremetal
*/
grubFactory({ menuentryStr, kernelPath, initrdPath, cmd, tftpIp, ipxe, ipxePath }) {
if (ipxe) {
return {
grubCfgSrc: `
set default="0"
set timeout=10
insmod tftp
set root=(tftp,${tftpIp})
menuentry 'iPXE ${menuentryStr}' {
echo "Loading iPXE with embedded script..."
echo "[INFO] TFTP Server: ${tftpIp}"
echo "[INFO] iPXE Binary: ${ipxePath}"
echo "[INFO] iPXE will execute embedded script (dhcp + chain)"
chainloader ${ipxePath}
boot
}
`,
};
}
return {
grubCfgSrc: `
set default="0"
set timeout=10
insmod nfs
insmod gzio
insmod http
insmod tftp
set root=(tftp,${tftpIp})
menuentry '${menuentryStr}' {
echo "${menuentryStr}"
echo " ${Underpost.version}"
echo "Date: ${new Date().toISOString()}"
${
cmd.match('/MAAS/metadata/by-id/')
? `echo "System ID: ${cmd.split('/MAAS/metadata/by-id/')[1].split('/')[0]}"`
: ''
}
echo "TFTP server: ${tftpIp}"
echo "Kernel path: ${kernelPath}"
echo "Initrd path: ${initrdPath}"
echo "Starting boot process..."
echo "Loading kernel..."
linux /${kernelPath} ${cmd}
echo "Loading initrd..."
initrd /${initrdPath}
echo "Booting..."
boot
}
`,
};
},
/**
* @method bootstrapHttpServerPortFactory
* @description Determines the bootstrap HTTP server port.
* @param {object} params - Parameters for determining the port.
* @param {number} [params.port] - The port passed via options.
* @param {string} params.workflowId - The workflow identifier.
* @param {object} params.workflowsConfig - The loaded workflows configuration.
* @returns {number} The determined port number.
* @memberof UnderpostBaremetal
*/
bootstrapHttpServerPortFactory({ port, workflowId, workflowsConfig }) {
return port || workflowsConfig[workflowId]?.bootstrapHttpServerPort || 8888;
},
/**
* @method resolveSshKeyPaths
* @description Resolves the SSH key pair and login user for commissioning/
* orchestration. The key pair always lives at `<dir>/id_rsa` + `<dir>/id_rsa.pub`.
*
* Key-dir precedence:
* 1. `--ssh-key-dir <dir>` (explicit path)
* 2. `--deploy-id <id>` (+ `--user <user>`) → `engine-private/conf/<id>/users/<user>`
* (the same user↔deployId↔key convention `src/cli/ssh.js` writes; no key path needed)
* 3. workflow `sshKeyDir`
* 4. workflow `deployId` (+ `user`)
* 5. default `engine-private/deploy`
*
* User precedence: `--user` > workflow `user` > `root`. A leading `~` in the
* resolved dir is expanded to the user's home.
* @param {object} params
* @param {object} [params.options={}] - CLI options (sshKeyDir, deployId, user).
* @param {object} [params.workflowsConfig={}] - Loaded workflows config.
* @param {string} [params.workflowId=''] - Active workflow id.
* @returns {{ keyDir: string, privateKeyPath: string, publicKeyPath: string, user: string, deployId: string }}
* @memberof UnderpostBaremetal
*/
resolveSshKeyPaths({ options = {}, workflowsConfig = {}, workflowId = '' } = {}) {
const wf = workflowsConfig?.[workflowId] || {};
const user = options.user || wf.user || 'root';
const deployId = options.deployId || wf.deployId || '';
let dir;
if (options.sshKeyDir) dir = options.sshKeyDir;
else if (options.deployId) dir = `engine-private/conf/${options.deployId}/users/${user}`;
else if (wf.sshKeyDir) dir = wf.sshKeyDir;
else if (deployId) dir = `engine-private/conf/${deployId}/users/${user}`;
else dir = 'engine-private/deploy';
if (dir.startsWith('~')) dir = path.join(process.env.HOME || '', dir.slice(1));
return { keyDir: dir, privateKeyPath: `${dir}/id_rsa`, publicKeyPath: `${dir}/id_rsa.pub`, user, deployId };
},
/**
* @method resolveInstallCredentials
* @description Resolves the login accounts baked into the deployed OS. Always
* creates TWO admin accounts so a console login is guaranteed:
* 1. the MAAS admin (MAAS_ADMIN_USERNAME / MAAS_ADMIN_PASS) — password login,
* 2. the deploy user (`--user`, e.g. admin) — its conf.node.json
* password (or MAAS_ADMIN_PASS), plus the SSH key.
* root always gets MAAS_ADMIN_PASS. The deploy user/password come from
* `engine-private/conf/<deployId>/conf.node.json` (`users[user]`).
* @param {object} params
* @param {object} [params.options={}] - CLI options.
* @param {object} [params.workflowsConfig={}] - Loaded workflows config.
* @param {string} [params.workflowId=''] - Active workflow id.
* @returns {{ rootPassword: string, adminUsername: string, adminPassword: string, deployUsername: string, deployPassword: string, confUser: object }}
* @memberof UnderpostBaremetal
*/
resolveInstallCredentials({ options = {}, workflowsConfig = {}, workflowId = '' } = {}) {
const { user, deployId } = Underpost.baremetal.resolveSshKeyPaths({ options, workflowsConfig, workflowId });
let confUser = {};
if (deployId) {
const confPath = `engine-private/conf/${deployId}/conf.node.json`;
if (fs.existsSync(confPath)) {
try {
confUser = JSON.parse(fs.readFileSync(confPath, 'utf8'))?.users?.[user] || {};
} catch (error) {
logger.warn(`Failed to parse ${confPath}: ${error.message}`);
}
} else {
logger.warn(`conf.node.json not found at ${confPath}; using MAAS_ADMIN_* defaults`);
}
}
const maasPass = process.env.MAAS_ADMIN_PASS || '';
const rootPassword = maasPass;
// User 1: the MAAS admin — guaranteed password login on the console.
const adminUsername = process.env.MAAS_ADMIN_USERNAME || 'maas';
const adminPassword = maasPass;
// User 2: the deploy user (only when distinct from root/maas admin).
const deployUsername = deployId && user && user !== 'root' && user !== adminUsername ? user : '';
const deployPassword = deployUsername ? confUser.password || maasPass : '';
if (!rootPassword) {
logger.warn(
'MAAS_ADMIN_PASS is empty — console password login will not work. Set MAAS_ADMIN_PASS in .env so root/admin have a usable password.',
);
}
return { rootPassword, adminUsername, adminPassword, deployUsername, deployPassword, confUser };
},
/**
* @method readEngineConfig
* @description Reads an engine config/secret value (e.g. GITHUB_TOKEN) on the
* controller. Prefers the loaded process env, falling back to
* `node bin config get --plain <key>`.
* @param {string} key - Config/env key name.
* @returns {string} The resolved value, or '' if unset.
* @memberof UnderpostBaremetal
*/
readEngineConfig(key) {
if (process.env[key]) return process.env[key];
const out = shellExec(`node bin config get --plain ${key}`, {
stdout: true,
silent: true,
silentOnError: true,
disableLog: true,
});
const value = `${out || ''}`.trim();
return value && value !== 'undefined' ? value : '';
},
/**
* @method resolveInstalledNetwork
* @description Resolves the static network config written into the deployed OS
* so the controller can reach it at a known IP after reboot. Uses the same IP
* the commission flow assigned, a /prefix from the netmask, the subnet's `.1`
* as gateway, and the configured DNS.
* @param {object} params
* @param {string} params.ipAddress - Static IP for the deployed OS.
* @param {string} params.netmask - Dotted netmask (e.g. 255.255.255.0).
* @param {string} params.dnsServer - DNS server.
* @returns {{ netIp: string, netPrefix: number, netGateway: string, netDns: string }}
* @memberof UnderpostBaremetal
*/
resolveInstalledNetwork({ ipAddress, netmask, dnsServer }) {
const prefix = `${netmask || ''}`
.split('.')
.reduce((acc, oct) => acc + ((parseInt(oct, 10).toString(2).match(/1/g) || []).length || 0), 0);
const netGateway = `${ipAddress || ''}`.replace(/\.\d+$/, '.1');
return { netIp: ipAddress || '', netPrefix: prefix || 24, netGateway, netDns: dnsServer || '8.8.8.8' };
},
/**
* @method commissioningWriteFilesFactory
* @description Generates the write_files configuration for the commissioning script.
* @param {object} params
* @param {object} params.machine - The machine object.
* @param {object} params.authCredentials - MAAS authentication credentials.
* @param {string} params.runnerHostIp - The IP address of the runner host.
* @memberof UnderpostBaremetal
* @returns {Array} The write_files array.
*/
commissioningWriteFilesFactory({ machine, authCredentials, runnerHostIp }) {
const { consumer_key, token_key, token_secret } = authCredentials;
return [
{
path: '/usr/local/bin/underpost-enlist.sh',
permissions: '0755',
owner: 'root:root',
content: `#!/bin/bash
# set -euo pipefail
CONSUMER_KEY="${consumer_key}"
TOKEN_KEY="${token_key}"
TOKEN_SECRET="${token_secret}"
LOG_FILE="/var/log/underpost-enlistment.log"
RESPONSE_FILE="/tmp/maas_response.txt"
STATUS_FILE="/tmp/maas_status.txt"
echo "Starting MAAS Commissioning Request..." | tee -a "$LOG_FILE"
curl -X POST \\
--location --verbose \\
--header "Authorization: OAuth oauth_version=\\"1.0\\", oauth_signature_method=\\"PLAINTEXT\\", oauth_consumer_key=\\"$CONSUMER_KEY\\", oauth_token=\\"$TOKEN_KEY\\", oauth_signature=\\"&$TOKEN_SECRET\\", oauth_nonce=\\"$(uuidgen)\\", oauth_timestamp=\\"$(date +%s)\\"" \\
-F "enable_ssh=1" \\
http://${runnerHostIp}:5240/MAAS/api/2.0/machines/${machine.system_id}/op-commission \\
--output "$RESPONSE_FILE" --write-out "%{http_code}" > "$STATUS_FILE" 2>> "$LOG_FILE"
HTTP_STATUS=$(cat "$STATUS_FILE")
echo "HTTP Status: $HTTP_STATUS" | tee -a "$LOG_FILE"
echo "Response Body:" | tee -a "$LOG_FILE"
cat "$RESPONSE_FILE" | tee -a "$LOG_FILE"
if [ "$HTTP_STATUS" -eq 200 ]; then
echo "Commissioning requested successfully." | tee -a "$LOG_FILE"
else
echo "ERROR: MAAS commissioning failed with status $HTTP_STATUS" | tee -a "$LOG_FILE"
exit 0
fi
`,
},
];
},
/**
* @method httpBootstrapServerStaticFactory
* @description Creates static files for the bootstrap HTTP server including cloud-init configuration.
* @param {object} params - Parameters for creating static files.
* @param {string} params.bootstrapHttpServerPath - The path where static files will be created.
* @param {string} params.hostname - The hostname of the client machine.
* @param {string} params.cloudConfigSrc - The cloud-init configuration YAML source.
* @param {string} params.kickstartSrc - The kickstart configuration content.
* @param {string} params.vendorData - The cloud-init vendor-data content.
* @param {string} params.isoUrl - Optional ISO URL to cache and serve.
* @memberof UnderpostBaremetal
* @returns {void}
*/
httpBootstrapServerStaticFactory({
bootstrapHttpServerPath = '',
hostname = '',
cloudConfigSrc = '',
kickstartSrc = '',
vendorData = '',
isoUrl = '',
}) {
shellExec(`mkdir -p ${bootstrapHttpServerPath}/${hostname}`);
Underpost.cloudInit.httpServerStaticFactory({ bootstrapHttpServerPath, hostname, cloudConfigSrc, vendorData });
Underpost.kickstart.httpServerStaticFactory({ bootstrapHttpServerPath, hostname, kickstartSrc });
if (isoUrl) {
const isoFilename = isoUrl.split('/').pop();
const isoCacheDir = `/var/tmp/live-iso`;
const isoCachePath = `${isoCacheDir}/${isoFilename}`;
const isoDestPath = `${bootstrapHttpServerPath}/${hostname}/${isoFilename}`;
if (!fs.existsSync(isoCachePath)) {
logger.info(`Downloading ISO to cache: ${isoUrl}`);
shellExec(`mkdir -p ${isoCacheDir}`);
shellExec(`wget --progress=bar:force -O ${isoCachePath} "${isoUrl}"`);
}
logger.info(`Copying ISO to bootstrap server: ${isoDestPath}`);
shellExec(`cp ${isoCachePath} ${isoDestPath}`);
}
},
/**
* @method httpBootstrapServerRunnerFactory
* @description Starts a simple HTTP server to serve boot files for network booting.
* @param {object} options - Options for the HTTP server.
* @param {string} hostname - The hostname of the client machine.
* @param {string} options.bootstrapHttpServerPath - The path to serve files from (default: './public/localhost').
* @param {number} options.bootstrapHttpServerPort - The port on which to start the HTTP server (default: 8888).
* @memberof UnderpostBaremetal
* @returns {void}
*/
httpBootstrapServerRunnerFactory(
options = { hostname: 'localhost', bootstrapHttpServerPath: './public/localhost', bootstrapHttpServerPort: 8888 },
) {
const port = options.bootstrapHttpServerPort || 8888;
const bootstrapHttpServerPath = options.bootstrapHttpServerPath || './public/localhost';
const hostname = options.hostname || 'localhost';
shellExec(`node bin run kill ${port}`, { silent: true });
const app = express();
app.use(loggerMiddleware(import.meta, 'debug', () => false));
// Lifecycle event sink. The ephemeral Kickstart/Anaconda runtime POSTs
// stage transitions (ssh-ready, installing, completed, …) here so the
// controller can observe progress without polling the machine itself.
// Events are kept in-memory (consumed by the orchestration step) and
// mirrored to disk for auditing.
app.use(express.json({ limit: '256kb' }));
app.post('/:hostname/status', (req, res) => {
const reqHostname = req.params.hostname;
const event = {
...(req.body && typeof req.body === 'object' ? req.body : {}),
receivedAt: new Date().toISOString(),
remoteIp: (req.headers['x-forwarded-for'] || req.socket?.remoteAddress || '').toString(),
};
const stage = event.stage || 'unknown';
const events = UnderpostBaremetal.bootstrapStatusEvents.get(reqHostname) || [];
events.push(event);
UnderpostBaremetal.bootstrapStatusEvents.set(reqHostname, events);
try {
const statusDir = `${bootstrapHttpServerPath}/${reqHostname}/status`;
fs.ensureDirSync(statusDir);
fs.writeFileSync(`${statusDir}/${stage}.json`, JSON.stringify(event, null, 2), 'utf8');
fs.appendFileSync(`${statusDir}/events.log`, `${JSON.stringify(event)}\n`, 'utf8');
} catch (error) {
logger.warn('Failed to persist bootstrap status event', { error: error.message });
}
logger.info(`Bootstrap status event [${reqHostname}] stage=${stage}`, event);
res.json({ ok: true, stage });
});
app.use('/', express.static(bootstrapHttpServerPath));
app.listen(port, () => {
logger.info(`Static file server running on port ${port}`);
});
// Configure iptables to allow incoming LAN connections
shellExec(
`sudo iptables -I INPUT 1 -p tcp -s 192.168.1.0/24 --dport ${port} -m conntrack --ctstate NEW -j ACCEPT`,
);
// Option for any host:
// sudo iptables -I INPUT 1 -p tcp --dport ${port} -m conntrack --ctstate NEW -j ACCEPT
shellExec(`sudo chown -R $(whoami):$(whoami) ${bootstrapHttpServerPath}`);
shellExec(`sudo sudo chmod 755 ${bootstrapHttpServerPath}`);
logger.info(`Started Bootstrap Http Server on port ${port}`);
},
/**
* @method updateKernelFiles
* @description Copies EFI bootloaders, kernel, and initrd images to the TFTP root path.
* It also handles decompression of the kernel if necessary for ARM64 compatibility,
* and extracts raw kernel images from PE32+ EFI wrappers (common in Rocky Linux ARM64).
* @param {object} params - The parameters for the function.
* @param {object} params.commissioningImage - The commissioning image configuration.
* @param {string} params.resourcesPath - The path where resources are located.
* @param {string} params.tftpRootPath - The TFTP root path.
* @param {object} params.kernelFilesPaths - Paths to kernel files.
* @memberof UnderpostBaremetal
* @returns {void}
*/
updateKernelFiles({ commissioningImage, resourcesPath, tftpRootPath, kernelFilesPaths }) {
// Copy EFI bootloaders to TFTP path.
const arch = resourcesPath.split('/').pop();
const efiFiles =
arch === 'arm64' || arch === 'aarch64' ? ['bootaa64.efi', 'grubaa64.efi'] : ['bootx64.efi', 'grubx64.efi'];
for (const file of efiFiles) {
shellExec(`sudo cp -a ${resourcesPath}/${file} ${tftpRootPath}/pxe/${file}`);
}
// Copy kernel and initrd images to TFTP path.
for (const file of Object.keys(kernelFilesPaths)) {
if (file == 'isoUrl') continue; // Skip URL entries
shellExec(`sudo cp -a ${kernelFilesPaths[file]} ${tftpRootPath}/pxe/${file}`);
// If the file is a kernel (vmlinuz-efi) and is gzipped, unzip it for GRUB compatibility on ARM64.
// GRUB on ARM64 often crashes with synchronous exception (0x200) if handling large compressed kernels directly.
if (file === 'vmlinuz-efi') {
const kernelDest = `${tftpRootPath}/pxe/${file}`;
const fileType = shellExec(`file ${kernelDest}`).stdout;
// Handle gzip compressed kernels
if (fileType.includes('gzip compressed data')) {
logger.info(`Decompressing kernel ${file} for ARM64 UEFI compatibility...`);
shellExec(`sudo mv ${kernelDest} ${kernelDest}.gz`);
shellExec(`sudo gunzip ${kernelDest}.gz`);
}
// Handle PE32+ EFI wrapped kernels (common in Rocky Linux ARM64)
// Rocky Linux ARM64 kernels are distributed as PE32+ EFI executables, which
// are bootable directly via UEFI firmware. However, GRUB's 'linux' command
// expects a raw ARM64 Linux kernel Image format, not a PE32+ wrapper.
if (fileType.includes('PE32+') || fileType.includes('EFI application')) {
logger.warn('Detected PE32+ EFI wrapped kernel. Need to extract raw kernel image for GRUB.');
}
}
}
},
/**
* @method kernelCmdBootParamsFactory
* @description Constructs kernel command line parameters for NFS booting.
* @param {object} options - Options for constructing the command line.
* @param {string} options.ipClient - The IP address of the client.
* @param {string} options.ipDhcpServer - The IP address of the DHCP server.
* @param {string} options.ipFileServer - The IP address of the file server.
* @param {string} options.ipConfig - The IP configuration method (e.g., 'dhcp').
* @param {string} options.netmask - The network mask.
* @param {string} options.hostname - The hostname of the client.
* @param {string} options.dnsServer - The DNS server address.
* @param {string} options.networkInterfaceName - The name of the network interface.
* @param {string} options.fileSystemUrl - The URL of the root filesystem.
* @param {number} options.bootstrapHttpServerPort - The port of the bootstrap HTTP server.
* @param {string} options.type - The type of boot ('iso-ram', 'chroot-debootstrap', 'chroot-container', 'iso-nfs', etc.).
* @param {string} options.macAddress - The MAC address of the client.
* @param {boolean} options.cloudInit - Whether to include cloud-init parameters.
* @param {object} options.machine - The machine object containing system_id.
* @param {boolean} [options.dev=false] - Whether to enable dev mode with dracut debugging parameters.
* @param {string} [options.osIdLike=''] - OS family identifier (e.g., 'rhel centos fedora' or 'debian ubuntu').
* @param {object} options.authCredentials - Authentication credentials for fetching files (if needed).
* @param {string} options.authCredentials.consumer_key - Consumer key for authentication.
* @param {string} options.authCredentials.consumer_secret - Consumer secret for authentication.
* @param {string} options.authCredentials.token_key - Token key for authentication.
* @param {string} options.authCredentials.token_secret - Token secret for authentication.
* @param {string} options.architecture - The architecture of the machine (e.g., 'amd64', 'arm64').
* @returns {object} An object containing the constructed command line string.
* @memberof UnderpostBaremetal
*/
kernelCmdBootParamsFactory(
options = {
ipClient: '',
ipDhcpServer: '',
ipFileServer: '',
ipConfig: '',
netmask: '',
hostname: '',
dnsServer: '',
networkInterfaceName: '',
fileSystemUrl: '',
bootstrapHttpServerPort: 8888,
type: '',
macAddress: '',
cloudInit: false,
dev: false,
osIdLike: '',
authCredentials: { consumer_key: '', consumer_secret: '', token_key: '', token_secret: '' },
architecture,
},
) {
// Construct kernel command line arguments for NFS boot.
const {
ipClient,
ipDhcpServer,
ipFileServer,
ipConfig,
netmask,
hostname,
dnsServer,
networkInterfaceName,
fileSystemUrl,
bootstrapHttpServerPort,
type,
macAddress,
cloudInit,
osIdLike,
architecture,
} = options;
// Determine OS family from osIdLike
const { isDebianBased, isRhelBased } = Underpost.baremetal.getFamilyBaseOs(options.osIdLike);
const ifaceName = networkInterfaceName ? networkInterfaceName : 'eth0';
const isStaticIp = ipConfig === 'none' || ipConfig === 'off';
const ipParam = isStaticIp
? `ip=${ipClient}:${ipFileServer}:${ipDhcpServer}:${netmask}:${hostname}:${ifaceName}:${ipConfig}:${dnsServer}`
: `ip=::::${hostname}:${ifaceName}:${ipConfig}`;
let nfsMountOptions = [];
if (type === 'chroot-debootstrap' || type === 'chroot-container') {
nfsMountOptions = [
'tcp',
'nfsvers=3',
'nolock',
'port=2049',
'hard',
'intr',
'rsize=32768',
'wsize=32768',
'acregmin=0',
'acregmax=0',
'acdirmin=0',
'acdirmax=0',
'noac',
];
} else if (type === 'iso-nfs' && isDebianBased) {
nfsMountOptions = ['nolock', 'nfsvers=3', 'tcp', 'hard', 'port=2049', 'rsize=32768', 'wsize=32768'];
}
const nfsOptions = nfsMountOptions.join(',');
const nfsServerPath = `${ipFileServer}:${process.env.NFS_EXPORT_PATH}/${hostname}`;
const nfsRootParam = `nfsroot=${nfsServerPath}${nfsOptions ? `,${nfsOptions}` : ''}`;
const casperNfsParams = [`nfsroot=${nfsServerPath}`, ...(nfsOptions ? [`nfsopts=${nfsOptions}`] : [])];
const permissionsParams = [
`rw`,
// `ro`
];
const kernelParams = [
...permissionsParams,
`ignore_uuid`,
`rootwait`,
`ipv6.disable=1`,
`fixrtc`,
// `console=serial0,115200`,
// `console=tty1`,
// `layerfs-path=filesystem.squashfs`,
// `root=/dev/ram0`,
// `toram`,
// 'nomodeset',
// `editable_rootfs=tmpfs`, // all writes to rootfs go to RAM, keeping underlying storage pristine
// `ramdisk_size=3550000`,
// `root=/dev/sda1`, // rpi4 usb port unit
'apparmor=0', // Disable AppArmor security
...(networkInterfaceName === 'eth0'
? [
'net.ifnames=0', // Disable predictable network interface names
'biosdevname=0', // Disable BIOS device naming
]
: []),
];
const performanceParams = [
// --- Boot Automation & Stability ---
'auto=true', // Enable automated installation/configuration
'noeject', // Do not attempt to eject boot media on reboot
`casper-getty`, // Enable console login for live sessions
'nowatchdog', // Disable watchdog timers to prevent unexpected reboots
'noprompt', // Don't wait for "Press Enter" during boot/reboot
// --- CPU & System Performance ---
'mitigations=off', // Disable CPU security mitigations for maximum speed
'clocksource=tsc', // Use fastest available hardware clock
'tsc=reliable', // Trust CPU clock without extra verification
'hpet=disable', // Disable legacy slow timer
'nohz=on', // Reduce overhead by disabling timer ticks on idle CPUs
// --- Memory & Hardware Optimization ---
'cma=40M', // Reserve contiguous RAM for RPi hardware/video
'zswap.enabled=1', // Use compressed RAM cache (vital for NFS/SD)
'zswap.compressor=zstd', // Best balance of speed and compression
'zswap.max_pool_percent=30', // Use max 30% of RAM as compressed storage
'zswap.zpool=zsmalloc', // Efficient memory management for zswap
'fsck.mode=skip', // Skip disk checks to accelerate boot
'max_loop=255', // Ensure enough loop devices for squashfs/snaps
// --- Immutable Filesystem ---
'overlayroot=tmpfs', // Run entire OS in RAM to protect storage
'overlayroot_cfgdisk=disabled', // Ignore external overlay configurations
];
let cmd = [];
if (type === 'iso-ram') {
if (isRhelBased) {
cmd = Underpost.kickstart.kernelParamsFactory(macAddress, [ipParam, ...kernelParams], options);
} else {
// ISO-RAM (Debian/Ubuntu): full live ISO downloaded into RAM via casper toram.
const netBootParams = [`netboot=url`];
if (fileSystemUrl) netBootParams.push(`url=${fileSystemUrl.replace('https', 'http')}`);
cmd = [ipParam, `boot=casper`, 'toram', ...netBootParams, ...kernelParams, ...performanceParams];
}
} else if (type === 'chroot-debootstrap' || type === 'chroot-container') {
let qemuNfsRootParams = [`root=/dev/nfs`, `rootfstype=nfs`];
cmd = [ipParam, ...qemuNfsRootParams, nfsRootParam, ...kernelParams];
} else {
// 'iso-nfs' — kernel/initrd from ISO, root filesystem served via NFS.
if (isDebianBased) {
cmd = [ipParam, `boot=casper`, `netboot=nfs`, ...casperNfsParams, ...kernelParams];
} else {
cmd = [ipParam, `netboot=nfs`, nfsRootParam, ...kernelParams, ...performanceParams];
}
}
// Add RHEL/Rocky/Fedora based images specific parameters
if (isRhelBased) {
cmd = cmd.concat([`rd.neednet=1`, `rd.timeout=180`, `selinux=0`, `enforcing=0`]);
if (options.dev) cmd = cmd.concat([`rd.shell`, `rd.debug`]);
}
// Add Debian/Ubuntu based images specific parameters
else if (isDebianBased) {
if (type !== 'iso-nfs') cmd = cmd.concat([`initrd=initrd.img`, `init=/sbin/init`]);
if (options.dev) cmd = cmd.concat([`debug`, `ignore_loglevel`]);
}
if (cloudInit) cmd = Underpost.cloudInit.kernelParamsFactory(macAddress, cmd, options);
// cmd.push('---');
const cmdStr = cmd.join(' ');
logger.info('Constructed kernel command line');
console.log(newInstance(cmdStr).bgRed.bold.black);
return { cmd: cmdStr };
},
/**
* @method waitForBootstrapStage
* @description Polls the in-memory bootstrap status events (populated by the
* bootstrap HTTP server POST sink) until the target stage is reported by the
* ephemeral runtime, or the timeout elapses.
* @param {object} params
* @param {string} params.hostname - Hostname key the runtime reports under.
* @param {string} params.stage - Stage to wait for (e.g. 'ssh-ready').
* @param {number} [params.timeoutMs=900000] - Maximum wait time in ms.
* @param {number} [params.intervalMs=2000] - Poll interval in ms.
* @returns {Promise<object|null>} The matching event, or null on timeout.
* @memberof UnderpostBaremetal
*/
async waitForBootstrapStage({ hostname, stage, timeoutMs = 15 * 60 * 1000, intervalMs = 2000 }) {
const deadline = Date.now() + timeoutMs;
let lastSeenCount = -1;
let lastHeartbeat = 0;
const heartbeatMs = 15000;
while (Date.now() < deadline) {
const events = UnderpostBaremetal.bootstrapStatusEvents.get(hostname) || [];
const match = events.find((e) => e.stage === stage);
if (match) return match;
const now = Date.now();
if (events.length !== lastSeenCount || now - lastHeartbeat >= heartbeatMs) {
lastSeenCount = events.length;
lastHeartbeat = now;
const remainingSec = Math.max(0, Math.round((deadline - now) / 1000));
logger.info(`Waiting for stage '${stage}' from ${hostname}`, {
stagesSeen: events.map((e) => e.stage),
eventCount: events.length,
remainingSec,
});
}
await timer(intervalMs);
}
logger.warn(`Timed out waiting for bootstrap stage '${stage}' from ${hostname}`, {
stagesSeen: (UnderpostBaremetal.bootstrapStatusEvents.get(hostname) || []).map((e) => e.stage),
});
return null;
},
/**
* @method commissionMonitor
* @description Monitors the MAAS discoveries and initiates machine creation and commissioning
* once a matching MAC address is found. It also opens terminal windows for live logs.
* @param {object} params - The parameters for the function.
* @param {string} params.macAddress - The MAC address to monitor for.
* @param {string} params.ipAddress - The IP address of the machine (used if MAC is all zeros).
* @param {string} [params.hostname] - The hostname for the machine (optional).
* @param {string} [params.architecture] - The architecture of the machine (optional).
* @param {object} [params.machine] - Existing machine payload to use (optional).
* @returns {Promise<void>} A promise object with machine and discovery details.
* @memberof UnderpostBaremetal
*/
async commissionMonitor({ macAddress, ipAddress, hostname, architecture, machine }) {
{
// Query observed discoveries from MAAS.
const discoveries = Underpost.baremetal.maasCliExec(`discoveries read`);
for (const discovery of discoveries) {
const discoverHostname = discovery.hostname
? discovery.hostname
: discovery.mac_organization
? discovery.mac_organization
: discovery.domain
? discovery.domain
: `generic-host-${s4()}${s4()}`;
console.log(discoverHostname.bgBlue.bold.white);
console.log('ip target:'.green + ipAddress, 'ip discovered:'.green + discovery.ip);
console.log('mac target:'.green + macAddress, 'mac discovered:'.green + discovery.mac_address);
if (discovery.ip === ipAddress) {
logger.info('Machine discovered!', discovery);
if (!machine) {
// Check if a machine with the discovered MAC already exists to avoid conflicts
const [existingMachine] =
Underpost.baremetal.maasCliExec(`machines read mac_address=${discovery.mac_address}`) || [];
if (existingMachine) {
logger.warn(
`Machine ${existingMachine.hostname} (${existingMachine.system_id}) already exists with MAC ${discovery.mac_address}`,
);
logger.info(
`Deleting existing machine ${existingMachine.system_id} to create new machine ${hostname}...`,
);
Underpost.baremetal.maasCliExec(`machine delete ${existingMachine.system_id}`);
}
logger.info('Creating new machine with discovered hardware MAC...', {
discoveredMAC: discovery.mac_address,
ipAddress,
hostname,
});
machine = Underpost.baremetal.machineFactory({
ipAddress,
macAddress: discovery.mac_address,
hostname,
architecture,
}).machine;
if (machine && machine.system_id) {
console.log('New machine system id:', machine.system_id.bgYellow.bold.black);
Underpost.baremetal.writeGrubConfigToFile({
grubCfgSrc: Underpost.baremetal
.getGrubConfigFromFile()
.grubCfgSrc.replaceAll('system-id', machine.system_id),
});
} else {
logger.error('Failed to create machine or obtain system_id', machine);
throw new Error('Machine creation failed');
}
} else {
const systemId = machine.system_id;
console.log('Using pre-registered machine system_id:', systemId.bgYellow.bold.black);
// Update the boot interface MAC if hardware MAC differs from pre-registered MAC
// This handles both hardware mode (macAddress is null) and MAC mismatch scenarios
if (macAddress === null || macAddress !== discovery.mac_address) {
logger.info('Updating machine interface with discovered hardware MAC...', {
preRegisteredMAC: macAddress || 'none (hardware mode)',
discoveredMAC: discovery.mac_address,
});
// Check current machine status before attempting state transitions
const currentMachine = Underpost.baremetal.maasCliExec(`machine read ${systemId}`);
const currentStatus = currentMachine ? currentMachine.status_name : 'Unknown';
logger.info('Current machine status before interface update:', { systemId, status: currentStatus });
// Only mark-broken if the machine is in a state that supports it (e.g. Ready, New, Allocated)
// Machines already in Broken state don't need to be marked broken again
if (currentStatus !== 'Broken') {
try {
Underpost.baremetal.maasCliExec(`machine mark-broken ${systemId}`);
logger.info('Machine marked as broken successfully');
} catch (markBrokenError) {
logger.warn('Failed to mark machine as broken, attempting interface update anyway...', {
error: markBrokenError.message,
currentStatus,
});
}
} else {
logger.info('Machine is already in Broken state, skipping mark-broken');
}
Underpost.baremetal.maasCliExec(
`interface update ${systemId} ${machine.boot_interface.id}` + ` mac_address=${discovery.mac_address}`,
);
// Re-check status before mark-fixed — only attempt if actually Broken
const updatedMachine = Underpost.baremetal.maasCliExec(`machine read ${systemId}`);
const updatedStatus = updatedMachine ? updatedMachine.status_name : 'Unknown';
if (updatedStatus === 'Broken') {
try {
Underpost.baremetal.maasCliExec(`machine mark-fixed ${systemId}`);
logger.info('Machine marked as fixed successfully');
} catch (markFixedError) {
logger.warn('Failed to mark machine as fixed:', { error: markFixedError.message });
}
} else {
logger.info('Machine is not in Broken state, skipping mark-fixed', { status: updatedStatus });
}
logger.info('✓ Machine interface MAC address updated successfully');
// commissioning_scripts=90-verify-user.sh
}
logger.info('Machine resource uri', machine.resource_uri);
for (const iface of machine.interface_set)
logger.info('Interface info', {
name: iface.name,
mac_address: iface.mac_address,
resource_uri: iface.resource_uri,
});
}
return { discovery, machine };
}
}
await timer(1000);
return await Underpost.baremetal.commissionMonitor({
macAddress,
ipAddress,
hostname,
architecture,
machine,
});
}
},
/**
* @method maasCliExec
* @description Executes a MAAS CLI command and returns the parsed JSON output.
* This method abstracts the execution of MAAS CLI commands, ensuring that the output is captured and parsed correctly.
* @param {string} cmd - The MAAS CLI command to execute (e.g., 'machines read').
* @returns {object|null} The parsed JSON output from the MAAS CLI command, or null if there is no output.
* @memberof UnderpostBaremetal
*/
maasCliExec(cmd) {
const output = shellExec(`maas ${process.env.MAAS_ADMIN_USERNAME} ${cmd}`, {
stdout: true,
silent: true,
}).trim();
try {
return output ? JSON.parse(output) : null;
} catch (error) {
console.log('output', output);
logger.error(error);
throw error;
}
},
/**
* @method maasAuthCredentialsFactory
* @description Retrieves MAAS API key credentials from the MAAS CLI.
* This method parses the output of `maas apikey` to extract the consumer key,
* consumer secret, token key, and token secret.
* @returns {object} An object containing the MAAS authentication credentials.
* @memberof UnderpostBaremetal
* @throws {Error} If the MAAS API key format is invalid.
*/
maasAuthCredentialsFactory() {
// Expected formats:
// <consumer_key>:<consumer_token>:<secret> (older format)
// <consumer_key>:<consumer_secret>:<token_key>:<token_secret> (newer format)
// Commands used to generate API keys:
// maas apikey --with-names --username ${process.env.MAAS_ADMIN_USERNAME}
// maas ${process.env.MAAS_ADMIN_USERNAME} account create-authorisation-token
// maas apikey --generate --username ${process.env.MAAS_ADMIN_USERNAME}
// Reference: https://github.com/CanonicalLtd/maas-docs/issues/647
const parts = shellExec(`maas apikey --with-names --username ${process.env.MAAS_ADMIN_USERNAME}`, {
stdout: true,
})
.trim()
.split(`\n`)[0] // Take only the first line of output.
.split(':'); // Split by colon to get individual parts.
let consumer_key, consumer_secret, token_key, token_secret;
// Determine the format of the API key and assign parts accordingly.
if (parts.length === 4) {
[consumer_key, consumer_secret, token_key, token_secret] = parts;
} else if (parts.length === 3) {
// Handle older 3-part format, setting consumer_secret as empty.
[consumer_key, token_key, token_secret] = parts;
consumer_secret = '';
token_secret = token_secret.split(' MAAS consumer')[0].trim(); // Clean up token secret.
} else {
// Throw an error if the format is not recognized.
throw new Error('Invalid token format');
}
logger.info('Maas api token generated', { consumer_key, consumer_secret, token_key, token_secret });
return { consumer_key, consumer_secret, token_key, token_secret };
},
/**
* @method mountBinfmtMisc
* @description Mounts the binfmt_misc filesystem to enable QEMU user-static binfmt support.
* This is necessary for cross-architecture execution within a chroot environment.
* @param {object} params - The parameters for the function.
* @memberof UnderpostBaremetal
* @returns {void}
*/
mountBinfmtMisc() {
// Install necessary packages for debootstrap and QEMU.
shellExec(`sudo dnf install -y iptables-legacy`);
shellExec(`sudo dnf install -y debootstrap`);
shellExec(`sudo dnf install -y kernel-modules-extra-$(uname -r)`);
// Reset QEMU user-static binfmt for proper cross-architecture execution.
shellExec(`sudo podman run --rm --privileged docker.io/multiarch/qemu-user-static:latest --reset -p yes`);
// Mount binfmt_misc filesystem.
shellExec(`sudo modprobe binfmt_misc`);
shellExec(`sudo mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc`, { silentOnError: true });
},
/**
* @method removeMachines
* @description Deletes all specified machines from MAAS.
* @param {object} params - The parameters for the function.
* @param {Array<object>} params.machines - An array of machine objects, each with a `system_id`.
* @param {Array<string>} [params.ignore] - An optional array of system IDs to ignore during deletion.
* @memberof UnderpostBaremetal
* @returns {Array<object>} An empty array after machines are removed.
*/
removeMachines({ machines, ignore }) {
for (const machine of machines) {
// Handle both string system_ids and machine objects
const systemId = typeof machine === 'string' ? machine : machine.system_id;
if (ignore && ignore.find((mId) => mId === systemId)) continue;
logger.info(`Removing machine: ${systemId}`);
Underpost.baremetal.maasCliExec(`machine delete ${systemId}`);
}
return [];
},
/**
* @method clearDiscoveries
* @description Clears all observed discoveries in MAAS and optionally forces a new scan.
* @param {object} params - The parameters for the function.
* @param {boolean} params.force - If true, forces a new discovery scan after clearing.
* @memberof UnderpostBaremetal
* @returns {void}
*/
clearDiscoveries({ force }) {
Underpost.baremetal.maasCliExec(`discoveries clear all=true`);
if (force === true) {
Underpost.baremetal.maasCliExec(`discoveries scan force=true`);
}
},
/**
* @method macMonitor
* @description Monitors for the presence of a MAC address file in the NFS host path.
* This is used to wait for the target machine to report its MAC address.
* @param {object} params - The parameters for the function.
* @param {string} params.nfsHostPath - The NFS host path where the MAC file is expected.
* @memberof UnderpostBaremetal
* @returns {Promise<void>} A promise that resolves when the MAC file is found or after a delay.
*/
async macMonitor({ nfsHostPath }) {
if (fs.existsSync(`${nfsHostPath}/underpost/mac`)) {
const mac = fs.readFileSync(`${nfsHostPath}/underpost/mac`, 'utf8').trim();
logger.info('Commissioning MAC', mac);
return;
}
await timer(1000);
await Underpost.baremetal.macMonitor({ nfsHostPath });
},
/**
* @method installGrubModules
* @description Installs the necessary GRUB modules for both ARM64 and AMD64 architectures.
* This ensures that the GRUB bootloader can properly load the kernel and initrd images
* during the network boot process, regardless of the target architecture.
* @memberof UnderpostBaremetal
* @returns {void}
*/
installGrubModules() {
if (!fs.existsSync('/usr/lib/grub/x86_64-efi')) shellExec(`sudo dnf install -y grub2-efi-x64-modules`);
if (!fs.existsSync('/usr/lib/grub/arm64-efi')) shellExec(`sudo dnf install -y grub2-efi-aa64-modules`);
},
/**
* @method crossArchBinFactory
* @description Copies the appropriate QEMU static binary into the NFS root filesystem
* for cross-architecture execution within a chroot environment.
* @param {object} params - The parameters for the function.
* @param {string} params.nfsHostPath - The path to the NFS root filesystem on the host.
* @memberof UnderpostBaremetal
* @param {'arm64'|'amd64'} params.debootstrapArch - The target architecture of the debootstrap environment.
* @returns {void}
*/
crossArchBinFactory({ nfsHostPath, bootstrapArch }) {
switch (bootstrapArch) {
case 'arm64':
// Copy QEMU static binary for ARM64.
shellExec(`sudo podman cp extract:/usr/bin/qemu-aarch64-static ${nfsHostPath}/usr/bin/`);
break;
case 'amd64':
// Copy QEMU static binary for AMD64.
shellExec(`sudo podman cp extract:/usr/bin/qemu-x86_64-static ${nfsHostPath}/usr/bin/`);
break;
default:
// Log a warning or throw an error for unsupported architectures.
logger.warn(`Unsupported bootstrap architecture: ${bootstrapArch}`);
break;
}
},
/**
* @method crossArchRunner
* @description Executes a series of shell commands within a chroot environment,
* optionally using QEMU for cross-architecture execution.
* @param {object} params - The parameters for the function.
* @param {string} params.nfsHostPath - The path to the NFS root filesystem on the host.
* @param {'arm64'|'amd64'} params.debootstrapArch - The target architecture of the debootstrap environment.
* @param {object} params.callbackMetaData - Metadata about the callback, including runner host architecture.
* @memberof UnderpostBaremetal
* @param {string[]} params.steps - An array of shell commands to execute.
* @returns {void}
*/
crossArchRunner({ nfsHostPath, bootstrapArch, callbackMetaData, steps }) {
// Render the steps with logging for better visibility during execution.
steps = Underpost.baremetal.stepsRender(steps, false);
let qemuCrossArchBash = '';
// Determine if QEMU is needed for cross-architecture execution.
if (bootstrapArch !== callbackMetaData.runnerHost.architecture)
switch (bootstrapArch) {
case 'arm64':
qemuCrossArchBash = '/usr/bin/qemu-aarch64-static ';
break;
case 'amd64':
qemuCrossArchBash = '/usr/bin/qemu-x86_64-static ';
break;
default:
// No QEMU prefix for unsupported or native architectures.
break;
}
// Execute the commands within the chroot environment using a heredoc.
shellExec(`sudo chroot ${nfsHostPath} ${qemuCrossArchBash}/bin/bash <<'EOF'
${steps}
EOF`);
},
/**
* @method stepsRender
* @description Renders an array of shell commands into a formatted string,
* optionally including YAML-style formatting and execution logging.
* This helps in visualizing and debugging the execution flow of provisioning steps.
* @param {string[]} [steps=[]] - An array of shell commands.
* @param {boolean} [yaml=true] - If true, formats the output as YAML list items.
* @memberof UnderpostBaremetal
* @returns {string} The formatted string of commands.
*/
stepsRender(steps = [], yaml = true) {
return steps
.map(
(step, i, a) =>
// Add a timestamp and step counter for better logging and traceability.
(yaml ? ' - ' : '') +
'echo "' +
(yaml ? '\\' : '') +
'$(date) | ' +
(i + 1) +
'/' +
a.length +
' - ' +
step.split('\n')[0] +
'"' +
`\n` +
`${yaml ? ' - ' : ''}${step}`,
)
.join('\n');
},
/**
* @method nfsMountCallback
* @description Manages NFS mounts and unmounts for the baremetal provisioning process.
* It checks the mount status and performs mount/unmount operations as requested.
* @param {object} params - The parameters for the function.
* @param {string} params.hostname - The hostname of the target machine.
* @param {string} params.nfsHostPath - The NFS host path for the target machine.
* @param {string} params.workflowId - The identifier for the workflow configuration.
* @param {boolean} [params.mount] - If true, attempts to mount the NFS paths.
* @param {boolean} [params.unmount] - If true, attempts to unmount the NFS paths.
* @param {number} [currentRecall=0] - The current recall attempt count for retries.
* @param {number} [maxRecalls=5] - The maximum number of recall attempts allowed.
* @memberof UnderpostBaremetal
* @returns {Promise<void>} A promise that resolves when the mount/unmount operations are complete.
*/
async nfsMountCallback({ hostname, nfsHostPath, workflowId, mount, unmount }, currentRecall = 0, maxRecalls = 5) {
// Mount binfmt_misc filesystem.
if (mount) Underpost.baremetal.mountBinfmtMisc();
const unMountCmds = [];
const mountCmds = [];
const workflowsConfig = Underpost.baremetal.loadWorkflowsConfig();
let recall = false;
if (!workflowsConfig[workflowId]) {
throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
}
if (
workflowsConfig[workflowId].type === 'chroot-debootstrap' ||
workflowsConfig[workflowId].type === 'chroot-container'
) {
const mounts = {
bind: ['/proc', '/sys', '/run'],
rbind: ['/dev'],
};
for (const mountCmd of Object.keys(mounts)) {
for (const mountPath of mounts[mountCmd]) {
const hostMountPath = `${process.env.NFS_EXPORT_PATH}/${hostname}${mountPath}`;
// Check if the path is already mounted using `mountpoint` command.
// `mountpoint` exits 1 when the path is not a mountpoint — silentOnError
// prevents ShellExecError so we can inspect stdout/stderr for the string.
const mountpointOut = shellExec(`mountpoint ${hostMountPath}`, {
silent: true,
stdout: true,
silentOnError: true,
});
const isPathMounted =
typeof mountpointOut === 'string' && mountpointOut.length > 0
? !mountpointOut.match('not a mountpoint') && !mountpointOut.match('No such file')
: false;
if (isPathMounted) {
logger.warn('Nfs path already mounted', mountPath);
if (unmount === true) {
// Unmount if requested.
unMountCmds.push(`sudo umount -Rfl ${hostMountPath}`);
if (!recall) recall = true;
}
} else {
if (mount === true) {
// Mount if requested and not already mounted.
mountCmds.push(`sudo mount --${mountCmd} ${mountPath} ${hostMountPath}`);
} else {
logger.warn('Nfs path not mounted', mountPath);
}
}
}
}
for (const unMountCmd of unMountCmds) shellExec(unMountCmd);
if (recall) {
if (currentRecall >= maxRecalls) {
throw new Error(
`Maximum recall attempts (${maxRecalls}) reached for nfsMountCallback. Hostname: ${hostname}`,
);
}
logger.info(`nfsMountCallback recall attempt ${currentRecall + 1}/${maxRecalls} for hostname: ${hostname}`);
await timer(1000);
return await Underpost.baremetal.nfsMountCallback(
{ hostname, nfsHostPath, workflowId, mount, unmount },
currentRecall + 1,
maxRecalls,
);
}
if (mountCmds.length > 0) {
shellExec(`sudo chown -R $(whoami):$(whoami) ${nfsHostPath}`);
shellExec(`sudo chmod -R 755 ${nfsHostPath}`);
for (const mountCmd of mountCmds) shellExec(mountCmd);
}
}
},
/**
* @method getHostArch
* @description Determines the architecture of the host machine.
* This is crucial for cross-compilation and selecting the correct QEMU binaries.
* @memberof UnderpostBaremetal
* @returns {{alias: 'amd64'|'arm64', name: 'x86_64'|'aarch64'}} The host architecture.
* @throws {Error} If the host architecture is unsupported.
*/
getHostArch() {
// `uname -m` returns e.g. 'x86_64' or 'aarch64'
const machine = shellExec('uname -m', { stdout: true, silent: true }).trim();
if (machine === 'x86_64') return { alias: 'amd64', name: 'x86_64' };
if (machine === 'aarch64') return { alias: 'arm64', name: 'aarch64' };
throw new Error(`Unsupported host architecture: ${machine}`);
},
/**
* @method rebuildNfsServer
* @description Configures NFS exports and aligns host firewall/NFS daemon ports for MAAS workflows.
* @param {object} params - The parameters for the function.
* @param {string} params.nfsHostPath - The path to the NFS server export.
* @memberof UnderpostBaremetal
* @param {string} [params.subnet='192.168.1.0/24'] - The subnet allowed to access the NFS export.
* @param {boolean} [params.nfsReset=false] - Flag to completely reset the NFS server (restart service).
* @param {string} [params.underpostRoot] - Repository root used to locate helper scripts.
* @returns {void}
*/
rebuildNfsServer({ nfsHostPath, subnet, nfsReset, underpostRoot }) {
if (!subnet) subnet = '192.168.1.0/24'; // Default subnet if not provided.
if (!underpostRoot) {
const __dirname = path.dirname(fileURLToPath(import.meta.url));
underpostRoot = path.resolve(__dirname, '../..');
}
const maasNatFirewalldPath = path.resolve(underpostRoot, 'scripts/maas-nat-firewalld.sh');
const nfsPorts = UnderpostBaremetal.NFS_V3_PORTS;
const exportOptions = ['rw', 'sync', 'no_root_squash', 'no_subtree_check', 'insecure'].join(',');
if (!fs.existsSync(maasNatFirewalldPath)) {
throw new Error(`MAAS firewalld helper not found: ${maasNatFirewalldPath}`);
}
// Write the NFS exports configuration to /etc/exports.
if (!fs.existsSync(nfsHostPath)) fs.mkdirSync(nfsHostPath, { recursive: true });
fs.writeFileSync(`/etc/exports`, `${nfsHostPath} ${subnet}(${exportOptions})`, 'utf8');
logger.info('Configuring MAAS firewalld and fixed NFSv3 ports...');
shellExec(
[
`MAAS_LAN_CIDR=${subnet}`,
`NFS_MODE=v3`,
`CONFIGURE_NFS_V3_PORTS=true`,
`NFS_MOUNTD_PORT=${nfsPorts.mountd}`,
`NFS_STATD_PORT=${nfsPorts.statd}`,
`NFS_STATD_OUTGOING_PORT=${nfsPorts.statdOutgoing}`,
`NFS_LOCKD_PORT=${nfsPorts.lockd}`,
`NFS_LOCKD_UDP_PORT=${nfsPorts.lockd}`,
`bash "${maasNatFirewalldPath}"`,
].join(' '),
);
logger.info('Reloading NFS exports...');
shellExec(`sudo exportfs -rav`);
// Display the currently active NFS exports for verification.
logger.info('Displaying active NFS exports');
shellExec(`sudo exportfs -s`);
if (nfsReset) {
logger.info('Restarting NFS server service...');
let restarted = false;
for (const unit of ['nfs-server', 'nfs-kernel-server']) {
const result = shellExec(`sudo systemctl restart ${unit}`, { silentOnError: true });
if (result.code === 0) {
restarted = true;
logger.info(`NFS server restarted via ${unit}.`);
break;
}
}
if (!restarted) logger.warn('Unable to restart nfs-server or nfs-kernel-server after export reload.');
}
},
/**
* @method checkQemuCrossArchSupport
* @description Checks for QEMU support when building for a different architecture.
* This is essential for validator bots that need to build images for architectures
* different from the host system (e.g., building arm64 on x86_64 or vice versa).
* @param {object} workflow - The workflow configuration object.
* @param {object} workflow.maas - The MAAS configuration.
* @param {string} workflow.maas.architecture - Target architecture (e.g., 'arm64/generic', 'amd64/generic').
* @memberof UnderpostBaremetal
* @throws {Error} If QEMU is not installed or doesn't support required machine types.
* @returns {void}
*/
checkQemuCrossArchSupport(workflow) {
// Check for QEMU support if building for a different architecture (validator bots case)
if (workflow.maas.architecture.startsWith('arm64') && process.arch !== 'arm64') {
// Building arm64/aarch64 on x86_64 host
// Check both /usr/local/bin (compiled) and system paths
let qemuAarch64Path = null;
if (shellExec('test -x /usr/local/bin/qemu-system-aarch64', { silentOnError: true }).code === 0) {
qemuAarch64Path = '/usr/local/bin/qemu-system-aarch64';
} else if (shellExec('which qemu-system-aarch64', { silentOnError: true }).code === 0) {
qemuAarch64Path = shellExec('which qemu-system-aarch64', { stdout: true }).trim();
}
if (!qemuAarch64Path) {
throw new Error(
'qemu-system-aarch64 is not installed. Please install it to build ARM64 images on x86_64 hosts.\n' +
'Run: node bin baremetal --dev --install-packer',
);
}
logger.info(`Found qemu-system-aarch64 at: ${qemuAarch64Path}`);
// Verify that the installed qemu supports the 'virt' machine type (required for arm64)
const machineHelp = shellExec(`${qemuAarch64Path} -machine help`).stdout;
if (!machineHelp.includes('virt')) {
throw new Error(
'The installed qemu-system-aarch64 does not support the "virt" machine type.\n' +
'This usually happens if qemu-system-aarch64 is a symlink to qemu-kvm on x86_64.\n' +
'Run: node bin baremetal --dev --install-packer',
);
}
} else if (workflow.maas.architecture.startsWith('amd64') && process.arch !== 'x64') {
// Building amd64/x86_64 on aarch64 host
// Check both /usr/local/bin (compiled) and system paths
let qemuX86Path = null;
if (shellExec('test -x /usr/local/bin/qemu-system-x86_64', { silentOnError: true }).code === 0) {
qemuX86Path = '/usr/local/bin/qemu-system-x86_64';
} else if (shellExec('which qemu-system-x86_64', { silentOnError: true }).code === 0) {
qemuX86Path = shellExec('which qemu-system-x86_64', { stdout: true }).trim();
}
if (!qemuX86Path) {
throw new Error(
'qemu-system-x86_64 is not installed. Please install it to build x86_64 images on aarch64 hosts.\n' +
'Run: node bin baremetal --dev --install-packer',
);
}
logger.info(`Found qemu-system-x86_64 at: ${qemuX86Path}`);
// Verify that the installed qemu supports the 'pc' or 'q35' machine type (required for x86_64)
const machineHelp = shellExec(`${qemuX86Path} -machine help`).stdout;
if (!machineHelp.includes('pc') && !machineHelp.includes('q35')) {
throw new Error(
'The installed qemu-system-x86_64 does not support the "pc" or "q35" machine type.\n' +
'Run: node bin baremetal --dev --install-packer',
);
}
}
},
/**
* @method bootConfFactory
* @description Generates the boot configuration file for specific workflows,
* primarily for Raspberry Pi 4 Model B. This configuration includes TFTP settings,
* MAC address override, and static IP configuration.
* @param {object} params - The parameters for the function.
* @param {string} params.workflowId - The identifier for the specific workflow.
* @param {string} params.tftpIp - The IP address of the TFTP server.
* @param {string} params.tftpPrefixStr - The TFTP prefix string for boot files.
* @param {string} params.macAddress - The MAC address to be set for the device.
* @param {string} params.clientIp - The static IP address for the client device.
* @param {string} params.subnet - The subnet mask for the client device.
* @param {string} params.gateway - The gateway IP address for the client device.
* @memberof UnderpostBaremetal
* @returns {string} The generated boot configuration content.
* @throws {Error} If an invalid workflow ID is provided.
*/
bootConfFactory({ workflowId, tftpIp, tftpPrefixStr, macAddress, clientIp, subnet, gateway }) {
if (workflowId.startsWith('rpi4mb')) {
// Instructions: Flash sd with Raspberry Pi OS lite and update:
// EEPROM (Electrically Erasable Programmable Read-Only Memory) like microcontrollers
// sudo rpi-eeprom-config --apply /boot/firmware/boot.conf
// sudo reboot
// vcgencmd bootloader_config
// shutdown -h now
return `[all]
BOOT_UART=0
WAKE_ON_GPIO=1
POWER_OFF_ON_HALT=0
ENABLE_SELF_UPDATE=1
DISABLE_HDMI=0
NET_INSTALL_ENABLED=1
DHCP_TIMEOUT=45000
DHCP_REQ_TIMEOUT=4000
TFTP_FILE_TIMEOUT=30000
BOOT_ORDER=0x21
# ─────────────────────────────────────────────────────────────
# TFTP configuration
# ─────────────────────────────────────────────────────────────
# Custom TFTP prefix string (e.g., based on MAC address, no colons)
#TFTP_PREFIX_STR=AA-BB-CC-DD-EE-FF/
# Optional PXE Option43 override (leave commented if unused)
#PXE_OPTION43="Raspberry Pi Boot"
# DHCP client GUID (Option 97); 0x34695052 is the FourCC for Raspberry Pi 4
#DHCP_OPTION97=0x34695052
TFTP_IP=${tftpIp}
TFTP_PREFIX=1
TFTP_PREFIX_STR=${tftpPrefixStr}/
# ─────────────────────────────────────────────────────────────
# Manually override Ethernet MAC address
# ─────────────────────────────────────────────────────────────
#MAC_ADDRESS=${macAddress}
# OTP MAC address override
#MAC_ADDRESS_OTP=0,1
# ─────────────────────────────────────────────────────────────
# Static IP configuration (bypasses DHCP completely)
# ─────────────────────────────────────────────────────────────
#CLIENT_IP=${clientIp}
SUBNET=${subnet}
GATEWAY=${gateway}`;
} else logger.warn(`No boot configuration factory defined for workflow ID: ${workflowId}`);
},
/**
* @method loadWorkflowsConfig
* @description Loads the commission workflows configuration from commission-workflows.json.
* Each workflow defines specific parameters like system provisioning type,
* kernel version, Chrony settings, debootstrap image details, and NFS mounts. *
* @memberof UnderpostBaremetal
*/
loadWorkflowsConfig() {
if (this._workflowsConfig) return this._workflowsConfig;
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const configPath = path.resolve(__dirname, '../../baremetal/commission-workflows.json');
this._workflowsConfig = fs.readJsonSync(configPath);
return this._workflowsConfig;
},
/**
* @property {object} packerMaasImageBuildWorkflows
* @description Configuration for PACKe mass image workflows.
* @memberof UnderpostBaremetal
*/
loadPackerMaasImageBuildWorkflows() {
if (this._packerMaasImageBuildWorkflows) return this._packerMaasImageBuildWorkflows;
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const configPath = path.resolve(__dirname, '../../baremetal/packer-workflows.json');
this._packerMaasImageBuildWorkflows = fs.readJsonSync(configPath);
return this._packerMaasImageBuildWorkflows;
},
/**
* Write Packer MAAS image build workflows configuration to file
* @param {object} workflows - The workflows configuration object
* @description Writes the Packer MAAS image build workflows to packer-workflows.json
* @memberof UnderpostBaremetal
*/
writePackerMaasImageBuildWorkflows(workflows) {
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const configPath = path.resolve(__dirname, '../../baremetal/packer-workflows.json');
fs.writeJsonSync(configPath, workflows, { spaces: 2 });
this._packerMaasImageBuildWorkflows = workflows;
return configPath;
},
};
}
export default UnderpostBaremetal;