@redpanda-data/docs-extensions-and-macros
Version:
Antora extensions and macros developed for Redpanda documentation.
1,359 lines (1,215 loc) • 114 kB
JavaScript
'use strict'
const { spawnSync } = require('child_process')
const crypto = require('crypto')
const path = require('path')
const fs = require('fs')
const os = require('os')
const semver = require('semver')
const { findRepoRoot } = require('../../cli-utils/doc-tools-utils')
const { generateRpkDocs, applyOverridesToTree, resolveReferences, shouldExcludeCommand, shouldUsePartialDir } = require('./generate-rpk-docs')
const { detectLinuxOnlyFromSource, warnIfDetectionLooksBroken } = require('./detect-platform-commands')
const { generateRpkDiff, printDiffReport, generateWhatsNewSection, flattenToMap } = require('./report-delta')
const { loadAndValidateOverrides, ValidationResult } = require('./validate-overrides')
const { validateDirectory, formatResults } = require('./validate-output')
/**
* Known rpk plugins that are managed separately (have install/uninstall commands)
*/
const KNOWN_PLUGINS = ['ai', 'check', 'connect', 'k8s', 'oxla']
/**
* Plugins whose docs can be refreshed individually with --plugin.
* oxla is excluded: it is a "Coming Soon" stub with no installable binary.
*/
const REFRESHABLE_PLUGINS = ['ai', 'check', 'connect', 'k8s']
/**
* Per-plugin install flags that pin a version (rpk <plugin> install <flag> <version>).
* k8s uses the generic flag name; the others embed the plugin name.
*/
const PLUGIN_INSTALL_VERSION_FLAGS = {
ai: '--ai-version',
check: '--check-version',
connect: '--connect-version',
k8s: '--plugin-version'
}
/**
* Manifest slugs at https://rpk-plugins.redpanda.com/<slug>/manifest.json
* where the slug differs from the rpk command name.
*/
const PLUGIN_MANIFEST_SLUGS = { ai: 'rpai' }
const PLUGIN_MANIFEST_HOST = 'https://rpk-plugins.redpanda.com'
/**
* Subcommands compiled into rpk itself for managed plugins. A plugin node
* whose children are only these never actually installed.
*/
const PLUGIN_SHIM_SUBCOMMANDS = new Set(['install', 'uninstall', 'upgrade'])
/**
* Parse Go version from 'go version' output
* @param {string} versionOutput - Output from 'go version' command
* @returns {string|null} Semver-compatible version string or null
*/
function parseGoVersion(versionOutput) {
// go version go1.26.4 darwin/arm64 -> 1.26.4
const match = versionOutput.match(/go(\d+\.\d+(?:\.\d+)?)/)
return match ? match[1] : null
}
/**
* Get required Go version from go.mod file
* @param {string} sourcePath - Path to rpk source directory
* @returns {string|null} Required Go version or null
*/
function getRequiredGoVersion(sourcePath) {
const goModPath = path.join(sourcePath, 'go.mod')
if (!fs.existsSync(goModPath)) {
return null
}
const content = fs.readFileSync(goModPath, 'utf8')
// go 1.26.4
const match = content.match(/^go\s+(\d+\.\d+(?:\.\d+)?)/m)
return match ? match[1] : null
}
/**
* Check if installed Go version meets requirements
* @param {string} installedVersion - Installed Go version
* @param {string} requiredVersion - Required Go version from go.mod
* @returns {boolean} True if version is sufficient
*/
function checkGoVersionSufficient(installedVersion, requiredVersion) {
// Normalize to semver format (add .0 if needed)
const normalize = (v) => {
const parts = v.split('.')
while (parts.length < 3) parts.push('0')
return parts.join('.')
}
return semver.gte(normalize(installedVersion), normalize(requiredVersion))
}
/**
* Extract all command paths from a command tree
* @param {Object} tree - Command tree
* @param {string} prefix - Command path prefix
* @returns {Set<string>} Set of all command paths
*/
function extractCommandPaths(tree, prefix = '') {
const paths = new Set()
const fullPath = prefix ? `${prefix} ${tree.name}` : tree.name || 'rpk'
paths.add(fullPath)
if (tree.commands && Array.isArray(tree.commands)) {
for (const cmd of tree.commands) {
const childPaths = extractCommandPaths(cmd, fullPath)
for (const p of childPaths) {
paths.add(p)
}
}
}
return paths
}
/**
* Detect Linux-only commands by comparing Linux and Darwin builds
* @param {Object} linuxTree - Command tree from Linux build
* @param {Object} darwinTree - Command tree from Darwin/macOS build
* @returns {Set<string>} Commands that exist only on Linux
*/
function detectLinuxOnlyByComparison(linuxTree, darwinTree) {
const linuxCommands = extractCommandPaths(linuxTree)
const darwinCommands = extractCommandPaths(darwinTree)
// Find commands in Linux but not in Darwin
const linuxOnly = new Set()
for (const cmd of linuxCommands) {
if (!darwinCommands.has(cmd)) {
linuxOnly.add(cmd)
}
}
return linuxOnly
}
/**
* Platform identifiers
*/
const PLATFORMS = {
LINUX: 'linux',
DARWIN: 'darwin',
WINDOWS: 'windows'
}
/**
* Get current platform identifier
* @returns {string}
*/
function getCurrentPlatform() {
const platform = os.platform()
if (platform === 'darwin') return PLATFORMS.DARWIN
if (platform === 'win32') return PLATFORMS.WINDOWS
return PLATFORMS.LINUX
}
/**
* Check if a command is a plugin by looking for install/uninstall subcommands
* @param {Object} command - Command object from rpk tree
* @returns {boolean}
*/
function isPlugin(command) {
if (!command.commands || !Array.isArray(command.commands)) return false
const subcommandNames = command.commands.map(c => c.name)
return subcommandNames.includes('install') && subcommandNames.includes('uninstall')
}
/**
* Detect all plugins in the rpk tree
* @param {Object} tree - Full rpk command tree
* @returns {string[]} Array of plugin names
*/
function detectPlugins(tree) {
if (!tree.commands) return []
return tree.commands
.filter(cmd => isPlugin(cmd))
.map(cmd => cmd.name)
}
/**
* Prepare rpk source directory from a GitHub ref (branch or tag)
* If sourcePath is provided and is a git repo, checkout the ref there
* If no sourcePath, do a sparse checkout from GitHub to a temp directory
* @param {string} sourceRef - Git ref (branch or tag, e.g., 'dev', 'v26.2.0')
* @param {string} [sourcePath] - Optional local path to existing repo
* @returns {string} Path to the rpk source directory (src/go/rpk)
*/
function prepareSourceFromRef(sourceRef, sourcePath = null) {
if (sourcePath) {
// Use existing local repo, checkout the specified ref
const absolutePath = path.resolve(sourcePath)
// Check if it's the rpk subdirectory or the repo root
let repoRoot = absolutePath
if (absolutePath.endsWith('src/go/rpk')) {
repoRoot = absolutePath.replace(/\/src\/go\/rpk$/, '')
} else if (fs.existsSync(path.join(absolutePath, 'src', 'go', 'rpk'))) {
// It's the repo root
} else {
throw new Error(
`Cannot determine repo root from ${absolutePath}\n` +
`Provide either the repo root or src/go/rpk directory.`
)
}
// Verify it's a git repo
if (!fs.existsSync(path.join(repoRoot, '.git'))) {
throw new Error(`Not a git repository: ${repoRoot}`)
}
console.log(`Checking out ref '${sourceRef}' in ${repoRoot}...`)
// Fetch and checkout
const fetchResult = spawnSync('git', ['fetch', 'origin', sourceRef], {
cwd: repoRoot,
encoding: 'utf8',
timeout: 120000
})
if (fetchResult.status !== 0) {
console.warn(`Warning: Could not fetch ref '${sourceRef}': ${fetchResult.stderr}`)
}
const checkoutResult = spawnSync('git', ['checkout', sourceRef], {
cwd: repoRoot,
encoding: 'utf8',
timeout: 30000
})
if (checkoutResult.status !== 0) {
throw new Error(`Failed to checkout ref '${sourceRef}': ${checkoutResult.stderr}`)
}
console.log(`Checked out ${sourceRef}`)
return path.join(repoRoot, 'src', 'go', 'rpk')
}
// No local path - do sparse checkout from GitHub
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpk-source-'))
const repoDir = path.join(tmpDir, 'redpanda')
console.log(`Sparse-cloning redpanda repo (ref: ${sourceRef}) to ${repoDir}...`)
// Clone with sparse checkout
const cloneResult = spawnSync('git', [
'clone',
'--depth', '1',
'--filter=blob:none',
'--sparse',
'--branch', sourceRef,
'https://github.com/redpanda-data/redpanda.git',
repoDir
], {
encoding: 'utf8',
timeout: 120000,
stdio: ['pipe', 'pipe', 'pipe']
})
if (cloneResult.status !== 0) {
throw new Error(
`Failed to clone redpanda repo with ref '${sourceRef}'.\n` +
`Make sure the branch or tag exists.\n` +
`Error: ${cloneResult.stderr}`
)
}
// Set sparse checkout to only get rpk
const sparseResult = spawnSync('git', ['sparse-checkout', 'set', 'src/go/rpk'], {
cwd: repoDir,
encoding: 'utf8',
timeout: 60000
})
if (sparseResult.status !== 0) {
throw new Error(`Failed to set sparse checkout: ${sparseResult.stderr}`)
}
console.log(`Sparse checkout complete`)
return path.join(repoDir, 'src', 'go', 'rpk')
}
/**
* Fetch rpk tree by running from Go source code
* Useful for pre-releases before Docker images are published
* @param {string} sourcePath - Path to rpk Go source directory (e.g., ~/redpanda/src/go/rpk)
* @returns {Object} Parsed JSON tree
*/
function fetchRpkTreeFromSource(sourcePath) {
// Verify the source path exists
if (!fs.existsSync(sourcePath)) {
throw new Error(
`rpk source directory not found: ${sourcePath}\n` +
'To use --from-source, you need a local checkout of the redpanda repository.\n' +
'Clone it with: git clone https://github.com/redpanda-data/redpanda.git\n' +
'Then point to: <repo>/src/go/rpk'
)
}
// Verify it looks like the right directory (should have cmd/rpk)
const mainPath = path.join(sourcePath, 'cmd', 'rpk', 'main.go')
if (!fs.existsSync(mainPath)) {
throw new Error(
`Invalid rpk source directory: ${sourcePath}\n` +
`Expected to find cmd/rpk/main.go. Make sure you point to the src/go/rpk directory.`
)
}
// Check if Go is installed
const goCheck = spawnSync('go', ['version'], { encoding: 'utf8', timeout: 5000 })
if (goCheck.status !== 0) {
throw new Error(
'Go is required for --from-source but was not found.\n' +
'Install Go from https://go.dev/ and ensure it\'s in your PATH.'
)
}
console.log(`Building and running rpk from source at ${sourcePath}...`)
console.log(`Go version: ${goCheck.stdout.trim()}`)
// Check Go version meets go.mod requirements
const installedGoVersion = parseGoVersion(goCheck.stdout)
const requiredGoVersion = getRequiredGoVersion(sourcePath)
if (installedGoVersion && requiredGoVersion) {
if (!checkGoVersionSufficient(installedGoVersion, requiredGoVersion)) {
throw new Error(
`Go version mismatch: installed ${installedGoVersion}, required >= ${requiredGoVersion}\n` +
`The rpk source (go.mod) requires Go ${requiredGoVersion} or newer.\n` +
'Update Go: brew upgrade go (macOS) or download from https://go.dev/dl/'
)
}
}
// Run rpk directly from source using go run
const result = spawnSync('go', ['run', 'cmd/rpk/main.go', '--print-tree'], {
cwd: sourcePath,
encoding: 'utf8',
timeout: 120000, // 2 minutes (includes build time)
maxBuffer: 50 * 1024 * 1024
})
if (result.status !== 0) {
const stderr = result.stderr || ''
if (stderr.includes('unknown flag')) {
throw new Error(
`rpk source does not support --print-tree flag.\n` +
`This feature requires rpk source from after the --print-tree feature was added.\n` +
'Update your source checkout: cd <repo> && git pull origin dev'
)
}
if (stderr.includes('go.mod') || stderr.includes('module')) {
throw new Error(
`Go module error while building rpk: ${stderr}\n` +
'Try running "go mod download" in the source directory first.'
)
}
throw new Error(
`Failed to build/run rpk from source: ${stderr}\n` +
'Common fixes:\n' +
' 1. Update Go to the latest version\n' +
' 2. Run "go mod download" in the source directory\n' +
' 3. Ensure the source is up to date: git pull origin dev'
)
}
try {
return JSON.parse(result.stdout)
} catch (err) {
throw new Error(
`Failed to parse rpk tree JSON from source build: ${err.message}\n` +
'The build succeeded but the output was not valid JSON.\n' +
'This may indicate a version mismatch or corrupted source.'
)
}
}
/**
* Build rpk from Go source inside a Linux Docker container (optional optimization).
* Builds rpk binary, installs plugins, then runs --print-tree for complete command coverage.
* Falls back to native Go build if Docker is unavailable.
* @param {string} sourcePath - Path to rpk Go source directory (e.g., ~/redpanda/src/go/rpk)
* @returns {Object} { tree, failedPlugins } — the parsed JSON tree plus the
* names of managed plugins whose install failed this run. Callers pass
* failedPlugins to generateRpkDocs as protectedPlugins so those plugins'
* existing pages and nav entries are preserved instead of treated as stale.
*/
function fetchRpkTreeFromLinuxSource(sourcePath, pluginPins = {}) {
// Resolve to absolute path
const absoluteSourcePath = path.resolve(sourcePath)
// Verify the source path exists
if (!fs.existsSync(absoluteSourcePath)) {
throw new Error(
`rpk source directory not found: ${absoluteSourcePath}\n` +
'Expected a checkout of the redpanda repository.\n' +
'Clone it with: git clone https://github.com/redpanda-data/redpanda.git'
)
}
// Verify it looks like the right directory (should have cmd/rpk)
const mainPath = path.join(absoluteSourcePath, 'cmd', 'rpk', 'main.go')
if (!fs.existsSync(mainPath)) {
throw new Error(
`Invalid rpk source directory: ${absoluteSourcePath}\n` +
'Expected to find cmd/rpk/main.go.\n' +
'Make sure you point to the src/go/rpk directory inside your redpanda checkout.'
)
}
// Docker is optional - used when available for Linux plugin support
const dockerCheck = spawnSync('docker', ['--version'], { encoding: 'utf8', timeout: 5000 })
if (dockerCheck.status !== 0) {
// Docker not available - this function shouldn't be called
throw new Error(
'Docker not available for Linux container build.\n' +
'Use fetchRpkTreeFromSource() for native Go build instead.'
)
}
console.log(`Building and running rpk from source in Linux container...`)
console.log(`Source path: ${absoluteSourcePath}`)
// Start a container with the Go image, mount source, build rpk, install plugins, then print-tree
// Use a long-running container so we can run multiple commands
// Pin the Go image to the exact version required by go.mod to prevent
// "go.mod requires go >= X.Y.Z (running X.Y.Z-1)" build failures when
// golang:1 resolves to a patch release behind the requirement.
const requiredGoVersion = getRequiredGoVersion(absoluteSourcePath)
const goImage = requiredGoVersion ? `golang:${requiredGoVersion}` : 'golang:1'
console.log('Starting build container...')
const createResult = spawnSync('docker', [
'run', '-d', '--rm',
'-v', `${absoluteSourcePath}:/rpk-source:ro`,
'-w', '/rpk-source',
goImage,
'sh', '-c', 'sleep 600' // Keep container alive for 10 minutes
], {
encoding: 'utf8',
timeout: 60000
})
let activeResult = createResult
let activeImage = goImage
if (createResult.status !== 0) {
const stderr = createResult.stderr || ''
if (stderr.includes('Cannot connect to the Docker daemon')) {
throw new Error(
'Docker daemon is not running.\n' +
'Start Docker Desktop or the Docker service and try again.'
)
}
// If the pinned tag couldn't be pulled (not yet on Docker Hub for a new
// patch release), retry with golang:1 before giving up.
const isPullFailure = requiredGoVersion && (
stderr.includes('manifest unknown') ||
stderr.includes('pull access denied') ||
stderr.includes('not found') ||
stderr.includes('repository does not exist')
)
if (isPullFailure) {
console.warn(`⚠ Could not pull ${goImage}: ${stderr.trim()}`)
console.log('Retrying with golang:1...')
activeImage = 'golang:1'
activeResult = spawnSync('docker', [
'run', '-d', '--rm',
'-v', `${absoluteSourcePath}:/rpk-source:ro`,
'-w', '/rpk-source',
activeImage,
'sh', '-c', 'sleep 600'
], {
encoding: 'utf8',
timeout: 60000
})
}
if (activeResult.status !== 0) {
throw new Error(
`Failed to create build container: ${activeResult.stderr || stderr}\n` +
'Make sure Docker is running and has sufficient resources.'
)
}
}
const containerId = activeResult.stdout.trim()
console.log(`Build container started: ${containerId.substring(0, 12)}`)
try {
// Step 1: Build rpk binary
console.log('Building rpk binary...')
// Module downloads from proxy.golang.org fail transiently (stream
// INTERNAL_ERROR), especially on a cold module cache. Retry: the second
// attempt reuses whatever the first already downloaded.
let buildResult
let binaryExists = false
for (let attempt = 1; attempt <= 4; attempt++) {
buildResult = spawnSync('docker', [
'exec', containerId,
'go', 'build', '-o', '/tmp/rpk', './cmd/rpk'
], {
encoding: 'utf8',
timeout: 300000 // 5 minutes per attempt
})
if (buildResult.status === 0) {
// Trust but verify: docker exec has been observed returning zero for
// a build that produced nothing, and everything downstream execs
// /tmp/rpk. Treat a phantom success as a failed attempt.
const binCheck = spawnSync('docker', [
'exec', containerId, 'test', '-x', '/tmp/rpk'
], { encoding: 'utf8', timeout: 15000 })
if (binCheck.status === 0) {
binaryExists = true
break
}
if (attempt < 4) {
console.warn(` Build attempt ${attempt} reported success but produced no binary; retrying...`)
continue
}
} else if (attempt < 4) {
const firstError = (buildResult.stderr || buildResult.signal || 'unknown error')
.toString().trim().split('\n').slice(-1)[0]
console.warn(` Build attempt ${attempt} failed (${firstError}); retrying...`)
}
}
if (!binaryExists) {
const stderr = buildResult.stderr || ''
throw new Error(
`Failed to build rpk in Linux container: ${stderr || 'build produced no binary'}\n` +
'Common causes:\n' +
' 1. Source code is out of date - run: git pull origin dev\n' +
' 2. Go module issues - the container will download dependencies automatically\n' +
' 3. Insufficient Docker resources - check Docker Desktop settings'
)
}
console.log(' ✓ rpk binary built')
// Step 2: Install plugins
console.log('Installing plugins for complete command coverage...')
const failedPlugins = []
for (const plugin of KNOWN_PLUGINS) {
// A pin installs a specific version instead of the manifest's latest.
// This is how pre-GA plugins (no version promoted to latest) get into a
// full regeneration at all — without a pin their install resolves
// nothing and their commands are absent from the tree.
const pin = pluginPins[plugin]
const versionFlag = PLUGIN_INSTALL_VERSION_FLAGS[plugin]
const runInstall = (pinned) => {
const args = ['exec', containerId, '/tmp/rpk', plugin, 'install']
if (pinned && pin && versionFlag) {
args.push(versionFlag, pin)
}
return spawnSync('docker', args, { encoding: 'utf8', timeout: 120000 })
}
console.log(` Installing plugin: ${plugin}${pin ? ` (pinned to ${pin})` : ''}...`)
let installResult = runInstall(true)
if (installResult.status !== 0 && pin) {
const output = `${installResult.stderr || ''}${installResult.stdout || ''}`
if (output.includes('unknown flag') || output.includes('is not valid')) {
console.warn(` rpk rejected the version pin for ${plugin}; retrying without the pin (installs latest)`)
installResult = runInstall(false)
}
}
if (installResult.status === 0) {
console.log(` ✓ ${plugin} installed`)
} else {
const stderr = installResult.stderr || ''
const stdout = installResult.stdout || ''
if (stderr.includes('already installed') || stdout.includes('already installed')) {
console.log(` ✓ ${plugin} already installed`)
} else if (stderr.includes('unknown command') || stderr.includes('Error: unknown command')) {
console.log(` - ${plugin} is not an installable plugin`)
} else {
// A failed install is non-fatal: generation continues, but this
// plugin's commands will be absent from the tree. Expected for
// beta-only plugins with no version pin — `rpk <plugin> install`
// finds no `latest` release because the plugin publisher's
// stableVersionRe only promotes pure X.Y.Z versions, so their
// commands only appear at GA unless the run pins a version.
// See redpanda-data/docs#1801.
console.warn(` ✗ Failed to install ${plugin}: ${stderr || stdout}`)
failedPlugins.push(plugin)
}
}
}
// Step 3: Run --print-tree with all plugins installed
console.log('Fetching rpk tree with plugins installed...')
const result = spawnSync('docker', [
'exec', containerId,
'/tmp/rpk', '--print-tree'
], {
encoding: 'utf8',
timeout: 120000,
maxBuffer: 50 * 1024 * 1024
})
if (result.status !== 0) {
const stderr = result.stderr || ''
if (stderr.includes('unknown flag')) {
throw new Error(
`rpk source does not support --print-tree flag.\n` +
`This feature requires rpk source from after the --print-tree feature was added.\n` +
'Update your source checkout: cd <repo> && git pull origin dev'
)
}
throw new Error(
`Failed to run rpk --print-tree in Linux container: ${stderr}\n` +
'The build succeeded but --print-tree failed.\n' +
'This may indicate a version or configuration issue.'
)
}
let tree
try {
tree = JSON.parse(result.stdout)
} catch (err) {
throw new Error(
`Failed to parse rpk tree JSON from Linux source build: ${err.message}\n` +
'The build and --print-tree succeeded but the output was not valid JSON.\n' +
'This may indicate a version mismatch or corrupted source.'
)
}
// Step 4: Fill in flags for plugin subtrees. Their commands come from
// --help-autocomplete, which carries no flag data, so without this every
// plugin command page renders an empty flags section.
console.log('Extracting flags from plugin command help output...')
for (const plugin of KNOWN_PLUGINS) {
const node = (tree.commands || []).find(c => c.name === plugin)
if (!node || !pluginNodeHasRealCommands(node)) continue
const enriched = enrichPluginTreeWithFlags(node, (argPath) => {
const helpResult = spawnSync('docker', [
'exec', containerId, '/tmp/rpk', ...argPath, '--help'
], { encoding: 'utf8', timeout: 30000 })
return helpResult.status === 0 ? helpResult.stdout : null
})
if (enriched > 0) {
console.log(` ${plugin}: extracted flags for ${enriched} command(s)`)
}
}
return { tree, failedPlugins }
} finally {
// Clean up container
console.log('Cleaning up build container...')
spawnSync('docker', ['stop', containerId], {
encoding: 'utf8',
timeout: 30000
})
}
}
/**
* Add platform markers to command tree based on source analysis
* @param {Object} tree - Command tree from rpk --print-tree
* @param {Set<string>} linuxOnlyCommands - Set of Linux-only command paths
* @returns {Object} Tree with platform availability info
*/
function addPlatformMarkersFromSource(tree, linuxOnlyCommands) {
const isLinuxOnly = (cmdPath) => {
// Check if this command or any parent is Linux-only from source detection
// Detection comes from: 1) Go build tags in source, 2) dynamic comparison of Linux vs Darwin builds
return linuxOnlyCommands.has(cmdPath) ||
[...linuxOnlyCommands].some(loc => cmdPath.startsWith(loc + ' '))
}
// Collect every tree path actually marked Linux-only so the persisted
// linux_only_commands list is fully expanded (each descendant listed),
// matching what dynamic Linux-vs-Darwin tree comparison produces.
const markedPaths = new Set()
const markCommands = (commands, parentPath = 'rpk') => {
if (!commands) return commands
return commands.map(cmd => {
const fullPath = `${parentPath} ${cmd.name}`
const linuxOnly = isLinuxOnly(fullPath)
if (linuxOnly) markedPaths.add(fullPath)
const platforms = linuxOnly
? [PLATFORMS.LINUX]
: [PLATFORMS.LINUX, PLATFORMS.DARWIN]
return {
...cmd,
platforms,
commands: markCommands(cmd.commands, fullPath)
}
})
}
// Log detected Linux-only commands
if (linuxOnlyCommands.size > 0) {
console.log(`Detected ${linuxOnlyCommands.size} Linux-only command path(s) from source:`)
for (const cmd of linuxOnlyCommands) {
console.log(` - ${cmd}`)
}
}
const markedCommands = markCommands(tree.commands)
return {
...tree,
platforms: [PLATFORMS.LINUX, PLATFORMS.DARWIN],
// Union of detected paths and marked tree paths: keeps detected roots
// even when the tree was built on a platform where they don't exist
linux_only_commands: [...new Set([...linuxOnlyCommands, ...markedPaths])].sort(),
commands: markedCommands
}
}
/**
* Get platform availability description for a command
* @param {string[]} platforms - Array of platform identifiers
* @returns {string} Human-readable description
*/
function getPlatformDescription(platforms) {
if (!platforms || platforms.length === 0) return ''
if (platforms.length >= 2 && platforms.includes(PLATFORMS.LINUX) && platforms.includes(PLATFORMS.DARWIN)) {
return '' // Available on all major platforms, no need to note
}
if (platforms.length === 1) {
switch (platforms[0]) {
case PLATFORMS.LINUX:
return 'Linux only'
case PLATFORMS.DARWIN:
return 'macOS only'
case PLATFORMS.WINDOWS:
return 'Windows only'
default:
return platforms[0]
}
}
return platforms.map(p => {
switch (p) {
case PLATFORMS.LINUX: return 'Linux'
case PLATFORMS.DARWIN: return 'macOS'
case PLATFORMS.WINDOWS: return 'Windows'
default: return p
}
}).join(', ')
}
/**
* Load overrides from JSON file with validation
* @param {string} overridesPath - Path to overrides file
* @param {Object} [commandTree] - Optional command tree for path validation
* @param {Object} [options] - Options
* @param {boolean} [options.strict=false] - If true, throw on validation errors
* @returns {Object|null} Overrides object or null if not found
*/
function loadOverrides(overridesPath, commandTree = null, options = {}) {
const { strict = false } = options
if (!overridesPath || !fs.existsSync(overridesPath)) {
return null
}
const { overrides, validation } = loadAndValidateOverrides(overridesPath, commandTree)
// Report validation issues
if (validation.errors.length > 0 || validation.warnings.length > 0) {
console.log('\n' + '='.repeat(60))
console.log('OVERRIDE VALIDATION RESULTS')
console.log('='.repeat(60))
console.log(validation.format())
console.log('='.repeat(60) + '\n')
}
// In strict mode, fail on errors
if (strict && !validation.valid) {
throw new Error(
`Override validation failed with ${validation.errors.length} error(s).\n` +
`Fix the issues above or run without --strict to proceed with warnings.`
)
}
// Warn but continue on non-strict validation failures
if (!validation.valid && !strict) {
console.warn(
`⚠ Proceeding with ${validation.errors.length} validation error(s). ` +
`Generated docs may be incorrect.`
)
}
return { overrides, validation }
}
/**
* Merge deprecation metadata for commands still present in the tree into the
* overrides file, so their pages render deprecation banners without manual
* curation. Hidden deprecated commands are excluded: they are absent from
* --print-tree, have no pages, and would only produce unknown-path warnings.
* Commands whose overrides already carry a `deprecated` value are left alone.
* @param {Object} deprecatedCommands - Map from scan-deprecated-commands.js
* @param {Object} tree - Current command tree
* @param {string} overridesPath - Path to overrides JSON file
*/
function mergeVisibleDeprecationsIntoOverrides(deprecatedCommands, tree, overridesPath) {
const entries = Object.entries(deprecatedCommands || {})
if (entries.length === 0 || !overridesPath || !fs.existsSync(overridesPath)) {
return
}
const treePaths = new Set(flattenToMap(tree).keys())
let overrides
try {
overrides = JSON.parse(fs.readFileSync(overridesPath, 'utf8'))
} catch (err) {
console.warn(`Warning: Could not parse overrides file for deprecation merge: ${err.message}`)
return
}
if (!overrides.commands) {
overrides.commands = {}
}
let annotated = 0
for (const [cmdPath, info] of entries) {
if (!treePaths.has(cmdPath)) continue
const existing = overrides.commands[cmdPath] || {}
if (existing.deprecated !== undefined) continue
overrides.commands[cmdPath] = {
...existing,
deprecated: true,
...(info.deprecatedMessage && !existing.deprecatedMessage
? { deprecatedMessage: info.deprecatedMessage } : {}),
...(info.replacement && !existing.replacement
? { replacement: info.replacement } : {})
}
annotated++
}
if (annotated > 0) {
fs.writeFileSync(overridesPath, JSON.stringify(overrides, null, 2), 'utf8')
console.log(`Annotated ${annotated} visible deprecated command(s) in ${overridesPath}`)
}
}
/**
* Save versioned JSON tree
* @param {Object} data - Data to save
* @param {string} version - Version string
* @param {string} dataDir - Output directory
* @returns {string} Path to saved file
*/
function saveVersionedJson(data, version, dataDir) {
const normalizedVersion = version.startsWith('v') ? version : `v${version}`
const fileName = `rpk-${normalizedVersion}.json`
const filePath = path.join(dataDir, fileName)
fs.mkdirSync(dataDir, { recursive: true })
fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8')
console.log(`Saved versioned JSON to ${filePath}`)
return filePath
}
/**
* Load existing versioned JSON
* @param {string} version - Version string
* @param {string} dataDir - Data directory
* @returns {Object|null} Loaded data or null
*/
function loadVersionedJson(version, dataDir) {
const normalizedVersion = version.startsWith('v') ? version : `v${version}`
const fileName = `rpk-${normalizedVersion}.json`
const filePath = path.join(dataDir, fileName)
if (!fs.existsSync(filePath)) {
return null
}
try {
const content = fs.readFileSync(filePath, 'utf8')
return JSON.parse(content)
} catch (err) {
console.warn(`Warning: Could not load ${filePath}: ${err.message}`)
return null
}
}
/**
* Update overrides file with introducedInVersion for new commands and flags
* @param {Object} diffData - Diff data with new commands and flags
* @param {string} overridesPath - Path to overrides JSON file
* @param {string} version - Version to set as introducedInVersion
* @param {Object} [pluginVersions] - Plugin versions keyed by rpk command
* name. Commands under a plugin subtree are stamped with the plugin's own
* version (the page note renders "introduced in <plugin> version X"), not
* the rpk version, because plugins release on their own cadence.
*/
function updateOverridesWithIntroducedVersions(diffData, overridesPath, version, pluginVersions = {}, options = {}) {
const hasNewCommands = diffData.details.newCommands && diffData.details.newCommands.length > 0
const hasNewFlags = diffData.details.newFlags && diffData.details.newFlags.length > 0
if (!hasNewCommands && !hasNewFlags) {
return
}
// "rpk connect lint" -> pluginVersions.connect, else the rpk version
const versionFor = (cmdPath) => {
const topLevel = cmdPath.split(' ')[1]
return pluginVersions[topLevel] || version
}
// Plugin-owned entries are only stamped when the caller vouches that the
// baseline is manifest-adjacent to this run's plugin version (see
// isPluginStampAttributable). "New relative to the snapshot" is not "new
// in this release" when the snapshot skipped releases. Callers that omit
// attributablePlugins keep legacy stamp-everything behavior.
const attributable = options.attributablePlugins
? new Set(options.attributablePlugins)
: null
const skippedByPlugin = new Map()
const stampable = (cmdPath) => {
const topLevel = cmdPath.split(' ')[1]
if (!KNOWN_PLUGINS.includes(topLevel) || !attributable) return true
if (attributable.has(topLevel)) return true
skippedByPlugin.set(topLevel, (skippedByPlugin.get(topLevel) || 0) + 1)
return false
}
let overrides = {}
if (fs.existsSync(overridesPath)) {
try {
overrides = JSON.parse(fs.readFileSync(overridesPath, 'utf8'))
} catch (err) {
console.warn(`Warning: Could not parse overrides file: ${err.message}`)
return
}
}
if (!overrides.commands) {
overrides.commands = {}
}
let commandsUpdated = 0
let flagsUpdated = 0
// Update new commands
if (hasNewCommands) {
for (const newCmd of diffData.details.newCommands) {
const cmdPath = newCmd.path
if (!stampable(cmdPath)) continue
if (!overrides.commands[cmdPath]) {
overrides.commands[cmdPath] = {}
}
// Only set if not already set (preserve manual overrides)
if (!overrides.commands[cmdPath].introducedInVersion) {
overrides.commands[cmdPath].introducedInVersion = versionFor(cmdPath)
commandsUpdated++
}
}
}
// Update new flags
if (hasNewFlags) {
for (const newFlag of diffData.details.newFlags) {
const cmdPath = newFlag.commandPath
const flagName = newFlag.flagName
if (!stampable(cmdPath)) continue
if (!overrides.commands[cmdPath]) {
overrides.commands[cmdPath] = {}
}
if (!overrides.commands[cmdPath].flags) {
overrides.commands[cmdPath].flags = {}
}
if (!overrides.commands[cmdPath].flags[flagName]) {
overrides.commands[cmdPath].flags[flagName] = {}
}
// Only set if not already set (preserve manual overrides)
if (!overrides.commands[cmdPath].flags[flagName].introducedInVersion) {
overrides.commands[cmdPath].flags[flagName].introducedInVersion = versionFor(cmdPath)
flagsUpdated++
}
}
}
for (const [plugin, count] of skippedByPlugin) {
console.warn(
`\u26a0 Skipped stamping introducedInVersion for ${count} new '${plugin}' entr${count === 1 ? 'y' : 'ies'}: ` +
`the baseline snapshot's ${plugin} version is not the release immediately before ` +
`${pluginVersions[plugin] || version} in the plugin manifest, so the introduction ` +
'version cannot be attributed automatically. Attribute manually against released binaries.'
)
}
if (commandsUpdated > 0 || flagsUpdated > 0) {
fs.writeFileSync(overridesPath, JSON.stringify(overrides, null, 2), 'utf8')
const updates = []
if (commandsUpdated > 0) updates.push(`${commandsUpdated} new command(s)`)
if (flagsUpdated > 0) updates.push(`${flagsUpdated} new flag(s)`)
console.log(`Updated ${overridesPath} with introducedInVersion for ${updates.join(' and ')}`)
}
}
/**
* Get the latest documented version from data directory
* @param {string} dataDir - Data directory path
* @returns {string|null} Latest version or null
*/
function getLatestDocumentedVersion(dataDir) {
if (!fs.existsSync(dataDir)) return null
const files = fs.readdirSync(dataDir)
.filter(f => f.startsWith('rpk-v') && f.endsWith('.json') && !f.includes('diff'))
.map(f => f.replace('rpk-', '').replace('.json', ''))
.filter(v => semver.valid(v))
.sort((a, b) => semver.compare(b, a)) // Descending
return files.length > 0 ? files[0] : null
}
/**
* Common locations where tech writers might have redpanda source checked out
*/
const COMMON_SOURCE_LOCATIONS = [
'~/redpanda/src/go/rpk',
'~/Documents/redpanda/src/go/rpk',
'~/repos/redpanda/src/go/rpk',
'~/code/redpanda/src/go/rpk',
'~/projects/redpanda/src/go/rpk',
'../redpanda/src/go/rpk',
'../../redpanda/src/go/rpk'
]
/**
* Try to find a local redpanda source checkout
* @returns {string|null} Path to rpk source directory, or null if not found
*/
function findLocalSource() {
const homeDir = os.homedir()
for (const location of COMMON_SOURCE_LOCATIONS) {
const expandedPath = location.replace('~', homeDir)
const absolutePath = path.resolve(expandedPath)
const mainGoPath = path.join(absolutePath, 'cmd', 'rpk', 'main.go')
if (fs.existsSync(mainGoPath)) {
return absolutePath
}
}
return null
}
/**
* Count total commands in tree (recursive)
* @param {Object} node - Tree node
* @returns {number} Total command count
*/
function countCommands(node) {
if (!node) return 0
let count = 1 // Count this node
if (node.commands && Array.isArray(node.commands)) {
for (const child of node.commands) {
count += countCommands(child)
}
}
return count
}
/**
* Update what's-new file with rpk changes from diff
* @param {Object} diffData - Diff data from generateRpkDiff
* @param {string} whatsNewPath - Path to what's-new.adoc file
* @param {string} version - Version string for display
*/
/**
* Build a predicate that reports whether a command path renders as a
* linkable page (not excluded and not routed to partials by the overrides).
* @param {Object|null} overridesData - Loaded overrides
* @returns {Function} (commandPath) => boolean
*/
/**
* Build a predicate that reports whether a command path has subcommands,
* which determines its page location (groups render into their own dir).
* @param {Object} tree - Full command tree
* @returns {Function} (commandPath) => boolean
*/
function makeSubcommandPredicate(tree) {
const commandMap = flattenToMap(tree)
return (commandPath) => {
const node = commandMap.get(commandPath)
return Boolean(node && (node.commands || []).length > 0)
}
}
function makeLinkablePredicate(overridesData) {
const resolved = overridesData ? resolveReferences(overridesData, overridesData) : null
return (commandPath) => {
// rpk cloud and rpk security secret render to partials (single-sourced
// into cloud docs), so this repo has no linkable pages for them
if (commandPath.startsWith('rpk cloud') || commandPath.startsWith('rpk security secret')) {
return false
}
if (!resolved) return true
return !shouldExcludeCommand(resolved, commandPath) &&
!shouldUsePartialDir(resolved, commandPath)
}
}
// Command subtrees whose changes never belong in the Self-Managed What's
// new. rpk ai's documentation home is adp-docs, and the ADP release notes
// already cover its CLI changes per release. The plugin-release receiver
// workflow excludes ai from --update-whats-new for exactly this reason; the
// full-regeneration path must agree, or a full run floods the Self-Managed
// release notes with rpk ai entries (a rename release alone produces 21 new
// plus 21 removed bullets).
const WHATS_NEW_EXCLUDED_SUBTREES = ['rpk ai']
/**
* Return a copy of diffData without entries under the excluded subtrees.
* Only the published What's-new block filters; diff reports and PR
* summaries keep the full picture.
* @param {Object} diffData - Diff from generateRpkDiff
* @param {string[]} [excluded] - Command-path prefixes to drop
* @returns {Object} Filtered copy
*/
function filterDiffForWhatsNew(diffData, excluded = WHATS_NEW_EXCLUDED_SUBTREES) {
const outside = (cmdPath) => !excluded.some(prefix =>
cmdPath === prefix || (typeof cmdPath === 'string' && cmdPath.startsWith(prefix + ' ')))
const details = diffData.details || {}
const filteredDetails = { ...details }
for (const key of ['newCommands', 'newlyDeprecatedCommands', 'removedCommands', 'descriptionChanges']) {
if (Array.isArray(details[key])) filteredDetails[key] = details[key].filter(e => outside(e.path))
}
for (const key of ['newFlags', 'removedFlags', 'changedDefaults', 'changedFlagTypes', 'changedFlagRequirements', 'changedFlagDescriptions']) {
if (Array.isArray(details[key])) filteredDetails[key] = details[key].filter(e => outside(e.commandPath))
}
return { ...diffData, details: filteredDetails }
}
function updateWhatsNewFile(diffData, whatsNewPath, version, options = {}) {
// Each block opens with a "=== <version>" heading so accumulated blocks
// (successive RCs, multiple plugin releases) never collide on section ids
const sectionHeading = options.sectionHeading || '== Redpanda CLI'
const whatsNewContent = generateWhatsNewSection(diffData, {
version,
blockLabel: version,
...options
})
if (!whatsNewContent) {
console.log('No Redpanda CLI changes to add to what\'s new')
return
}
if (!fs.existsSync(whatsNewPath)) {
console.warn(`Warning: what's-new file not found: ${whatsNewPath}`)
console.log('Generated what\'s-new content:')
console.log(whatsNewContent)
return
}
const existingContent = fs.readFileSync(whatsNewPath, 'utf8')
// Version-scoped marker block: re-runs for the same version replace their
// own block, and later versions (for example, successive RCs in a beta
// cycle, or plugin releases) append their own blocks inside the existing
// Redpanda CLI section instead of being dropped. Writers can edit or
// remove blocks freely; the automation only ever touches content between
// its own markers for the same version label.
const startMarker = `// AUTOGEN-RPK-CHANGES ${version} START`
const endMarker = `// AUTOGEN-RPK-CHANGES ${version} END`
const sectionBody = whatsNewContent.replace(/^== [^\n]*\n+/, '')
const block = `${startMarker}\n${sectionBody.trimEnd()}\n${endMarker}`
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
if (existingContent.includes(startMarker)) {
// Replace this version's existing block (idempotent re-runs)
const blockRe = new RegExp(`${escapeRe(startMarker)}[\\s\\S]*?${escapeRe(endMarker)}`)
const updatedContent = existingContent.replace(blockRe, block)
fs.writeFileSync(whatsNewPath, updatedContent, 'utf8')
console.log(`Refreshed existing ${version} block in what's-new file: ${whatsNewPath}`)
return
}
const headingMatch = existingContent.match(new RegExp(`^${escapeRe(sectionHeading)}[^\\n]*$`, 'm'))
if (headingMatch) {
// Append this version's block at the end of the existing section, just
// before the next level-2 heading (or end of file)
const sectionStart = headingMatch.index + headingMatch[0].length
const rest = existingContent.slice(sectionStart)
const nextHeading = rest.search(/\n== /)
const insertAt = nextHeading === -1 ? existingContent.length : sectionStart + nextHeading
const before = existingContent.slice(0, insertAt).replace(/\s*$/, '\n\n')
const after = existingContent.slice(insertAt).replace(/^\n*/, '\n')
fs.writeFileSync(whatsNewPath, `${before}${block}${after}`, 'utf8')
console.log(`Appended ${version} block to the "${sectionHeading}" section in: ${whatsNewPath}`)
return
}
// Find a good insertion point - before the last section or at the end
// Look for a pattern like "== New configuration properties" or similar
const insertionPatterns = [
/^== New configuration properties/m,
/^== Deprecations/m,
/^== Bug fixes/m,
/^== See also/m
]
let insertIndex = -1
for (const pattern of insertionPatterns) {
const match = existingContent.match(pattern)
if (match) {
insertIndex = match.index
break
}
}
const fullSection = `${sectionHeading}\n\n${block}\n`
let updatedContent
if (insertIndex > 0) {
// Insert before the matched section
updatedContent = existingContent.slice(0, insertIndex) +
fullSection + '\n' +
existingContent.slice(insertIndex)
} else {
// Append at the end
updatedContent = existingContent.replace(/\s*$/, '\n\n') + fullSection
}
fs.writeFileSync(whatsNewPath, updatedContent, 'utf8')
console.log(`Created "${sectionHeading}" section in what's-new file: ${whatsNewPath}`)
}
/**
* Build an rpk binary natively from Go source.
* @param {string} sourcePath - Path to rpk Go source directory (src/go/rpk)
* @param {string} outPath - Where to write the binary
* @returns {string} Path to the built binary
*/
function buildRpkBinary(sourcePath, outPath) {
const goCheck = spawnSync('go', ['version'], { encoding: 'utf8', timeout: 5000 })
if (goCheck.status !== 0) {
throw new Error(
'Go is required to build rpk from source but was not found.\n' +
'Install Go from https://go.dev/ and ensure it\'s in your PATH.'
)
}
const installedGoVersion = parseGoVersion(goCheck.stdout)
const requiredGoVersion = getRequiredGoVersion(sourcePath)
if (installedGoVersion && requiredGoVersion &&
!checkGoVersionSufficient(installedGoVersion, requiredGoVersion)) {
throw new Error(
`Go version mismatch: installed ${installedGoVersion}, required >= ${requiredGoVersion}\n` +
`The rpk source (go.mod) requires Go ${requiredGoVersion} or newer.`
)
}
console.log(`Building rpk from source at ${sourcePath}...`)
const buildResult = spawnSync('go', ['build', '-o', outPath, './cmd/rpk'], {
cwd: sourcePath,
encoding: 'utf8',
timeout: 300000
})
if (buildResult.status !== 0) {
throw new Error(`Failed to build rpk from source: ${buildResult.stderr}`)
}
return outPath
}
/**
* Download an official rpk release binary for the current platform.
* @param {string} tag - Release tag (e.g., v26.1.12)
* @param {string} destDir - Directory to download and extract into
* @returns {string|null} Path to the extracted binary, or null if the
* release asset is unavailable (caller falls back to a source build)
*/
function downloadRpkRelease(tag, destDir) {
const osName = { darwin: 'darwin', linux: 'linux', win32: 'windows' }[process.platform]
const archName = { arm64: 'arm64', x64: 'amd64' }[process.arch]
if (!osName || !archName) {
console.warn(`No rpk release asset for platform ${process.platform}/${process.arch}`)
return null
}
const assetName = `rpk-${osName}-${archName}.zip`
const baseUrl = `https://github.com/redpanda-data/redpanda/releases/download/${tag}`
const zipPath = path.join(destDir, assetName)
console.log(`Downloading ${assetName} for ${tag}...`)
const curlResult = spawnSync('curl', [
'-fL', '--retry', '5', '--retry-all-errors',
'--connect-timeout', '30', '--max-time', '300',
'-o', zipPath, `${baseUrl}/${assetName}`
], { encoding: 'utf8', timeout: 360000 })
if (curlResult.status !== 0) {
console.warn(`Could not download rpk release for ${tag} (draft or missing release asset)`)
return null
}
// Verify against the release checksum file when it exists
const checksumAsset = `rpk_${tag.replace(/^v/, '')}_checksums.txt`
const checksumPath = path.join(destDir, checksumAsset)
const checksumResult = spawnSync('curl', [
'-fsSL', '--retry', '3', '--connect-timeout', '30', '--max-time', '60',
'-o', checksumPath, `${baseUrl}/${checksumAsset}`
], { encoding: 'utf8', timeout: 90000 })
if (checksumResult.status === 0) {
const expectedLine = fs.readFileSync(checksumPath, 'utf8')
.split('\n')
.find(line => line.trim().endsWith(assetName))
if (expectedLine) {
const expected = expectedLine.trim().split(/\s+/)[0]
const actual = crypto.createHash('sha256').update(fs.readFileSync(zipPath)).digest('hex')
if (expected !== actual) {
throw new Error(
`Checksum mismatch for ${assetName} (${tag}):\n` +
` expected ${expected}\n actual ${actual}`
)
}
console.log('Checksum verified')
}
} else {
console.warn('No checksum file published for this release; skipping verification')
}
const unzipResult = spawnSync('unzip', ['-o', zipPath, '-d', destDir], {
encoding: 'utf8',
timeout: 60000
})
if (unzipResult.status !== 0) {
throw new Error(`Failed to extract ${assetName}: ${unzipResult.stderr}`)
}
const binPath = path.join(destDir, 'rpk')
if (!fs.existsSync(binPath)) {
throw new Error(`Extracted archive did not contain an rpk binary: ${zipPath}`)
}
fs.chmodSync(binPath, 0o755)
return binPath
}
/**
* Get an rpk binary matching the given version.
* Prefers the official release download (published stable releases only;
* RC releases are drafts, so their assets are not publicly downloadable).
* Falls back to building from source at the tag.
* @param {string} rpkVersion - Version tag from the snapshot (e.g., v26.1.12, v26.2.1-rc2)
* @param {Object} [options]
* @param {string} [options.rpkBin] - Existing binary to use, skipping download/build
* @returns {string} Path to an rpk binary
*/
function acquireRpkBinary(rpkVersion, op