@antora/collector-extension
Version:
An Antora extension that runs the specified commands per content root and imports the specified files into the content aggregate.
421 lines (405 loc) • 21.8 kB
JavaScript
const { createHash } = require('node:crypto')
const expandPath = require('@antora/expand-path-helper')
const fs = require('node:fs')
const { promises: fsp } = fs
const getUserCacheDir = require('cache-directory')
const { globStream } = require('fast-glob')
const invariably = { false: () => false, null: () => null, void: () => undefined }
const ospath = require('node:path')
const { posix: path } = ospath
const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
const runCommand = require('@antora/run-command-helper')
const { pipeline, Writable } = require('node:stream')
const forEach = (write) => new Writable({ objectMode: true, write })
const yaml = require('js-yaml')
const CONTEXT_VARIABLE_REF_RX = /\$\{\{ *([a-z0-9_.]+)(?: as (json|yaml))? *\}\}/gi
const GLOB_OPTS = { ignore: ['.git'], objectMode: true, onlyFiles: false, unique: false }
const PACKAGE_NAME = require('../package.json').name
module.exports.register = function ({ config: { createWorktrees = 'auto', keepWorktrees = false } }) {
this.once('contentAggregated', async ({ playbook, contentAggregate }) => {
let logger
const playbookEnv = playbook.env
const silent = playbook.runtime?.silent
const quiet = silent || playbook.runtime?.quiet
const cacheDir = ospath.join(getBaseCacheDir(playbook), 'collector')
await fsp.mkdir(cacheDir, { recursive: true })
const git = this.require('@antora/content-aggregator/git')
const gitCache = {}
const runCache = { where: {} }
const managedWorktrees = new Map()
for (const componentVersionBucket of contentAggregate.slice()) {
const { origins = [] } = componentVersionBucket
for (const origin of origins) {
const { url, gitdir, refname, reftype, remote, worktree, startPath, descriptor } = origin
let collectorConfig = descriptor?.ext?.collector || []
if (!(Array.isArray(collectorConfig) ? collectorConfig.length : (collectorConfig = [collectorConfig]))) continue
let worktreeDir = worktree
let worktreeConfig = collectorConfig[0].worktree
if (!worktreeConfig) worktreeConfig = worktreeConfig === false ? { create: 'always' } : {}
if (worktreeConfig.checkout === 'always') worktreeConfig.checkout = !!(worktreeConfig.create = 'always')
const createWorktree = !worktree || (worktreeConfig.create ?? createWorktrees) === 'always'
const checkoutWorktree = worktreeConfig.checkout !== false
if (createWorktree) {
const keepWorktree = 'keep' in worktreeConfig ? worktreeConfig.keep : keepWorktrees
worktreeDir = ospath.join(cacheDir, generateWorktreeFolderName(origin, keepWorktree))
if (managedWorktrees.has(worktreeDir)) {
managedWorktrees.get(worktreeDir).origins.add(origin)
if (!checkoutWorktree) await fsp.rm(worktreeDir, { force: true, recursive: true })
} else {
managedWorktrees.set(worktreeDir, { origins: new Set([origin]), keep: keepWorktree })
if (!checkoutWorktree || keepWorktree !== true) await fsp.rm(worktreeDir, { force: true, recursive: true })
}
}
origin.collectorWorktree = worktreeDir
const expandPathContext = { base: worktreeDir, cwd: worktreeDir, dot: ospath.join(worktreeDir, startPath) }
const collectors = collectorConfig.map((collector) => {
const { clean: cleanConfig = [], run: runConfig = {}, scan: scanConfig = [] } = collector
let cleans
return {
clean: (cleans = (Array.isArray(cleanConfig) ? cleanConfig : [cleanConfig]).reduce((accum, clean) => {
if (typeof clean === 'string' ? (clean = { dir: clean }) : typeof clean.dir === 'string') {
accum.push({ dir: expandPath(clean.dir, expandPathContext) })
}
return accum
}, [])),
run: (Array.isArray(runConfig) ? runConfig : [runConfig]).reduce((accum, run) => {
if (typeof run === 'string' ? (run = { command: run }) : typeof run.command === 'string') {
let command = run.command.trim()
if (!command) return accum
const dir = typeof run.dir === 'string' ? expandPath(run.dir, expandPathContext) : worktreeDir
const nodeExecPath = process.execPath
const nodeExecDir = ospath.dirname(nodeExecPath)
const env = Object.assign({}, playbookEnv, {
NODE: nodeExecPath,
NPM: `${nodeExecDir}${ospath.sep}npm`,
NPX: `${nodeExecDir}${ospath.sep}npx`,
PWD: dir,
})
if (env.npm_command === 'exec') {
delete env.npm_config_package
delete env.npm_config_yes
}
let envConfig = run.env
Object.isFrozen(playbook) &&
Object.defineProperty((playbook = Object.assign({}, playbook)), 'env', { enumerable: false })
const contextVars = { origin, playbook, env }
if (envConfig && typeof envConfig === 'object') {
if (!Array.isArray(envConfig)) {
envConfig = Object.entries(envConfig).map(([name, value]) => ({ name, value }))
}
const evaluateWithVars = evaluate.bind(null, contextVars)
for (const envEntry of envConfig) {
if (envEntry && typeof envEntry === 'object' && envEntry.name) {
env[envEntry.name] = evaluateWithVars(envEntry.value)
}
}
}
let local = !!run.local
if (!local) {
if ((local = command.startsWith('./'))) {
command = command.slice(2)
} else if (command.startsWith('$N') && /^\$(?:NODE|NPM|NPX) /.test(command)) {
const spaceIdx = command.indexOf(' ')
command = `"${contextVars.env[command.slice(1, spaceIdx)]}"${command.slice(spaceIdx)}`
} else if (command.split(' ', 1)[0].endsWith('.js')) {
command = `"${contextVars.env.NODE}" ${command}`
}
}
command = evaluate(contextVars, command)
const failure = 'failure' in run ? run.failure : 'on_failure' in run ? run.on_failure : run.onFailure
accum.push(Object.assign({}, run, { command, dir, env, local, shell: !!run.shell, failure }))
}
return accum
}, []),
scan: (Array.isArray(scanConfig) ? scanConfig : [scanConfig]).reduce((accum, scan) => {
if (typeof scan === 'string' ? (scan = { dir: scan }) : typeof scan.dir === 'string') {
const dir = expandPath(scan.dir, expandPathContext)
if (scan.clean) cleans.push({ dir })
const into = ('into' in scan ? scan.into : scan.base) || {}
accum.push(Object.assign({}, scan, { dir, into: typeof into === 'string' ? { dir: into } : into }))
}
return accum
}, []),
}
})
if (createWorktree) {
if (checkoutWorktree) {
const cache = gitCache[gitdir] || (gitCache[gitdir] = {})
const ref = `refs/${reftype === 'branch' ? 'head' : reftype}s/${refname}`
const bare = worktree === undefined
await prepareWorktree(git, { fs, cache, dir: worktreeDir, gitdir, ref, remote, bare })
} else {
await fsp.mkdir(worktreeDir, { recursive: true })
}
}
for (const { clean: cleans, run: runs, scan: scans } of collectors) {
for (const clean of cleans) await fsp.rm(clean.dir, { recursive: true, force: true })
for (const { dir: cwd, command, local, shell, env, failure } of runs) {
let commandContext
try {
if (!quiet && (logger ??= this.getLogger(PACKAGE_NAME)).isLevelEnabled('info')) {
commandContext = createCommandContext(url, remote, startPath, worktree, reftype, refname)
logger.info(`run.command: ${command}${commandContext}`)
}
const stdout = quiet ? 'ignore' : 'print'
const stderr = silent ? 'ignore' : 'print'
await runCommand(command, { cwd, local, shell, env, stdout, stderr, cache: runCache })
} catch (err) {
commandContext ??= createCommandContext(url, remote, startPath, worktree, reftype, refname)
err.message = err.message.replace(/$/m, commandContext)
const failureAction = typeof failure === 'string' ? failure : 'throw'
if (failureAction === 'throw') throw Object.assign(err, { message: `(${PACKAGE_NAME}): ${err.message}` })
if (failureAction === 'log' || failureAction.startsWith('log.')) {
;(logger ??= this.getLogger(PACKAGE_NAME))[failureAction.split('log.')[1] || 'error'](err)
}
}
}
for (const { dir, files, into, private: private_ } of scans) {
let componentVersionDesc
const scannedFiles = await srcFs(dir, files, into.dir).then((result) =>
result.filter((file) => (file.path === 'antora.yml' ? !(componentVersionDesc = file) : true))
)
let requestedBucketName = 'name' in into ? String(into.name) : componentVersionBucket.name
let requestedBucketVersion = 'version' in into ? String(into.version || '') : componentVersionBucket.version
let targetBucket = componentVersionBucket
let targetOrigin = origin
let replaceTargetBucket
if ((componentVersionDesc &&= loadComponentVersionDescriptor(componentVersionDesc))) {
if ('name' in componentVersionDesc) requestedBucketName = String(componentVersionDesc.name)
if ('version' in componentVersionDesc) requestedBucketVersion = String(componentVersionDesc.version || '')
// maybe move this logic to below (and add check for componentVersionDesc)?
if (
(replaceTargetBucket = !componentVersionDesc.create) &&
!(targetBucket.name === requestedBucketName && targetBucket.version === requestedBucketVersion)
) {
if (
contentAggregate.find(
(candidate) =>
candidate.name === requestedBucketName && candidate.version === requestedBucketVersion
)
) {
const ctx = createCommandContext(url, remote, startPath, worktree, reftype, refname)
const identity = `${requestedBucketVersion}@${requestedBucketName}`
const msg = `Cannot update identity of bucket to ${identity} since bucket with that identity exists`
throw new Error(`(${PACKAGE_NAME}): ${msg}${ctx}`)
}
}
delete componentVersionDesc.create
Object.assign(componentVersionDesc, { name: requestedBucketName, version: requestedBucketVersion })
}
if (
!replaceTargetBucket &&
!(targetBucket.name === requestedBucketName && targetBucket.version === requestedBucketVersion)
) {
targetBucket = contentAggregate.find(
(candidate) => candidate.name === requestedBucketName && candidate.version === requestedBucketVersion
)
if (targetBucket) {
targetOrigin = targetBucket.nav?.origin // nav.origin holds the primary descriptor
} else {
const descriptor = { name: requestedBucketName, version: requestedBucketVersion }
targetOrigin = Object.assign({}, origin, { descriptor })
managedWorktrees.has(targetOrigin.collectorWorktree) &&
managedWorktrees.get(targetOrigin.collectorWorktree).origins.add(targetOrigin)
targetBucket = Object.assign({}, descriptor, {
files: [],
nav: Object.assign([], { origin: targetOrigin }),
origins: [targetOrigin],
})
contentAggregate.push(targetBucket)
}
}
if (componentVersionDesc) {
Object.assign(targetBucket, componentVersionDesc)
if (targetOrigin) Object.assign(targetOrigin.descriptor, componentVersionDesc)
if (componentVersionDesc.nav != null) targetBucket.nav.origin = targetOrigin
if (replaceTargetBucket && componentVersionDesc.prerelease == null) {
if ('prerelease' in componentVersionDesc) delete targetBucket.prerelease
}
}
const targetFiles = targetBucket.files
for (const file of scannedFiles) {
const relpath = file.path
const existingFile = targetFiles.find((it) => it.path === relpath)
if (private_ != null) Object.assign(existingFile || file, { private: private_ })
if (existingFile) {
Object.assign(existingFile, { contents: file.contents, stat: file.stat })
} else {
const src = file.src
const scannedRelpath = src.abspath.slice(worktreeDir.length + 1)
Object.assign(src, { origin, scanned: posixify ? posixify(scannedRelpath) : scannedRelpath })
if (createWorktree) Object.assign(src, { realpath: src.abspath, abspath: src.scanned })
targetFiles.push(file)
}
}
}
}
}
}
const deferredWorktreeRemovals = new Map()
for (const [worktreeDir, { origins, keep }] of managedWorktrees) {
if (keep === true) continue
if (typeof keep === 'string' && keep.startsWith('until:')) {
const eventName = keep === 'until:exit' ? 'contextClosed' : keep.slice(6)
const removal = { worktreeDir, origins }
const removals = deferredWorktreeRemovals.get(eventName)
removals ? removals.push(removal) : deferredWorktreeRemovals.set(eventName, [removal])
continue
}
await removeWorktree(worktreeDir, origins)
}
for (const [eventName, removals] of deferredWorktreeRemovals) {
this.once(eventName, () =>
Promise.all(removals.map(({ worktreeDir, origins }) => removeWorktree(worktreeDir, origins)))
)
}
})
}
/**
* Prepares a git worktree from the specified gitdir, making use of the existing clone. If the worktree already exists
* from a previous iteration, the worktree is reset. A valid worktree is one that contains a .git/index file. Otherwise,
* a fresh worktree is created. If the gitdir contains an index file, that index file is temporarily overwritten to
* prepare the worktree and later restored before the function returns.
*/
async function prepareWorktree (git, repo) {
const { dir: worktreeDir, gitdir, ref, remote = 'origin', bare, cache } = repo
delete repo.remote
const currentIndexPath = ospath.join(gitdir, 'index')
const currentIndexPathBak = currentIndexPath + '~'
const restoreIndex = (await fsp.rename(currentIndexPath, currentIndexPathBak).catch(invariably.false)) === undefined
const worktreeGitdir = ospath.join(worktreeDir, '.git')
const worktreeIndexPath = ospath.join(worktreeGitdir, 'index')
try {
let force = true
try {
await mv(worktreeIndexPath, currentIndexPath) // isomorphic-git requires index to be in commondir
await removeUntrackedFiles(git, repo)
} catch {
force = false
await fsp.unlink(currentIndexPath).catch(invariably.void) // index file not needed in this case
await fsp.rm(worktreeDir, { recursive: true, force: true })
await fsp.mkdir(worktreeGitdir, { recursive: true })
Reflect.ownKeys(cache).forEach((it) => it.toString() === 'Symbol(PackfileCache)' || delete cache[it])
}
let head
if (ref.startsWith('refs/heads/')) {
head = `ref: ${ref}`
const branchName = ref.slice(11)
if (bare || !(await git.listBranches(repo)).includes(branchName)) {
await git.branch({ ...repo, ref: branchName, object: `refs/remotes/${remote}/${branchName}`, force: true })
}
} else {
head = await git.resolveRef(repo)
}
await git.checkout({ ...repo, force, noUpdateHead: true, track: false })
await fsp.writeFile(ospath.join(worktreeGitdir, 'commondir'), `${gitdir}\n`, 'utf8')
await fsp.writeFile(ospath.join(worktreeGitdir, 'HEAD'), `${head}\n`, 'utf8')
await mv(currentIndexPath, worktreeIndexPath)
} finally {
if (restoreIndex) await fsp.rename(currentIndexPathBak, currentIndexPath)
}
}
async function removeWorktree (worktreeDir, origins) {
for (const origin of origins) delete origin.collectorWorktree
await fsp.rm(worktreeDir, { recursive: true })
}
function generateWorktreeFolderName ({ url, gitdir, refname, worktree }, keepWorktrees) {
const refnameQualifier = keepWorktrees ? '@' + refname.replace(/[/]/g, '-') : undefined
if (worktree === undefined) {
const folderName = ospath.basename(gitdir, '.git')
if (!refnameQualifier) return folderName
const lastHyphenIdx = folderName.lastIndexOf('-')
return `${folderName.slice(0, lastHyphenIdx)}${refnameQualifier}${folderName.slice(lastHyphenIdx)}`
}
let normalizedUrl = (url || gitdir).toLowerCase()
if (posixify) normalizedUrl = posixify(normalizedUrl)
normalizedUrl = normalizedUrl.replace(/(?:[/]?\.git|[/])$/, '')
const slug = ospath.basename(normalizedUrl) + (refnameQualifier || '')
const hash = createHash('sha1').update(normalizedUrl).digest('hex')
return `${slug}-${hash}`
}
function getBaseCacheDir ({ dir: dot, runtime: { cacheDir: preferredDir } }) {
return preferredDir == null
? getUserCacheDir(`antora${process.env.NODE_ENV === 'test' ? '-test' : ''}`) || ospath.join(dot, '.cache/antora')
: expandPath(preferredDir, { dot })
}
function srcFs (cwd, globs = '**/*', into = undefined) {
return new Promise((resolve, reject, accum = []) =>
pipeline(
globStream(globs, Object.assign({ cwd }, GLOB_OPTS)),
forEach(({ path: relpath, dirent }, _, done) => {
if (dirent.isDirectory()) return done()
let relpathPosix = relpath
const abspath = posixify ? ospath.join(cwd, (relpath = ospath.normalize(relpath))) : cwd + '/' + relpath
fsp.stat(abspath).then((stat) => {
fsp.readFile(abspath).then((contents) => {
const basename = ospath.basename(relpathPosix)
const extname = ospath.extname(relpathPosix)
const stem = basename.slice(0, basename.length - extname.length)
if (into) relpathPosix = path.join('.', into, relpathPosix)
accum.push({
path: relpathPosix,
contents,
stat,
src: { path: relpathPosix, basename, stem, extname, abspath },
})
done()
}, done)
}, done)
}),
(err) => (err ? reject(err) : resolve(accum))
)
)
}
function loadComponentVersionDescriptor (file) {
try {
return camelCaseKeys(yaml.load(file.contents, { schema: yaml.CORE_SCHEMA }), ['asciidoc', 'ext'])
} catch (err) {
throw Object.assign(err, { message: `(${PACKAGE_NAME}): antora.yml has invalid syntax; ${err.message}` })
}
}
function camelCaseKeys (o, stopPaths = [], p = '') {
if (Array.isArray(o)) return o.map((it) => camelCaseKeys(it, stopPaths, p))
if (o == null || o.constructor !== Object) return o
const pathPrefix = p && p + '.'
const accum = {}
for (const [k, v] of Object.entries(o)) {
const camelKey = k.toLowerCase().replace(/[_-]([a-z0-9])/g, (_, l, idx) => (idx ? l.toUpperCase() : l))
accum[camelKey] = ~stopPaths.indexOf(pathPrefix + camelKey) ? v : camelCaseKeys(v, stopPaths, pathPrefix + camelKey)
}
return accum
}
function removeUntrackedFiles (git, repo) {
const trees = [git.STAGE({}), git.WORKDIR()]
const map = (relpath, [sEntry]) => {
if (relpath === '.') return
if (relpath === '.git') return null
if (sEntry == null) return fsp.rm(ospath.join(repo.dir, relpath), { recursive: true }).then(invariably.null)
return sEntry.mode().then((mode) => (mode === 0o120000 ? null : undefined))
}
return git.walk({ ...repo, trees, map })
}
function mv (from, to) {
return fsp.cp(from, to).then(() => fsp.rm(from))
}
function evaluate (vars, expression) {
if (typeof expression !== 'string') return expression == null ? undefined : String(expression)
if (!~expression.indexOf('${{')) return expression
return expression.replace(CONTEXT_VARIABLE_REF_RX, (_, propertyRef, format) => {
const value = dereferenceProperty(propertyRef, vars)
if (format && (format = format.toLowerCase()) === 'json') return JSON.stringify(value)
if (format === 'yaml') return yaml.dump(value, { noArrayIndent: true })
return String(value ?? '')
})
}
function dereferenceProperty (propertyRef, vars) {
const [propertyRoot, ...propertyPath] = propertyRef.split('.')
if (!(propertyRoot in vars)) return
return propertyPath.length ? propertyPath.reduce((v, p) => v?.[p], vars[propertyRoot]) : vars[propertyRoot]
}
function createCommandContext (url, remote, startPath, worktree, reftype, refname) {
const refQualifier = worktree ? ' <worktree>' : remote && worktree === false ? ` <remotes/${remote}>` : ''
const pathInfo = startPath ? ` | start path: ${startPath}` : ''
return ` (url: ${worktree || url} | ${reftype}: ${refname}${refQualifier}${pathInfo})`
}