@opendevise/antora-git-lfs-extension/lib/index.js

An Antora extension that uses the native git command to clone content source repositories in the playbook marked as lfs enabled and records the oid of all lfs files.

'use strict'

const { createHash } = require('node:crypto')
const expandPath = require('@antora/expand-path-helper')
const fsp = require('node:fs/promises')
const os = require('node:os')
const ospath = require('node:path')
const runCommand = require('@antora/run-command-helper')

const invariably = { false: () => false, newObject: () => ({}) }

module.exports.register = function ({ config }) {
  this.once('playbookBuilt', ({ playbook }) => {
    const defaultBranches = playbook.content.branches
    const lfsContentSources = playbook.content.sources.filter((it) => {
      if (!(it.lfs && ~it.url.indexOf(':') && /:(?:\/\/|[^/\\])/.test(it.url))) return false
      let refs
      if ((refs = refsAsArray(it.commits)).length) {
        it.lfs = { commit: refs[0] }
        return true
      }
      if ((refs = refsAsArray('branches' in it ? it.branches : defaultBranches)).length) {
        it.lfs = { branch: refs[0] }
        return true
      }
      return false
    })
    if (!lfsContentSources.length) return
    let workspaceDir = resolveWorkspaceDir(config.workspaceDir, playbook.runtime.cacheDir, playbook.dir)
    workspaceDir = '.' + ospath.sep + ospath.relative(playbook.dir, workspaceDir)
    for (const contentSource of lfsContentSources) {
      delete contentSource.commits
      contentSource.branches = 'HEAD'
      contentSource.remoteUrl = contentSource.url
      contentSource.url = workspaceDir + ospath.sep + generateWorktreeName(contentSource.url, contentSource.lfs)
    }
    this.updateVariables({ playbook })
  })

  this.once('beforeProcess', async ({ playbook }) => {
    const lfsContentSources = playbook.content.sources.filter((it) => it.lfs && it.remoteUrl)
    if (!lfsContentSources.length) return
    const processed = []
    for (const contentSource of lfsContentSources) {
      const cloneDir = ospath.join(playbook.dir, contentSource.url)
      if (processed.includes(cloneDir)) continue
      processed.push(cloneDir)
      const cached = await fsp.stat(cloneDir).then((stat) => stat.isDirectory(), invariably.false)
      const commit = contentSource.lfs.commit
      if (commit) {
        let fetch
        if (cached) {
          fetch = playbook.runtime.fetch
        } else {
          await fsp.mkdir(cloneDir, { recursive: true })
          await git('init', ['.'], cloneDir)
          fetch = true
        }
        if (fetch) {
          if (!playbook.runtime.quiet) console.log(`[fetch] ${commit} of ${contentSource.remoteUrl}`)
          await git('fetch', [contentSource.remoteUrl, commit], cloneDir)
        }
        await git('checkout', ['FETCH_HEAD'], cloneDir)
      } else {
        let gitCmd
        const gitArgs = []
        let gitCwd
        if (cached) {
          if (playbook.runtime.fetch) gitCmd = 'pull'
          gitCwd = cloneDir
        } else {
          gitCmd = 'clone'
          const branch = contentSource.lfs.branch
          if (!isDefaultBranch(branch)) gitArgs.push('-b', branch)
          gitArgs.push('-q', contentSource.remoteUrl, cloneDir)
        }
        if (gitCmd) {
          if (!playbook.runtime.quiet) console.log(`[${gitCmd}] ${contentSource.remoteUrl}`)
          await git(gitCmd, gitArgs, gitCwd)
        }
      }
    }
  })

  this.once('contentClassified', async ({ contentCatalog }) => {
    const gitLfsFilesByWorktree = {}
    for (const file of contentCatalog.getFiles()) {
      const { webUrl, worktree } = file.src.origin ?? {}
      if (!(webUrl && worktree)) continue
      if (!(worktree in gitLfsFilesByWorktree)) {
        gitLfsFilesByWorktree[worktree] = await git('lfs', ['ls-files', '-l'], worktree).then(
          (data) =>
            data
              .trimEnd()
              .split('\n')
              .reduce((accum, line) => {
                const [oid, filepath] = line.split(/ [*-] /, 2)
                return Object.assign(accum, { [filepath]: webUrl + '#' + oid })
              }, {}),
          invariably.newObject
        )
      }
      file.src.oid = gitLfsFilesByWorktree[worktree][file.src.path]
    }
  })
}

function refsAsArray (refs) {
  if (!refs) return []
  if (Array.isArray(refs)) return refs.map(String)
  return String(refs).split(/\s*,\s*/)
}

function generateWorktreeName (url, { branch, commit }) {
  const normalizedUrl = url.toLowerCase().replace(/\.git/, '')
  const basename = normalizedUrl.split('/').pop()
  const refQualifier = branch ? (isDefaultBranch(branch) ? '' : '-' + branch) : '-' + commit
  return basename + refQualifier + '-' + createHash('sha1').update(normalizedUrl).digest('hex')
}

function getCacheDir (userCacheDir) {
  if (userCacheDir) return userCacheDir
  let basedir
  switch (process.platform) {
    case 'win32':
      if ((basedir = process.env.APPDATA)) return ospath.join(basedir, 'antora', 'Caches')
      break
    case 'darwin':
      if ((basedir = os.homedir())) return path.join(basedir, 'Library/Caches', 'antora')
      break
    case 'linux':
      if ((basedir = process.env.XDG_CACHE_HOME)) return ospath.join(basedir, 'antora')
      if ((basedir = os.homedir())) return ospath.join(basedir, '.cache', 'antora')
  }
  return ospath.join(process.cwd(), '.cache', 'antora')
}

function isDefaultBranch (name) {
  return name === 'HEAD' || name === '.' || name === 'main'
}

function git (command, args, cwd) {
  const baseCall = process.env.GIT_COMMAND || 'git'
  return runCommand(baseCall, [command].concat(args), { cwd }).then((stdout) => stdout.toString().trimEnd())
}

function resolveWorkspaceDir (requestedDir, userCacheDir, startDir) {
  const workspaceDir = requestedDir || getCacheDir(userCacheDir) + ospath.sep + 'content-with-lfs'
  return expandPath(workspaceDir, { dot: startDir })
}