@redpanda-data/docs-extensions-and-macros/tools/fetch-from-github.js

Antora extensions and macros developed for Redpanda documentation.

const fs = require('fs');
const path = require('path');

// Use shared Octokit client
function loadOctokit() {
  const octokit = require('../cli-utils/octokit-client');
  return octokit;
}

async function saveFile(content, saveDir, filename) {
  await fs.promises.mkdir(saveDir, { recursive: true });
  const target = path.join(saveDir, filename);
  await fs.promises.writeFile(target, content);
  console.log(`Saved: ${target}`);
}

/**
 * Fetch file or directory from GitHub repository
 * @param {string} owner - Repository owner (for example, "redpanda-data")
 * @param {string} repo - Repository name (for example, "redpanda-operator")
 * @param {string} remotePath - Path to file or directory in repository
 * @param {string} saveDir - Local directory to save files to
 * @param {string} [customFilename] - Optional custom filename (for single files)
 * @param {string} [ref=null] - Git ref (branch, tag, or commit SHA). Defaults to repository's default branch if null
 * @returns {Promise<void>}
 */
async function fetchFromGithub(owner, repo, remotePath, saveDir, customFilename, ref = null) {
  const octokit = await loadOctokit();

  try {
    const params = { owner, repo, path: remotePath };
    if (ref) {
      params.ref = ref;
    }
    const resp = await octokit.repos.getContent(params);
    if (Array.isArray(resp.data)) {
      // directory
      for (const item of resp.data) {
        if (item.type === 'file') {
          await fetchFromGithub(owner, repo, item.path, saveDir, customFilename, ref);
        } else if (item.type === 'dir') {
          // For directories, maintain the directory structure
          const nestedDir = path.join(saveDir, path.basename(item.path));
          await fetchFromGithub(owner, repo, item.path, nestedDir, null, ref);
        }
      }
    } else {
      // single file
      const content = Buffer.from(resp.data.content, 'base64').toString();
      const filename = customFilename || path.basename(resp.data.path);
      await saveFile(content, saveDir, filename);
    }
  } catch (error) {
    if (error.status === 403 && error.message.includes('rate limit')) {
      throw new Error(`GitHub API rate limit exceeded. Consider using a token via VBOT_GITHUB_API_TOKEN environment variable.`);
    } else if (error.status === 404) {
      throw new Error(`Path not found: ${remotePath} in ${owner}/${repo}`);
    } else {
      throw new Error(`Failed to fetch from GitHub: ${error.message}`);
    }
  }
}

module.exports = fetchFromGithub