@redpanda-data/docs-extensions-and-macros/tools/redpanda-connect/migrate-metadata-to-partials.js

Antora extensions and macros developed for Redpanda documentation.

'use strict';

/**
 * One-time migration for the metadata-partial change.
 *
 * Existing connector reference pages carry their `== Metadata` block inline in
 * the main page (`modules/components/pages/<type>/<name>.adoc`), where normal
 * regeneration never refreshes it. This script moves that block into a
 * regenerated partial (`modules/components/partials/metadata/<type>/<name>.adoc`)
 * and replaces the inline block in the page with an include directive, matching
 * the pattern already used for the fields and examples partials.
 *
 * After migration, `doc-tools generate rpcn-connector-docs` keeps the partial in
 * sync with the connector's upstream description on every run.
 *
 * Usage (run from the docs repo root):
 *   npx doc-tools generate migrate-rpcn-metadata            # dry run, reports changes
 *   npx doc-tools generate migrate-rpcn-metadata --write    # apply changes
 */

const fs = require('fs');
const path = require('path');
const { globSync } = require('glob');
const { locateMetadata, metadataIncludeLine } = require('./metadata-utils.js');

const PARTIAL_BANNER =
  '// This content is autogenerated. Do not edit manually. Metadata fields come from the connector\'s description in the Connect source. To change them, update the connector description upstream, or use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1247543314/Generate+reference+docs+for+Redpanda+Connect';

const PAGES_ROOT = path.resolve(process.cwd(), 'modules/components/pages');
const PARTIALS_ROOT = path.resolve(process.cwd(), 'modules/components/partials/metadata');

/**
 * True when a generated metadata partial carries no actual metadata — either it
 * is only the autogeneration banner / comments, or it has no `== Metadata`
 * heading. The generator writes such a partial when the connector's upstream
 * description has no `== Metadata` section.
 * @param {string} partial
 * @returns {boolean}
 */
function isEmptyMetadataPartial (partial) {
  // Reuse the same literal-block-aware parser used for extraction, so a
  // `== Metadata` line inside a `----` code block (for example an example that
  // literally contains that text) is not mistaken for a real metadata section.
  return !locateMetadata(partial);
}

/** Recursively collect .adoc files under a directory (minimatch glob). */
function collectAdocFiles (dir) {
  if (!fs.existsSync(dir)) return [];
  return globSync('**/*.adoc', { cwd: dir, absolute: true, nodir: true });
}

/**
 * Migrate inline `== Metadata` blocks in connector pages to regenerated
 * partials plus include directives.
 * @param {object} [options]
 * @param {boolean} [options.write=false] Apply changes (otherwise dry run).
 * @returns {{migrated:number, skipped:number}}
 */
function migrateMetadataToPartials ({ write = false } = {}) {
  const pages = collectAdocFiles(PAGES_ROOT);

  let migrated = 0;
  let skipped = 0;

  for (const page of pages) {
    // Connector pages live in a <type> subdirectory (pages/<type>/<name>.adoc).
    // Skip pages directly under pages/ — they are not connector pages, and the
    // parent-directory name would be a bogus type.
    if (path.dirname(page) === PAGES_ROOT) { skipped++; continue; }

    const content = fs.readFileSync(page, 'utf8');
    const found = locateMetadata(content);
    if (!found) { skipped++; continue; }

    // Derive type directory and connector name from the page path.
    const typeDir = path.basename(path.dirname(page));
    const name = path.basename(page, '.adoc');
    const item = { typeDir, name };

    // If the block is already an include, nothing to do.
    if (/include::connect:components:partial\$metadata\//.test(found.block)) { skipped++; continue; }

    const partialPath = path.join(PARTIALS_ROOT, typeDir, `${name}.adoc`);
    const partialContent = `${PARTIAL_BANNER}\n\n${found.block}\n`;
    const newPage = content.slice(0, found.start) + metadataIncludeLine(item) + content.slice(found.end);

    // Prefer a partial already produced by the generator (source of truth).
    // Only seed one from the page's inline block when none exists yet, so the
    // migration works standalone but never clobbers generated content.
    const partialExists = fs.existsSync(partialPath);

    // Guard against metadata loss: when the generator produced an EMPTY partial
    // (the connector's upstream description has no == Metadata section), the
    // page's inline block is the only real metadata. Migrating it to an include
    // would replace live metadata with an empty partial. Skip these — the
    // connector's upstream description must gain a == Metadata section before
    // the page can be migrated.
    if (partialExists && isEmptyMetadataPartial(fs.readFileSync(partialPath, 'utf8'))) {
      console.log(`SKIP (generated partial is empty; keeping inline metadata): ${typeDir}/${name}`);
      skipped++;
      continue;
    }

    console.log(`${write ? 'MIGRATE' : 'WOULD MIGRATE'}: ${typeDir}/${name}`);
    console.log(`  -> partial: ${path.relative(process.cwd(), partialPath)}${partialExists ? ' (exists, kept)' : ' (seeded from page)'}`);

    if (write) {
      if (!partialExists) {
        fs.mkdirSync(path.dirname(partialPath), { recursive: true });
        fs.writeFileSync(partialPath, partialContent);
      }
      fs.writeFileSync(page, newPage);
    }
    migrated++;
  }

  console.log(`\n${write ? 'Migrated' : 'Would migrate'} ${migrated} page(s); ${skipped} without an inline metadata section.`);
  if (!write) console.log('Dry run only. Re-run with --write to apply.');

  return { migrated, skipped };
}

module.exports = { migrateMetadataToPartials, isEmptyMetadataPartial };

// Allow direct execution for local development; the supported entry point is
// `npx doc-tools generate migrate-rpcn-metadata`.
if (require.main === module) {
  migrateMetadataToPartials({ write: process.argv.includes('--write') });
}