@redpanda-data/docs-extensions-and-macros/tools/property-extractor/generate-handlebars-docs.js

Antora extensions and macros developed for Redpanda documentation.

const AUTOGEN_NOTICE = '// This content is autogenerated. Do not edit manually. To override descriptions, use the doc-tools CLI with the --overrides option: https://redpandadata.atlassian.net/wiki/spaces/DOC/pages/1396244485/Review+Redpanda+configuration+properties\n';
'use strict';

const fs = require('fs');
const path = require('path');
const handlebars = require('handlebars');
const helpers = require('./helpers');

/**
 * Handlebars documentation generator for Redpanda configuration properties.
 *
 * Supports custom template overrides using environment variables:
 * - TEMPLATE_PROPERTY: Individual property section template
 * - TEMPLATE_TOPIC_PROPERTY: Individual topic property section template
 * - TEMPLATE_DEPRECATED_PROPERTY: Individual deprecated property section template
 * - TEMPLATE_DEPRECATED: Deprecated properties page template
 *
 * Behavior flags (environment variables):
 * - GENERATE_PARTIALS=1      Generate consolidated property partials and deprecated partials
 * - OUTPUT_PARTIALS_DIR=<p>  Destination for consolidated partials (required if GENERATE_PARTIALS=1)
 *
 * CLI Usage: node generate-handlebars-docs.js <input-file> <output-dir>
 */

// Register helpers
Object.entries(helpers).forEach(([name, fn]) => {
  if (typeof fn !== 'function') {
    console.error(`Error: Helper "${name}" is not a function`);
    process.exit(1);
  }
  handlebars.registerHelper(name, fn);
});

/**
 * Determines if a property is related to object storage.
 * @param {Object} prop - The property object
 * @returns {boolean} True if the property is object storage related
 */
function isObjectStorageProperty(prop) {
  return prop.name && (
    prop.name.includes('cloud_storage') ||
    prop.name.includes('s3_') ||
    prop.name.includes('azure_') ||
    prop.name.includes('gcs_') ||
    prop.name.includes('archival_') ||
    prop.name.includes('remote_') ||
    prop.name.includes('tiered_')
  );
}

/**
 * Gets template path, checking environment variables for custom paths first
 */
function getTemplatePath(defaultPath, envVar) {
  const customPath = process.env[envVar];
  if (customPath && fs.existsSync(customPath)) {
    console.log(`📄 Using custom template: ${customPath}`);
    return customPath;
  }
  return defaultPath;
}

/**
 * Register Handlebars partials used to render property documentation.
 */
function registerPartials() {
  const templatesDir = path.join(__dirname, 'templates');

  try {
    console.log('📝 Registering Handlebars templates');

    const propertyTemplatePath = getTemplatePath(
      path.join(templatesDir, 'property.hbs'),
      'TEMPLATE_PROPERTY'
    );
    if (!fs.existsSync(propertyTemplatePath)) {
      throw new Error(`Property template not found: ${propertyTemplatePath}`);
    }
    handlebars.registerPartial('property', fs.readFileSync(propertyTemplatePath, 'utf8'));

    const topicPropertyTemplatePath = getTemplatePath(
      path.join(templatesDir, 'topic-property.hbs'),
      'TEMPLATE_TOPIC_PROPERTY'
    );
    if (!fs.existsSync(topicPropertyTemplatePath)) {
      throw new Error(`Topic property template not found: ${topicPropertyTemplatePath}`);
    }
    handlebars.registerPartial('topic-property', fs.readFileSync(topicPropertyTemplatePath, 'utf8'));

    const deprecatedPropertyTemplatePath = getTemplatePath(
      path.join(templatesDir, 'deprecated-property.hbs'),
      'TEMPLATE_DEPRECATED_PROPERTY'
    );
    if (!fs.existsSync(deprecatedPropertyTemplatePath)) {
      throw new Error(`Deprecated property template not found: ${deprecatedPropertyTemplatePath}`);
    }
    handlebars.registerPartial('deprecated-property', fs.readFileSync(deprecatedPropertyTemplatePath, 'utf8'));

    console.log('Done: Registered all partials');
  } catch (error) {
    console.error('Error: Failed to register Handlebars templates:');
    console.error(`   ${error.message}`);
    throw error;
  }
}

/**
 * Generate AsciiDoc partial files grouping input properties by scope (cluster, topic, broker, object-storage).
 *
 * Reads property and topic templates, groups provided properties by their config_scope (treating keys as authoritative property names),
 * renders each property into the appropriate template, writes combined partial files to "<partialsDir>/properties/<type>-properties.adoc",
 * and invokes the optional onRender callback for every rendered property name. Entries missing a name or config_scope are skipped;
 * duplicate keys are detected, warned about, and skipped.
 *
 * @param {Object<string, Object>} properties - Map of property key → property object; the map key is used as the property's name.
 * @param {string} partialsDir - Destination directory under which a "properties" subdirectory will be created for output files.
 * @param {(name: string) => void} [onRender] - Optional callback invoked with each rendered property's name.
 * @returns {number} The total number of properties rendered and written to partial files.
 */
function generatePropertyPartials(properties, partialsDir, onRender) {
  console.log(`📝 Generating consolidated property partials in ${partialsDir}…`);

  const propertyTemplate = handlebars.compile(
    fs.readFileSync(getTemplatePath(path.join(__dirname, 'templates', 'property.hbs'), 'TEMPLATE_PROPERTY'), 'utf8')
  );
  const topicTemplate = handlebars.compile(
    fs.readFileSync(getTemplatePath(path.join(__dirname, 'templates', 'topic-property.hbs'), 'TEMPLATE_TOPIC_PROPERTY'), 'utf8')
  );

  const propertiesPartialsDir = path.join(partialsDir, 'properties');
  fs.mkdirSync(propertiesPartialsDir, { recursive: true });

  const propertyGroups = { cluster: [], topic: [], broker: [], 'object-storage': [] };

  // Track processed property keys to detect duplicates by unique key
  const processedKeys = new Set();
  
  Object.entries(properties).forEach(([key, prop]) => {
    if (!prop.name || !prop.config_scope) return;

    // Skip if we've already processed this key
    if (processedKeys.has(key)) {
      console.warn(`Warning: Duplicate key detected: ${key}`);
      return;
    }
    processedKeys.add(key);

    // Ensure the property uses the key as its name for consistency
    // This fixes issues where key != name field due to bugs in the source code
    prop.name = key;

    switch (prop.config_scope) {
      case 'topic':
        propertyGroups.topic.push(prop);
        break;
      case 'broker':
        propertyGroups.broker.push(prop);
        break;
      case 'cluster':
        if (isObjectStorageProperty(prop)) propertyGroups['object-storage'].push(prop);
        else propertyGroups.cluster.push(prop);
        break;
      case 'object-storage':
        propertyGroups['object-storage'].push(prop);
        break;
      default:
        console.warn(`Warning: Unknown config_scope: ${prop.config_scope} for ${prop.name}`);
        break;
    }
  });

  let totalCount = 0;

  Object.entries(propertyGroups).forEach(([type, props]) => {
    if (props.length === 0) return;
    props.sort((a, b) => String(a.name || '').localeCompare(String(b.name || '')));
    const selectedTemplate = type === 'topic' ? topicTemplate : propertyTemplate;
    const pieces = [];
    props.forEach(p => {
      if (typeof onRender === 'function') {
        try { onRender(p.name); } catch (err) { /* swallow callback errors */ }
      }
      pieces.push(selectedTemplate(p));
    });
    const content = pieces.join('\n');
    const filename = `${type}-properties.adoc`;
    fs.writeFileSync(path.join(propertiesPartialsDir, filename), AUTOGEN_NOTICE + content, 'utf8');
    console.log(`Done: Generated ${filename} (${props.length} properties)`);
    totalCount += props.length;
  });

  console.log(`Done: Done. ${totalCount} total properties.`);
  return totalCount;
}

/**
 * Generate deprecated properties documentation.
 */
function generateDeprecatedDocs(properties, outputDir) {
  const templatePath = getTemplatePath(
    path.join(__dirname, 'templates', 'deprecated-properties.hbs'),
    'TEMPLATE_DEPRECATED'
  );
  const template = handlebars.compile(fs.readFileSync(templatePath, 'utf8'));

  const deprecatedProperties = Object.values(properties).filter(p => p.is_deprecated);
  const brokerProperties = deprecatedProperties
    .filter(p => p.config_scope === 'broker')
    .sort((a, b) => a.name.localeCompare(b.name));
  const clusterProperties = deprecatedProperties
    .filter(p => p.config_scope === 'cluster')
    .sort((a, b) => a.name.localeCompare(b.name));

  const data = {
    deprecated: deprecatedProperties.length > 0,
    brokerProperties: brokerProperties.length ? brokerProperties : null,
    clusterProperties: clusterProperties.length ? clusterProperties : null
  };

  const outputPath = process.env.OUTPUT_PARTIALS_DIR
    ? path.join(process.env.OUTPUT_PARTIALS_DIR, 'deprecated', 'deprecated-properties.adoc')
    : path.join(outputDir, 'partials', 'deprecated', 'deprecated-properties.adoc');

  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
  fs.writeFileSync(outputPath, AUTOGEN_NOTICE + template(data), 'utf8');
  console.log(`Done: Generated ${outputPath}`);
  return deprecatedProperties.length;
}

/**
 * Generate topic-property-mappings.adoc
 */
function generateTopicPropertyMappings(properties, partialsDir) {
  const templatesDir = path.join(__dirname, 'templates');
  const mappingsTemplatePath = getTemplatePath(
    path.join(templatesDir, 'topic-property-mappings.hbs'),
    'TEMPLATE_TOPIC_PROPERTY_MAPPINGS'
  );
  if (!fs.existsSync(mappingsTemplatePath)) {
    throw new Error(`topic-property-mappings.hbs template not found: ${mappingsTemplatePath}`);
  }
  const topicProperties = Object.values(properties).filter(
    p => p.is_topic_property && p.corresponding_cluster_property && !p.exclude_from_docs
  );
  if (topicProperties.length === 0) {
    console.log('ℹ️ No topic properties with corresponding_cluster_property found. Skipping topic-property-mappings.adoc.');
    return 0;
  }
  const hbsSource = fs.readFileSync(mappingsTemplatePath, 'utf8');
  const hbs = handlebars.compile(hbsSource);
  const rendered = hbs({ topicProperties });
  const propertiesPartialsDir = path.join(partialsDir, 'properties');
  fs.mkdirSync(propertiesPartialsDir, { recursive: true });
  const mappingsOut = path.join(propertiesPartialsDir, 'topic-property-mappings.adoc');
  fs.writeFileSync(mappingsOut, AUTOGEN_NOTICE + rendered, 'utf8');
  console.log(`Done: Generated ${mappingsOut}`);
  return topicProperties.length;
}

/**
 * Generate error reports for missing descriptions, deprecated, and undocumented properties.
 */
function generateErrorReports(properties, documentedProperties = []) {
  const emptyDescriptions = [];
  const deprecatedProperties = [];
  const allKeys = Object.keys(properties);

  // Use documentedProperties array (property names that were rendered into partials)
  const documentedSet = new Set(documentedProperties);
  const undocumented = [];

    Object.entries(properties).forEach(([key, p]) => {
    const name = p.name || key;
    const desc = p.description;

    if (p.is_deprecated) deprecatedProperties.push(name);

    // Ensure description is a non-empty string (exclude deprecated properties)
    if (!p.is_deprecated && (typeof desc !== 'string' || desc.trim().length === 0)) {
      emptyDescriptions.push(name);
    }

    if (!documentedSet.has(name)) undocumented.push(name);
  });


  const total = allKeys.length;
  const nonDeprecatedTotal = total - deprecatedProperties.length;
  const pctEmpty = nonDeprecatedTotal ? ((emptyDescriptions.length / nonDeprecatedTotal) * 100).toFixed(2) : '0.00';
  const pctDeprecated = total ? ((deprecatedProperties.length / total) * 100).toFixed(2) : '0.00';
  const pctUndocumented = total ? ((undocumented.length / total) * 100).toFixed(2) : '0.00';

  console.log(`Empty descriptions: ${emptyDescriptions.length} (${pctEmpty}%) (excludes deprecated)`);
  console.log(`Deprecated: ${deprecatedProperties.length} (${pctDeprecated}%)`);
  console.log(`Not documented: ${undocumented.length} (${pctUndocumented}%)`);

  return {
    empty_descriptions: emptyDescriptions.sort(),
    deprecated_properties: deprecatedProperties.sort(),
    undocumented_properties: undocumented.sort(),
  };
}

/**
 * Main generator
 */
function generateAllDocs(inputFile, outputDir) {
  const data = JSON.parse(fs.readFileSync(inputFile, 'utf8'));
  // Support both 'properties' (from property_extractor.py) and 'topic_properties' (from topic_property_extractor.py)
  const properties = data.properties || data.topic_properties || {};

  registerPartials();

  let partialsCount = 0;
  let deprecatedCount = 0;
  const documentedProps = []; // Track which property names were rendered

  if (process.env.GENERATE_PARTIALS === '1' && process.env.OUTPUT_PARTIALS_DIR) {
    console.log('📄 Generating property partials and deprecated docs...');
    deprecatedCount = generateDeprecatedDocs(properties, outputDir);

    // Generate property partials using the shared helper and collect names via callback
    partialsCount = generatePropertyPartials(properties, process.env.OUTPUT_PARTIALS_DIR, name => documentedProps.push(name));
    try {
      generateTopicPropertyMappings(properties, process.env.OUTPUT_PARTIALS_DIR);
    } catch (err) {
      console.error(`Error: Failed to generate topic-property-mappings.adoc: ${err.message}`);
    }
  } else {
    console.log('📄 Skipping partial generation (set GENERATE_PARTIALS=1 and OUTPUT_PARTIALS_DIR to enable)');
  }

  const errors = generateErrorReports(properties, documentedProps);

  const totalProperties = Object.keys(properties).length;
  const notRendered = errors.undocumented_properties.length;
  const pctRendered = totalProperties
    ? ((partialsCount / totalProperties) * 100).toFixed(2)
    : '0.00';

  console.log('\nSummary:');
  console.log(`   Total properties found:      ${totalProperties}`);
  console.log(`   Property partials generated: ${partialsCount} (${pctRendered}% of total)`);
  console.log(`   Not documented:              ${notRendered}`);
  console.log(`   Deprecated properties:       ${deprecatedCount}`);

  if (notRendered > 0) {
    console.log('Ignored:\n   ' + errors.undocumented_properties.join('\n   '));
  }

  return {
    totalProperties,
    generatedPartials: partialsCount,
    undocumentedProperties: errors.undocumented_properties,
    deprecatedProperties: deprecatedCount,
    percentageRendered: pctRendered
  };
}


module.exports = {
  generateAllDocs,
  generateDeprecatedDocs,
  generatePropertyPartials
};

// CLI
if (require.main === module) {
  const args = process.argv.slice(2);
  if (args.length < 2) {
    console.error('Usage: node generate-handlebars-docs.js <input-file> <output-dir>');
    process.exit(1);
  }

  const [inputFile, outputDir] = args;
  if (!fs.existsSync(inputFile)) {
    console.error(`Error: Input file not found: ${inputFile}`);
    process.exit(1);
  }

  try {
    generateAllDocs(inputFile, outputDir);
    console.log('Done: Documentation generation completed successfully');
  } catch (err) {
    console.error(`Error: ${err.message}`);
    process.exit(1);
  }
}