@redpanda-data/docs-extensions-and-macros/tools/property-extractor/Makefile

Antora extensions and macros developed for Redpanda documentation.

.PHONY: build venv clean redpanda-git treesitter generate-docs check

# Default tag (can be overridden via `make TAG=v25.1.1`)
TAG       ?= dev

# Derive a “major.minor” or rc identifier from TAG for folder naming
VERSION := $(shell \
  if echo "$(TAG)" | grep -qE '^v?[0-9]+\.[0-9]+'; then \
	echo "$(TAG)" \
	  | sed -E 's/^v?([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc[0-9]+)?.*/\1\3/' \
	  | sed 's/-rc/rc/'; \
  else \
	echo "$(TAG)"; \
  fi)

# Paths
REPO_ROOT     := $(shell git rev-parse --show-toplevel)
MODULE_ROOT   := $(shell cd "$(dir $(realpath $(lastword $(MAKEFILE_LIST))))"/../.. && pwd)
TOOL_ROOT     := $(MODULE_ROOT)/tools/property-extractor
TMP_ROOT      := $(TOOL_ROOT)/tmp
REDPANDA_SRC  := $(TMP_ROOT)/redpanda
TREESITTER_DIR:= $(TOOL_ROOT)/tree-sitter/tree-sitter-cpp
VENV          := $(TOOL_ROOT)/tmp/redpanda-property-extractor-venv
PYTHON        := $(VENV)/bin/python
TREE_SITTER   := npx tree-sitter

# Output directory configuration (can be overridden by environment variables)
OUTPUT_AUTOGENERATED_DIR ?= $(REPO_ROOT)/modules/reference
OUTPUT_ASCIIDOC_DIR      ?= $(OUTPUT_AUTOGENERATED_DIR)/pages
# Always save versioned JSON files to attachments directory
OUTPUT_JSON_DIR          ?= $(OUTPUT_AUTOGENERATED_DIR)/attachments
# Diff JSON files are saved to overrides directory (if OVERRIDES is specified)
OUTPUT_DIFF_DIR          ?= $(if $(OVERRIDES),$(or $(dir $(OVERRIDES)),$(OUTPUT_AUTOGENERATED_DIR)/attachments),$(OUTPUT_AUTOGENERATED_DIR)/attachments)
OUTPUT_PARTIALS_DIR      ?= $(OUTPUT_AUTOGENERATED_DIR)/partials

# --- Main build: venv, fetch code, build parser, extract & docgen ---
build: venv redpanda-git treesitter
	@echo "🔧 Building with Redpanda tag: $(TAG)"
	@mkdir -p $(TOOL_ROOT)/gen
	@if [ -n "$(OVERRIDES)" ] && [ ! -f "$(OVERRIDES)" ]; then \
	  echo "❌ Error: Overrides file '$(OVERRIDES)' does not exist."; \
	  echo "   Please check the path or omit the OVERRIDES variable to continue without overrides."; \
	  exit 1; \
	fi
	@cd $(TOOL_ROOT) && \
	  $(PYTHON) -W ignore property_extractor.py \
		--recursive \
		--path $(REDPANDA_SRC) \
		--output gen/properties-output.json \
		--enhanced-output gen/$(TAG)-properties.json \
		$(if $(OVERRIDES),$(if $(shell [ -f "$(OVERRIDES)" ] && echo exists),--overrides $(OVERRIDES),),) \
		$(if $(CLOUD_SUPPORT),--cloud-support,)
	@echo "✅ JSON generated at $(TOOL_ROOT)/gen/properties-output.json"
	@echo "✅ Enhanced JSON (with overrides) generated at $(TOOL_ROOT)/gen/$(TAG)-properties.json"
	@$(MAKE) generate-docs

# --- Ensure Python venv & dependencies ---
venv: $(TOOL_ROOT)/requirements.txt
	@if [ ! -d "$(VENV)" ]; then \
	  echo "🐍 Creating virtual environment in $(VENV)..."; \
	  python3 -m venv $(VENV); \
	else \
	  echo "🐍 Virtual environment already exists at $(VENV)"; \
	fi; \
	echo "🔄 Upgrading pip and installing requirements..."; \
	$(VENV)/bin/pip install --upgrade pip --quiet; \
	$(VENV)/bin/pip install --no-cache-dir -r $< --quiet;

# --- Clean out all generated state ---
clean:
	@echo "🧹 Cleaning up…"
	@rm -rf $(VENV) $(TMP_ROOT) $(REPO_ROOT)/autogenerated

# --- Clone or update Redpanda, checkout TAG or default branch ---
redpanda-git:
	@mkdir -p "$(TMP_ROOT)"
	@echo "🔄 Cloning/updating Redpanda into $(REDPANDA_SRC)…"
	@if [ -d "$(REDPANDA_SRC)" ]; then \
	  git -C "$(REDPANDA_SRC)" fetch --all --tags -q; \
	else \
	  GH_TOKEN=$${REDPANDA_GITHUB_TOKEN:-$${GITHUB_TOKEN:-$${GH_TOKEN}}}; \
	  if [ -n "$$GH_TOKEN" ]; then \
	    echo "🔑 Using authenticated clone (token provided)"; \
	    git clone -q https://$$GH_TOKEN@github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \
	  else \
	    git clone -q https://github.com/redpanda-data/redpanda.git "$(REDPANDA_SRC)"; \
	  fi; \
	fi; \
	if git -C "$(REDPANDA_SRC)" rev-parse --verify -q "origin/$(TAG)" >/dev/null 2>&1; then \
	  echo "🔖 Checking out remote branch 'origin/$(TAG)'"; \
	  git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)" 2>/dev/null || git -C "$(REDPANDA_SRC)" checkout -q -b "$(TAG)" "origin/$(TAG)"; \
	  git -C "$(REDPANDA_SRC)" reset --hard "origin/$(TAG)" -q; \
	elif git -C "$(REDPANDA_SRC)" rev-parse --verify -q "$(TAG)" >/dev/null 2>&1; then \
	  echo "🔖 Checking out '$(TAG)'"; \
	  git -C "$(REDPANDA_SRC)" checkout -q "$(TAG)"; \
	else \
	  DEFAULT_BRANCH=$$(git -C "$(REDPANDA_SRC)" symbolic-ref --short refs/remotes/origin/HEAD); \
	  echo "⚠️ Tag '$(TAG)' not found; falling back to '$${DEFAULT_BRANCH}'"; \
	  git -C "$(REDPANDA_SRC)" checkout -q "$${DEFAULT_BRANCH}"; \
	fi

# --- Clone Tree-sitter grammar & generate parser ---
treesitter:
	@echo "🌲 Ensuring tree-sitter-cpp grammar…"
	@if [ ! -d "$(TREESITTER_DIR)/.git" ]; then \
	  rm -rf "$(TREESITTER_DIR)"; \
	  git clone https://github.com/tree-sitter/tree-sitter-cpp.git "$(TREESITTER_DIR)"; \
	fi
	@echo "🔄 Ensuring tree-sitter-cpp is at compatible version v0.20.5…"
	@cd "$(TREESITTER_DIR)" && \
	  git fetch --tags -q && \
	  git checkout -q v0.20.5
	@echo "🔧 Generating parser in $(TREESITTER_DIR)…"
	@cd "$(TREESITTER_DIR)" && export CFLAGS="-Wno-unused-but-set-variable" && npm install --silent && export CFLAGS="-Wno-unused-but-set-variable" && $(TREE_SITTER) generate

# --- Install Node.js dependencies for Handlebars ---
node-deps:
	@echo "📦 Installing Node.js dependencies…"
	@cd $(TOOL_ROOT) && npm install --silent

# --- Turn the JSON into AsciiDoc pages under the specified output directories ---
generate-docs: node-deps
	@echo "📝 Generating AsciiDoc pages in $(OUTPUT_ASCIIDOC_DIR)…"
	@mkdir -p "$(OUTPUT_ASCIIDOC_DIR)"
	@mkdir -p "$(OUTPUT_JSON_DIR)"
	@# Use the enhanced properties file (with overrides) for documentation generation if it exists
	@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
	  cd $(TOOL_ROOT) && \
	    node generate-handlebars-docs.js "gen/$(TAG)-properties.json" "$(OUTPUT_ASCIIDOC_DIR)"; \
	else \
	  cd $(TOOL_ROOT) && \
	    node generate-handlebars-docs.js "gen/properties-output.json" "$(OUTPUT_ASCIIDOC_DIR)"; \
	fi
	@echo "📄 Copying properties JSON files to $(OUTPUT_JSON_DIR)…"
	@if [ -f "$(TOOL_ROOT)/gen/$(TAG)-properties.json" ]; then \
	  echo "   Source: $(TOOL_ROOT)/gen/$(TAG)-properties.json"; \
	  echo "   Target: $(OUTPUT_JSON_DIR)/redpanda-properties-$(TAG).json"; \
	  cp "$(TOOL_ROOT)/gen/$(TAG)-properties.json" "$(OUTPUT_JSON_DIR)/redpanda-properties-$(TAG).json" && \
	  echo "   ✅ Successfully copied redpanda-properties-$(TAG).json" || \
	  { echo "   ❌ Failed to copy redpanda-properties-$(TAG).json"; exit 1; }; \
	  if [ -f "$(OUTPUT_JSON_DIR)/redpanda-properties-$(TAG).json" ]; then \
	    echo "   📊 Verification: Target file exists and has $$(wc -c < "$(OUTPUT_JSON_DIR)/redpanda-properties-$(TAG).json") bytes"; \
	  else \
	    echo "   ❌ Verification failed: Target file does not exist"; exit 1; \
	  fi; \
	else \
	  echo "   ⚠️  Source file $(TOOL_ROOT)/gen/$(TAG)-properties.json not found, skipping copy"; \
	fi
	@echo "🧹 Cleaning up old versioned JSON files (keeping only 2 most recent)…"
	@cd "$(OUTPUT_JSON_DIR)" && \
	  ls -t redpanda-properties-*.json 2>/dev/null | tail -n +3 | while read file; do \
	    echo "   Removing old file: $$file"; \
	    rm -f "$$file"; \
	  done || true
	@echo "✅ Docs generated at $(OUTPUT_AUTOGENERATED_DIR)"

# --- Compare two property files and generate a diff report ---
# Usage: make compare OLD_TAG=v25.2.1 NEW_TAG=v25.2.9 OVERRIDES=path/to/overrides.json
# This will compare the two versions and save the diff to the overrides directory
compare:
	@if [ -z "$(OLD_TAG)" ] || [ -z "$(NEW_TAG)" ]; then \
	  echo "❌ Error: Both OLD_TAG and NEW_TAG must be specified"; \
	  echo "   Usage: make compare OLD_TAG=v25.2.1 NEW_TAG=v25.2.9 OVERRIDES=path/to/overrides.json"; \
	  exit 1; \
	fi
	@if [ -z "$(OVERRIDES)" ]; then \
	  echo "❌ Error: OVERRIDES must be specified to determine where to save the diff"; \
	  echo "   Usage: make compare OLD_TAG=v25.2.1 NEW_TAG=v25.2.9 OVERRIDES=path/to/overrides.json"; \
	  exit 1; \
	fi
	@if [ ! -f "$(OUTPUT_JSON_DIR)/redpanda-properties-$(OLD_TAG).json" ]; then \
	  echo "❌ Error: Old properties file not found: $(OUTPUT_JSON_DIR)/redpanda-properties-$(OLD_TAG).json"; \
	  exit 1; \
	fi
	@if [ ! -f "$(OUTPUT_JSON_DIR)/redpanda-properties-$(NEW_TAG).json" ]; then \
	  echo "❌ Error: New properties file not found: $(OUTPUT_JSON_DIR)/redpanda-properties-$(NEW_TAG).json"; \
	  exit 1; \
	fi
	@echo "Comparing $(OLD_TAG) → $(NEW_TAG)"
	@echo "   Diff will be saved to: $(OUTPUT_DIFF_DIR)"
	@cd $(TOOL_ROOT) && \
	  node compare-properties.js \
	    "$(OUTPUT_JSON_DIR)/redpanda-properties-$(OLD_TAG).json" \
	    "$(OUTPUT_JSON_DIR)/redpanda-properties-$(NEW_TAG).json" \
	    $(OLD_TAG) \
	    $(NEW_TAG) \
	    "$(OUTPUT_DIFF_DIR)" \
	    "redpanda-property-changes-$(OLD_TAG)-to-$(NEW_TAG).json"
	@echo "✅ Diff saved to $(OUTPUT_DIFF_DIR)/redpanda-property-changes-$(OLD_TAG)-to-$(NEW_TAG).json"
	@$(MAKE) cleanup-diffs OVERRIDES="$(OVERRIDES)"

# --- Helper target to cleanup old diff files ---
cleanup-diffs:
	@echo "🧹 Cleaning up old diff JSON files (keeping only 2 most recent)…"
	@cd "$(OUTPUT_DIFF_DIR)" && \
	  ls -t redpanda-property-changes-*.json 2>/dev/null | tail -n +3 | while read file; do \
	    echo "   Removing old file: $$file"; \
	    rm -f "$$file"; \
	  done || true

# --- Debug helper to print all the key paths/vars ---
check:
	@echo "MODULE_ROOT:              $(MODULE_ROOT)"
	@echo "TOOL_ROOT:                $(TOOL_ROOT)"
	@echo "REDPANDA_SRC:             $(REDPANDA_SRC)"
	@echo "TREESITTER:               $(TREESITTER_DIR)"
	@echo "VENV:                     $(VENV)"
	@echo "PYTHON:                   $(PYTHON)"
	@echo "OVERRIDES:                $(OVERRIDES)"
	@echo "OUTPUT_ASCIIDOC_DIR:      $(OUTPUT_ASCIIDOC_DIR)"
	@echo "OUTPUT_JSON_DIR:          $(OUTPUT_JSON_DIR)"
	@echo "OUTPUT_DIFF_DIR:          $(OUTPUT_DIFF_DIR)"
	@echo "OUTPUT_AUTOGENERATED_DIR: $(OUTPUT_AUTOGENERATED_DIR)"