@redpanda-data/docs-extensions-and-macros/tools/redpanda-connect/rpcn-connector-docs-handler.js

Antora extensions and macros developed for Redpanda documentation.

'use strict'

const { spawnSync } = require('child_process')
const path = require('path')
const fs = require('fs')
const { findRepoRoot } = require('../../cli-utils/doc-tools-utils')
const { getAntoraValue, setAntoraValue } = require('../../cli-utils/antora-utils')
const fetchFromGithub = require('../fetch-from-github.js')
const { generateRpcnConnectorDocs } = require('./generate-rpcn-connector-docs.js')
const { getRpkConnectVersion, printDeltaReport } = require('./report-delta')
const { discoverIntermediateReleases, findCloudVersionForDate } = require('./github-release-utils')
const { analyzeAllBinaries } = require('./connector-binary-analyzer.js')
const parseCSVConnectors = require('./parse-csv-connectors.js')
const semver = require('semver')

/**
 * Cap description to two sentences
 * @param {string} description - Full description text
 * @returns {string} Description capped to two sentences
 */
function capToTwoSentences (description) {
  if (!description) return ''

  const hasProblematicContent = (text) => {
    return /```[\s\S]*?```/.test(text) ||
           /`[^`]+`/.test(text) ||
           /^[=#]+\s+.+$/m.test(text) ||
           /\n/.test(text)
  }

  const abbreviations = [
    /https?:\/\/[^\s]+?(?=[.!?](?:\s|$)|\s|$)/gi,  // Protect URLs from being split by sentence detection (non-greedy, preserve trailing punctuation)
    /\bv\d+\.\d+(?:\.\d+)?/gi,
    /\d+\.\d+/g,
    /\be\.g\./gi,
    /\bi\.e\./gi,
    /\betc\./gi,
    /\bvs\./gi,
    /\bDr\./gi,
    /\bMr\./gi,
    /\bMs\./gi,
    /\bMrs\./gi,
    /\bSt\./gi,
    /\bNo\./gi
  ]

  let normalized = description
  const placeholders = []

  abbreviations.forEach((abbrevRegex, idx) => {
    normalized = normalized.replace(abbrevRegex, (match) => {
      const placeholder = `__ABBREV${idx}_${placeholders.length}__`
      placeholders.push({ placeholder, original: match })
      return placeholder
    })
  })

  normalized = normalized.replace(/\.{3,}/g, (match) => {
    const placeholder = `__ELLIPSIS_${placeholders.length}__`
    placeholders.push({ placeholder, original: match })
    return placeholder
  })

  const sentenceRegex = /[^.!?]+[.!?]+(?:\s|$)/g
  const sentences = normalized.match(sentenceRegex)

  if (!sentences || sentences.length === 0) {
    let result = normalized
    placeholders.forEach(({ placeholder, original }) => {
      result = result.replace(placeholder, original)
    })
    return result
  }

  let maxSentences = 2

  if (sentences.length >= 2) {
    let secondSentence = sentences[1]
    placeholders.forEach(({ placeholder, original }) => {
      secondSentence = secondSentence.replace(new RegExp(placeholder, 'g'), original)
    })

    if (hasProblematicContent(secondSentence)) {
      maxSentences = 1
    }
  }

  let result = sentences.slice(0, maxSentences).join('')

  placeholders.forEach(({ placeholder, original }) => {
    result = result.replace(new RegExp(placeholder, 'g'), original)
  })

  return result.trim()
}

/**
 * Remove platform metadata fields from connectors
 * @param {Array} connectors - Array of connector objects
 */
function stripPlatformMetadata (connectors) {
  if (!Array.isArray(connectors)) return
  connectors.forEach(c => {
    delete c.cloudSupported
    delete c.requiresCgo
    delete c.cloudOnly
  })
}

/**
 * Augment connector data with platform metadata from binary analysis.
 * This adds cloudSupported/requiresCgo flags and includes CGO-only and cloud-only connectors.
 *
 * @param {Object} connectorData - Connector index object with arrays for each component type
 * @param {Object} binaryAnalysis - Binary analysis results from analyzeAllBinaries()
 * @returns {Object} Object with augmentedData, augmentedCount, addedCgoCount, addedCloudOnlyCount
 */
function augmentConnectorData (connectorData, binaryAnalysis) {
  if (!binaryAnalysis) {
    return { augmentedData: connectorData, augmentedCount: 0, addedCgoCount: 0, addedCloudOnlyCount: 0 }
  }

  // Deep clone to avoid mutating original
  const augmentedData = JSON.parse(JSON.stringify(connectorData))

  const cloudSet = new Set(
    (binaryAnalysis.comparison?.inCloud || []).map(c => `${c.type}:${c.name}`)
  )
  const cgoOnlySet = new Set(
    (binaryAnalysis.cgoOnly || []).map(c => `${c.type}:${c.name}`)
  )

  let augmentedCount = 0
  let addedCgoCount = 0
  let addedCloudOnlyCount = 0

  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
    'buffers', 'metrics', 'scanners', 'tracers']

  for (const type of connectorTypes) {
    if (!Array.isArray(augmentedData[type])) {
      augmentedData[type] = []
    }

    // Add cloudSupported and requiresCgo flags to existing connectors
    for (const connector of augmentedData[type]) {
      const key = `${type}:${connector.name}`
      connector.cloudSupported = cloudSet.has(key)
      connector.requiresCgo = cgoOnlySet.has(key)
      augmentedCount++
    }

    // Add CGO-only connectors that aren't in the OSS list
    if (binaryAnalysis.cgoOnly) {
      for (const cgoConn of binaryAnalysis.cgoOnly) {
        if (cgoConn.type === type) {
          const exists = augmentedData[type].some(c => c.name === cgoConn.name)
          if (!exists) {
            // Singularize type for consistency with stored data (except "metrics" which stays plural)
            const componentType = cgoConn.type === 'metrics' ? 'metrics' : cgoConn.type.replace(/s$/, '')
            augmentedData[type].push({
              ...cgoConn,
              type: componentType,
              cloudSupported: false,
              requiresCgo: true
            })
            addedCgoCount++
          }
        }
      }
    }

    // Add cloud-only connectors that aren't in the OSS list
    if (binaryAnalysis.comparison?.cloudOnly) {
      for (const cloudConn of binaryAnalysis.comparison.cloudOnly) {
        if (cloudConn.type === type) {
          const exists = augmentedData[type].some(c => c.name === cloudConn.name)
          if (!exists) {
            // Singularize type for consistency with stored data (except "metrics" which stays plural)
            const componentType = cloudConn.type === 'metrics' ? 'metrics' : cloudConn.type.replace(/s$/, '')
            augmentedData[type].push({
              ...cloudConn,
              type: componentType,
              cloudSupported: true,
              requiresCgo: false,
              cloudOnly: true
            })
            addedCloudOnlyCount++
          }
        }
      }
    }
  }

  return { augmentedData, augmentedCount, addedCgoCount, addedCloudOnlyCount }
}

/**
 * Update whats-new.adoc with new release information
 * @param {Object} params - Parameters
 * @param {string} params.dataDir - Data directory path
 * @param {string} params.oldVersion - Old version string
 * @param {string} params.newVersion - New version string
 * @param {Object} params.binaryAnalysis - Binary analysis data
 */
function updateWhatsNew ({ dataDir, oldVersion, newVersion, binaryAnalysis }) {
  try {
    const whatsNewPath = path.join(findRepoRoot(), 'modules/get-started/pages/whats-new.adoc')
    if (!fs.existsSync(whatsNewPath)) {
      console.error(`Error: Unable to update release notes: 'whats-new.adoc' was not found at: ${whatsNewPath}`)
      return
    }

    const diffPath = path.join(dataDir, `connect-diff-${oldVersion}_to_${newVersion}.json`)
    if (!fs.existsSync(diffPath)) {
      console.error(`Error: Unable to update release notes: The connector diff JSON was not found at: ${diffPath}`)
      return
    }

    let diff
    try {
      diff = JSON.parse(fs.readFileSync(diffPath, 'utf8'))
    } catch (jsonErr) {
      console.error(`Error: Unable to parse connector diff JSON at ${diffPath}: ${jsonErr.message}`)
      return
    }

    let whatsNew
    try {
      whatsNew = fs.readFileSync(whatsNewPath, 'utf8')
    } catch (readErr) {
      console.error(`Error: Unable to read whats-new.adoc at ${whatsNewPath}: ${readErr.message}`)
      return
    }

    const versionRe = new RegExp(`^== Version ${diff.comparison.newVersion.replace(/[-.]/g, '\\$&')}(?:\\r?\\n|$)`, 'm')
    const match = versionRe.exec(whatsNew)
    let startIdx = match ? match.index : -1
    let endIdx = -1
    if (startIdx !== -1) {
      const rest = whatsNew.slice(startIdx + 1)
      const nextMatch = /^== Version /m.exec(rest)
      endIdx = nextMatch ? startIdx + 1 + nextMatch.index : whatsNew.length
    }

    let releaseNotesLink = ''
    if (diff.comparison && diff.comparison.newVersion) {
      releaseNotesLink = `link:https://github.com/redpanda-data/connect/releases/tag/v${diff.comparison.newVersion}[See the full release notes^].\n\n`
    }
    let section = `\n== Version ${diff.comparison.newVersion}\n\n${releaseNotesLink}`

    // Separate Bloblang and regular components
    const bloblangComponents = []
    const regularComponents = []

    if (diff.details.newComponents && diff.details.newComponents.length) {
      // Filter out cloud-only connectors - they don't go in whats-new.adoc
      const nonCloudOnlyComponents = diff.details.newComponents.filter(comp => {
        const isCloudOnly = diff.binaryAnalysis?.details?.cloudOnly?.some(cloudComp => {
          return cloudComp.name === comp.name && cloudComp.type === comp.type
        })
        return !isCloudOnly
      })

      for (const comp of nonCloudOnlyComponents) {
        if (comp.type === 'bloblang-functions' || comp.type === 'bloblang-methods') {
          bloblangComponents.push(comp)
        } else {
          const isCgoOnly = diff.binaryAnalysis?.details?.cgoOnly?.some(cgo => {
            return cgo.name === comp.name && cgo.type === comp.type
          })

          regularComponents.push({
            ...comp,
            requiresCgo: isCgoOnly
          })
        }
      }
    }

    // Bloblang updates section
    if (bloblangComponents.length > 0) {
      section += '=== Bloblang updates\n\n'
      section += 'This release adds the following new Bloblang capabilities:\n\n'

      const byType = {}
      for (const comp of bloblangComponents) {
        if (!byType[comp.type]) byType[comp.type] = []
        byType[comp.type].push(comp)
      }

      for (const [type, comps] of Object.entries(byType)) {
        if (type === 'bloblang-functions') {
          section += '* Functions:\n'
          for (const comp of comps) {
            section += `** xref:guides:bloblang/functions.adoc#${comp.name}[\`${comp.name}\`]`
            if (comp.status && comp.status !== 'stable') section += ` (${comp.status})`
            const desc = comp.summary || comp.description
            if (desc) {
              section += `: ${capToTwoSentences(desc)}`
            } else {
              section += `\n+\n// TODO: Add description for ${comp.name} function`
            }
            section += '\n'
          }
        } else if (type === 'bloblang-methods') {
          section += '* Methods:\n'
          for (const comp of comps) {
            section += `** xref:guides:bloblang/methods.adoc#${comp.name}[\`${comp.name}\`]`
            if (comp.status && comp.status !== 'stable') section += ` (${comp.status})`
            const desc = comp.summary || comp.description
            if (desc) {
              section += `: ${capToTwoSentences(desc)}`
            } else {
              section += `\n+\n// TODO: Add description for ${comp.name} method`
            }
            section += '\n'
          }
        }
      }
      section += '\n'
    }

    // Component updates section
    if (regularComponents.length > 0) {
      section += '=== Component updates\n\n'
      section += 'This release adds the following new components:\n\n'

      section += '[cols="1m,1a,1a,3a"]\n'
      section += '|===\n'
      section += '|Component |Type |Status |Description\n\n'

      for (const comp of regularComponents) {
        const typeLabel = comp.type.charAt(0).toUpperCase() + comp.type.slice(1)
        const statusLabel = comp.status || '-'
        let desc = comp.summary || (comp.description ? capToTwoSentences(comp.description) : '// TODO: Add description')

        if (comp.requiresCgo) {
          const cgoNote = '\nNOTE: Requires a cgo-enabled binary. See the xref:install:index.adoc[installation guides] for details.'
          desc = desc.startsWith('// TODO') ? cgoNote : `${desc}\n\n${cgoNote}`
        }

        const typePlural = comp.type.endsWith('s') ? comp.type : `${comp.type}s`
        section += `|xref:components:${typePlural}/${comp.name}.adoc[${comp.name}]\n`
        section += `|${typeLabel}\n`
        section += `|${statusLabel}\n`
        section += `|${desc}\n\n`
      }

      section += '|===\n\n'
    }

    // New fields section
    if (diff.details.newFields && diff.details.newFields.length) {
      const regularFields = diff.details.newFields.filter(field => {
        const [type] = field.component.split(':')
        return type !== 'bloblang-functions' && type !== 'bloblang-methods'
      })

      if (regularFields.length > 0) {
        section += '\n=== New field support\n\n'
        section += 'This release adds support for the following new fields:\n\n'
        section += buildFieldsTable(regularFields, capToTwoSentences)
      }
    }

    // Deprecated components section
    if (diff.details.deprecatedComponents && diff.details.deprecatedComponents.length) {
      section += '\n=== Deprecations\n\n'
      section += 'The following components are now deprecated:\n\n'

      section += '[cols="1m,1a,3a"]\n'
      section += '|===\n'
      section += '|Component |Type |Description\n\n'

      for (const comp of diff.details.deprecatedComponents) {
        const typeLabel = comp.type.charAt(0).toUpperCase() + comp.type.slice(1)
        const descText = comp.summary || comp.description
        const desc = descText ? capToTwoSentences(descText) : '-'

        if (comp.type === 'bloblang-functions') {
          section += `|xref:guides:bloblang/functions.adoc#${comp.name}[${comp.name}]\n`
        } else if (comp.type === 'bloblang-methods') {
          section += `|xref:guides:bloblang/methods.adoc#${comp.name}[${comp.name}]\n`
        } else {
          section += `|xref:components:${comp.type}/${comp.name}.adoc[${comp.name}]\n`
        }
        section += `|${typeLabel}\n`
        section += `|${desc}\n\n`
      }

      section += '|===\n\n'
    }

    // Deprecated fields section
    if (diff.details.deprecatedFields && diff.details.deprecatedFields.length) {
      const regularDeprecatedFields = diff.details.deprecatedFields.filter(field => {
        const [type] = field.component.split(':')
        return type !== 'bloblang-functions' && type !== 'bloblang-methods'
      })

      if (regularDeprecatedFields.length > 0) {
        if (!diff.details.deprecatedComponents || diff.details.deprecatedComponents.length === 0) {
          section += '\n=== Deprecations\n\n'
        } else {
          section += '\n'
        }
        section += 'The following fields are now deprecated:\n\n'
        section += buildFieldsTable(regularDeprecatedFields, capToTwoSentences)
      }
    }

    // Changed defaults section
    if (diff.details.changedDefaults && diff.details.changedDefaults.length) {
      const regularChangedDefaults = diff.details.changedDefaults.filter(change => {
        const [type] = change.component.split(':')
        return type !== 'bloblang-functions' && type !== 'bloblang-methods'
      })

      if (regularChangedDefaults.length > 0) {
        section += '\n=== Default value changes\n\n'
        section += 'This release includes the following default value changes:\n\n'
        section += buildChangedDefaultsTable(regularChangedDefaults, capToTwoSentences)
      }
    }

    // Update the file
    let contentWithoutOldSection = whatsNew
    if (startIdx !== -1) {
      contentWithoutOldSection = whatsNew.slice(0, startIdx) + whatsNew.slice(endIdx)
    }

    const versionHeading = /^== Version /m
    const firstMatch = versionHeading.exec(contentWithoutOldSection)
    const insertIdx = firstMatch ? firstMatch.index : contentWithoutOldSection.length

    const updated = contentWithoutOldSection.slice(0, insertIdx) + section + '\n' + contentWithoutOldSection.slice(insertIdx)

    if (startIdx !== -1) {
      console.log(`♻️  whats-new.adoc: replaced section for Version ${diff.comparison.newVersion}`)
    } else {
      console.log(`Done: whats-new.adoc updated with Version ${diff.comparison.newVersion}`)
    }

    fs.writeFileSync(whatsNewPath, updated, 'utf8')
  } catch (err) {
    console.error(`Error: Failed to update whats-new.adoc: ${err.message}`)
  }
}

/**
 * Build a fields table for whats-new.adoc
 * @param {Array} fields - Field data
 * @param {Function} capFn - Caption function
 * @returns {string} AsciiDoc table
 */
function buildFieldsTable (fields, capFn) {
  const byField = {}
  for (const field of fields) {
    const [type, compName] = field.component.split(':')
    if (!byField[field.field]) {
      byField[field.field] = {
        description: field.description,
        components: []
      }
    }
    byField[field.field].components.push({ type, name: compName })
  }

  let section = '[cols="1m,3,2a"]\n'
  section += '|===\n'
  section += '|Field |Description |Affected components\n\n'

  for (const [fieldName, info] of Object.entries(byField)) {
    const byType = {}
    for (const comp of info.components) {
      if (!byType[comp.type]) byType[comp.type] = []
      byType[comp.type].push(comp.name)
    }

    let componentList = ''
    for (const [type, names] of Object.entries(byType)) {
      if (componentList) componentList += '\n\n'

      const typeLabel = names.length === 1
        ? type.charAt(0).toUpperCase() + type.slice(1)
        : type.charAt(0).toUpperCase() + type.slice(1) + (type.endsWith('s') ? '' : 's')

      componentList += `*${typeLabel}:*\n\n`
      names.forEach(name => {
        componentList += `* xref:components:${type}/${name}.adoc#${fieldName}[${name}]\n`
      })
    }

    const desc = info.description ? capFn(info.description) : '// TODO: Add description'

    section += `|${fieldName}\n`
    section += `|${desc}\n`
    section += `|${componentList}\n\n`
  }

  section += '|===\n\n'
  return section
}

/**
 * Build changed defaults table for whats-new.adoc
 * @param {Array} changedDefaults - Changed defaults data
 * @param {Function} capFn - Caption function
 * @returns {string} AsciiDoc table
 */
function buildChangedDefaultsTable (changedDefaults, capFn) {
  const byFieldAndDefaults = {}
  for (const change of changedDefaults) {
    const [type, compName] = change.component.split(':')
    const compositeKey = `${change.field}|${String(change.oldDefault)}|${String(change.newDefault)}`
    if (!byFieldAndDefaults[compositeKey]) {
      byFieldAndDefaults[compositeKey] = {
        field: change.field,
        oldDefault: change.oldDefault,
        newDefault: change.newDefault,
        description: change.description,
        components: []
      }
    }
    byFieldAndDefaults[compositeKey].components.push({ type, name: compName })
  }

  let section = '[cols="1m,1,1,3,2a"]\n'
  section += '|===\n'
  section += '|Field |Old default |New default |Description |Affected components\n\n'

  for (const [, info] of Object.entries(byFieldAndDefaults)) {
    const formatDefault = (val) => {
      if (val === undefined || val === null) return 'none'
      if (typeof val === 'string') return val
      if (typeof val === 'number' || typeof val === 'boolean') return String(val)
      return JSON.stringify(val)
    }

    const oldVal = formatDefault(info.oldDefault)
    const newVal = formatDefault(info.newDefault)
    const desc = info.description ? capFn(info.description) : '// TODO: Add description'

    const byType = {}
    for (const comp of info.components) {
      if (!byType[comp.type]) byType[comp.type] = []
      byType[comp.type].push(comp.name)
    }

    let componentList = ''
    for (const [type, names] of Object.entries(byType)) {
      if (componentList) componentList += '\n\n'

      const typeLabel = names.length === 1
        ? type.charAt(0).toUpperCase() + type.slice(1)
        : type.charAt(0).toUpperCase() + type.slice(1) + (type.endsWith('s') ? '' : 's')

      componentList += `*${typeLabel}:*\n\n`
      names.forEach(name => {
        componentList += `* xref:components:${type}/${name}.adoc#${info.field}[${name}]\n`
      })
    }

    section += `|${info.field}\n`
    section += `|${oldVal}\n`
    section += `|${newVal}\n`
    section += `|${desc}\n`
    section += `|${componentList}\n\n`
  }

  section += '|===\n\n'
  return section
}

/**
 * Log a collapsed list of files
 * @param {string} label - Label for the list
 * @param {Array} filesArray - Array of file paths
 * @param {number} maxToShow - Maximum items to show
 */
function logCollapsed (label, filesArray, maxToShow = 10) {
  console.log(`  • ${label}: ${filesArray.length} total`)
  const sample = filesArray.slice(0, maxToShow)
  sample.forEach(fp => console.log(`    – ${fp}`))
  const remaining = filesArray.length - sample.length
  if (remaining > 0) {
    console.log(`    … plus ${remaining} more`)
  }
  console.log('')
}


/**
 * Load or fetch connector data for a specific version
 * @param {string} version - Version to load (e.g., "4.50.0")
 * @param {string} dataDir - Directory where JSON files are stored
 * @param {Object} options - Options for fetching if needed
 * @returns {Promise<Object>} Parsed connector data
 */
async function loadConnectorDataForVersion(version, dataDir, options = {}) {
  const dataFile = path.join(dataDir, `connect-${version}.json`);

  // If forceFresh is set, always fetch even if file exists
  // This ensures we have accurate connector lists for diff comparison
  if (!options.forceFresh && fs.existsSync(dataFile)) {
    console.log(`✓ Using existing data file: connect-${version}.json`);
    const data = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
    return data;
  }

  // Fetch fresh data
  const reason = options.forceFresh ? 'force refresh requested' : 'file not found';
  console.log(`📥 Fetching data for ${version} (${reason})...`);

  try {
    // Try installing that specific version and fetching data
    console.log(`   Installing Redpanda Connect version ${version}...`);
    const installResult = spawnSync('rpk', ['connect', 'install', '--connect-version', version, '--force'], {
      stdio: 'pipe'
    });

    if (installResult.status !== 0) {
      throw new Error(`Failed to install Connect version ${version}`);
    }

    // Fetch connector list
    const tmpFile = path.join(dataDir, `connect-${version}.tmp.json`);
    const fd = fs.openSync(tmpFile, 'w');
    const listResult = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], {
      stdio: ['ignore', fd, 'pipe']
    });
    fs.closeSync(fd);

    if (listResult.status !== 0) {
      throw new Error(`Failed to fetch connector list for version ${version}`);
    }

    // Parse and validate
    const rawJson = fs.readFileSync(tmpFile, 'utf8');
    const parsed = JSON.parse(rawJson);

    // Move to final location
    fs.renameSync(tmpFile, dataFile);

    console.log(`✓ Successfully fetched data for version ${version}`);
    return parsed;
  } catch (error) {
    console.error(`❌ Failed to fetch data for version ${version}: ${error.message}`);
    throw new Error(`Cannot process version ${version} - data unavailable`);
  }
}

/**
 * Main handler for rpcn-connector-docs command
 * @param {Object} options - Command options
 */
async function handleRpcnConnectorDocs (options) {
  const dataDir = path.resolve(process.cwd(), options.dataDir)
  fs.mkdirSync(dataDir, { recursive: true })

  const timestamp = new Date().toISOString()

  let newVersion
  let dataFile
  let binaryAnalysis = null
  let draftsWritten = 0
  let draftFiles = []
  let needsAugmentation = false
  let csvMetadata = []

  if (options.fetchConnectors) {
    try {
      if (options.connectVersion) {
        if (!semver.valid(options.connectVersion)) {
          console.error(`Error: Invalid --connect-version format: ${options.connectVersion}`)
          console.error('Expected format: X.Y.Z (e.g., 4.50.0)')
          process.exit(1)
        }
        console.log(`Installing Redpanda Connect version ${options.connectVersion}...`)
        const installResult = spawnSync('rpk', ['connect', 'install', '--connect-version', options.connectVersion, '--force'], {
          stdio: 'inherit'
        })
        if (installResult.status !== 0) {
          throw new Error(`Failed to install Connect version ${options.connectVersion}`)
        }
        console.log(`Done: Installed Redpanda Connect version ${options.connectVersion}`)
        newVersion = options.connectVersion
      } else {
        newVersion = getRpkConnectVersion()
      }
      console.log(`Fetching connector data from Connect ${newVersion}...`)

      const tmpFile = path.join(dataDir, `connect-${newVersion}.tmp.json`)
      const finalFile = path.join(dataDir, `connect-${newVersion}.json`)

      const fd = fs.openSync(tmpFile, 'w')
      const r = spawnSync('rpk', ['connect', 'list', '--format', 'json-full'], { stdio: ['ignore', fd, 'inherit'] })
      fs.closeSync(fd)

      const rawJson = fs.readFileSync(tmpFile, 'utf8')
      const parsed = JSON.parse(rawJson)
      fs.writeFileSync(finalFile, JSON.stringify(parsed, null, 2))
      fs.unlinkSync(tmpFile)
      dataFile = finalFile
      needsAugmentation = true
      console.log(`Done: Fetched connector data for version ${newVersion}`)

      // Fetch info.csv
      try {
        console.log(`Fetching info.csv for Connect v${newVersion}...`)
        const csvFile = path.join(dataDir, `connect-info-${newVersion}.csv`)

        if (!fs.existsSync(csvFile)) {
          await fetchFromGithub(
            'redpanda-data',
            'connect',
            'internal/plugins/info.csv',
            dataDir,
            `connect-info-${newVersion}.csv`,
            `v${newVersion}`
          )
          console.log(`Done: Fetched info.csv for version ${newVersion}`)
        } else {
          console.log(`✓ CSV already exists: connect-info-${newVersion}.csv`)
        }
      } catch (csvErr) {
        console.warn(`Warning: Failed to fetch info.csv: ${csvErr.message}`)
      }

      // Parse CSV metadata
      try {
        const csvFile = path.join(dataDir, `connect-info-${newVersion}.csv`)
        if (fs.existsSync(csvFile)) {
          csvMetadata = await parseCSVConnectors(csvFile, console)
        }
      } catch (csvParseErr) {
        console.warn(`Warning: Failed to parse info.csv: ${csvParseErr.message}`)
      }

      // Fetch Bloblang examples
      try {
        console.log(`Fetching Bloblang playground examples for Connect v${newVersion}...`)
        const examplesFile = path.join(dataDir, `bloblang-samples-${newVersion}.json`)

        if (!fs.existsSync(examplesFile)) {
          const tempExamplesDir = path.join(dataDir, `temp-playground-${newVersion}`)
          await fetchFromGithub(
            'redpanda-data',
            'connect',
            'docs/guides/bloblang/playground',
            tempExamplesDir,
            null,
            `v${newVersion}`
          )

          const yaml = require('js-yaml')
          const bloblangSamples = {}
          const files = fs.readdirSync(tempExamplesDir).filter(f => f.endsWith('.yaml') || f.endsWith('.yml'))

          for (const file of files) {
            try {
              const content = fs.readFileSync(path.join(tempExamplesDir, file), 'utf8')
              const parsedYaml = yaml.load(content)
              if (parsedYaml.title && parsedYaml.input && parsedYaml.mapping) {
                bloblangSamples[file] = parsedYaml
              }
            } catch (err) {
              console.warn(`Warning: Failed to parse ${file}: ${err.message}`)
            }
          }

          fs.writeFileSync(examplesFile, JSON.stringify(bloblangSamples, null, 2))
          fs.rmSync(tempExamplesDir, { recursive: true, force: true })
          console.log(`Done: Fetched ${Object.keys(bloblangSamples).length} Bloblang examples`)
        } else {
          console.log(`✓ Bloblang samples already exist: bloblang-samples-${newVersion}.json`)
        }
      } catch (examplesErr) {
        console.warn(`Warning: Failed to fetch Bloblang examples: ${examplesErr.message}`)
      }
    } catch (err) {
      console.error(`Error: Failed to fetch connectors: ${err.message}`)
      process.exit(1)
    }
  } else {
    const candidates = fs.readdirSync(dataDir)
      .filter(f => /^connect-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.json$/.test(f))
      .map(f => {
        const match = f.match(/^connect-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.json$/)
        return match ? match[1] : null
      })
      .filter(v => v && semver.valid(v))

    if (candidates.length === 0) {
      console.error('Error: No valid connect-<version>.json found. Use --fetch-connectors.')
      process.exit(1)
    }

    const sortedVersions = semver.rsort(candidates)
    newVersion = sortedVersions[0]
    dataFile = path.join(dataDir, `connect-${newVersion}.json`)
  }

  // ========================================================================
  // Multi-Release Processing: Discover and process intermediate releases
  // ========================================================================

  const processIntermediate = !options.skipIntermediate && !options.oldData
  let versionsToProcess = []
  let intermediateProcessingResults = []

  if (processIntermediate) {
    // Determine starting version
    let startVersion = options.fromVersion

    if (startVersion && !semver.valid(startVersion)) {
      console.error(`Error: Invalid --from-version format: ${startVersion}`)
      console.error('Expected format: X.Y.Z (e.g., 4.50.0)')
      process.exit(1)
    }

    if (!startVersion) {
      // Try antora.yml first
      startVersion = getAntoraValue('asciidoc.attributes.latest-connect-version')

      // Fallback: check existing data files
      if (!startVersion) {
        const existingVersions = fs.readdirSync(dataDir)
          .filter(f => /^connect-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.json$/.test(f))
          .filter(f => f !== path.basename(dataFile))
          .map(f => {
            const match = f.match(/^connect-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.json$/)
            return match ? match[1] : null
          })
          .filter(v => v && semver.valid(v))

        if (existingVersions.length > 0) {
          const sortedVersions = semver.rsort(existingVersions)
          startVersion = sortedVersions[0]
        }
      }
    }

    if (startVersion && startVersion !== newVersion) {
      console.log(`\n${'='.repeat(80)}`)
      console.log(`🔍 Checking for intermediate releases between ${startVersion} and ${newVersion}...`)
      console.log('='.repeat(80))

      try {
        const intermediateReleases = await discoverIntermediateReleases(
          startVersion,
          newVersion,
          { includePrerelease: false, useCache: true }
        )

        versionsToProcess = intermediateReleases.map(r => r.version)

        // Process all version pairs EXCEPT the last one (which will be handled by the main flow)
        if (versionsToProcess.length > 2) {
          console.log(`\n📦 Processing ${versionsToProcess.length - 2} intermediate release(s)...\n`)

          for (let i = 0; i < versionsToProcess.length - 2; i++) {
            const fromVer = versionsToProcess[i]
            const toVer = versionsToProcess[i + 1]

            console.log(`\n${'─'.repeat(80)}`)
            console.log(`📋 Processing intermediate release: ${fromVer} → ${toVer}`)
            console.log('─'.repeat(80) + '\n')

            try {
              // Load data for both versions - FORCE FRESH to avoid stale/incomplete data
              // This ensures we have accurate connector lists for both versions
              console.log(`Loading connector data for ${fromVer} (fresh fetch)...`)
              const oldData = await loadConnectorDataForVersion(fromVer, dataDir, { forceFresh: true })

              console.log(`Loading connector data for ${toVer} (fresh fetch)...`)
              const newData = await loadConnectorDataForVersion(toVer, dataDir, { forceFresh: true })

              const analysisOptions = {
                skipCloud: false,
                skipCgo: false,
                cgoVersion: options.cgoVersion || null
              }

              // Determine cloud version for OLD release date
              const oldReleaseInfo = intermediateReleases.find(r => r.version === fromVer)
              let cloudVersionForOldRelease = options.cloudVersion || null

              if (!options.cloudVersion && oldReleaseInfo && oldReleaseInfo.date) {
                cloudVersionForOldRelease = await findCloudVersionForDate(oldReleaseInfo.date, { useCache: true })
                if (cloudVersionForOldRelease) {
                  console.log(`   Using cloud version ${cloudVersionForOldRelease} for old release ${fromVer}`)
                } else {
                  cloudVersionForOldRelease = fromVer
                }
              }

              // Determine cloud version for NEW release date
              const newReleaseInfo = intermediateReleases.find(r => r.version === toVer)
              let cloudVersionForRelease = options.cloudVersion || null

              if (!options.cloudVersion && newReleaseInfo && newReleaseInfo.date) {
                cloudVersionForRelease = await findCloudVersionForDate(newReleaseInfo.date, { useCache: true })
                if (cloudVersionForRelease) {
                  console.log(`   Using cloud version ${cloudVersionForRelease} for new release ${toVer}`)
                } else {
                  cloudVersionForRelease = toVer
                }
              }

              // Run binary analysis for BOTH versions to ensure symmetric comparison
              console.log(`\nAnalyzing binaries for OLD version ${fromVer}...`)
              const oldAnalysis = await analyzeAllBinaries(
                fromVer,
                cloudVersionForOldRelease,
                dataDir,
                analysisOptions
              )

              console.log(`Done: Binary analysis for ${fromVer}:`)
              console.log(`   - OSS version: ${oldAnalysis.ossVersion}`)
              if (oldAnalysis.cgoOnly && oldAnalysis.cgoOnly.length > 0) {
                console.log(`   - CGO-only connectors: ${oldAnalysis.cgoOnly.length}`)
              }

              console.log(`\nAnalyzing binaries for NEW version ${toVer}...`)
              const intermediateAnalysis = await analyzeAllBinaries(
                toVer,
                cloudVersionForRelease,
                dataDir,
                analysisOptions
              )

              console.log(`Done: Binary analysis for ${toVer}:`)
              console.log(`   - OSS version: ${intermediateAnalysis.ossVersion}`)
              if (intermediateAnalysis.cloudVersion) {
                console.log(`   - Cloud version: ${intermediateAnalysis.cloudVersion}`)
              }
              if (intermediateAnalysis.comparison) {
                console.log(`   - Connectors in cloud: ${intermediateAnalysis.comparison.inCloud.length}`)
                console.log(`   - Self-hosted only: ${intermediateAnalysis.comparison.notInCloud.length}`)
                if (intermediateAnalysis.comparison.cloudOnly) {
                  console.log(`   - Cloud-only: ${intermediateAnalysis.comparison.cloudOnly.length}`)
                }
              }
              if (intermediateAnalysis.cgoOnly && intermediateAnalysis.cgoOnly.length > 0) {
                console.log(`   - CGO-only connectors: ${intermediateAnalysis.cgoOnly.length}`)
              }

              // Augment BOTH oldData and newData with their respective binary analysis
              // This ensures symmetric comparison - both have CGO/cloud connectors and metadata
              const { augmentedData: augmentedOldData, addedCgoCount: oldCgoCount, addedCloudOnlyCount: oldCloudOnlyCount } =
                augmentConnectorData(oldData, oldAnalysis)

              const { augmentedData: augmentedNewData, addedCgoCount: newCgoCount, addedCloudOnlyCount: newCloudOnlyCount } =
                augmentConnectorData(newData, intermediateAnalysis)

              console.log(`   - Augmented oldData: +${oldCgoCount} CGO-only, +${oldCloudOnlyCount} cloud-only`)
              console.log(`   - Augmented newData: +${newCgoCount} CGO-only, +${newCloudOnlyCount} cloud-only`)

              // Generate diff with BOTH augmented (symmetric comparison)
              console.log(`\nGenerating diff: ${fromVer} -> ${toVer}...`)
              const { generateConnectorDiffJson } = require('./report-delta.js')

              const diffData = generateConnectorDiffJson(
                augmentedOldData,
                augmentedNewData,
                {
                  oldVersion: fromVer,
                  newVersion: toVer,
                  timestamp,
                  binaryAnalysis: intermediateAnalysis,
                  oldBinaryAnalysis: oldAnalysis
                }
              )

              // Save diff
              const diffPath = path.join(dataDir, `connect-diff-${fromVer}_to_${toVer}.json`)
              fs.writeFileSync(diffPath, JSON.stringify(diffData, null, 2), 'utf8')
              console.log(`✓ Diff saved: ${path.basename(diffPath)}`)

              // Save augmented data back to file for subsequent iterations
              // This ensures the next version pair has consistent metadata when loading this version as oldData
              const augmentedDataPath = path.join(dataDir, `connect-${toVer}.json`)
              fs.writeFileSync(augmentedDataPath, JSON.stringify(augmentedNewData, null, 2), 'utf8')
              console.log(`✓ Augmented data saved: connect-${toVer}.json`)

              // Update what's-new if requested
              if (options.updateWhatsNew) {
                console.log(`Updating what's-new.adoc for ${toVer}...`)
                updateWhatsNew({ dataDir, oldVersion: fromVer, newVersion: toVer, binaryAnalysis: intermediateAnalysis })
              }

              intermediateProcessingResults.push({
                fromVersion: fromVer,
                toVersion: toVer,
                diffPath,
                success: true
              })

              console.log(`✅ Completed processing: ${fromVer} → ${toVer}\n`)
            } catch (err) {
              console.error(`❌ Error processing ${fromVer} → ${toVer}: ${err.message}`)
              console.error('   Continuing with next version...\n')

              intermediateProcessingResults.push({
                fromVersion: fromVer,
                toVersion: toVer,
                error: err.message,
                success: false
              })
            }
          }

          console.log(`\n${'='.repeat(80)}`)
          console.log(`✓ Intermediate release processing complete`)
          console.log(`   Processed: ${intermediateProcessingResults.filter(r => r.success).length}/${intermediateProcessingResults.length} version pairs`)
          console.log('='.repeat(80) + '\n')
        }
      } catch (err) {
        console.warn(`\n⚠️  Warning: Failed to discover intermediate releases: ${err.message}`)
        console.warn('   Falling back to single version comparison...\n')
      }
    }
  }

  // ========================================================================
  // Main Processing: Handle the latest version (final iteration)
  // ========================================================================

  console.log('Generating connector partials...')
  let partialsWritten, partialFiles

  try {
    const result = await generateRpcnConnectorDocs({
      data: dataFile,
      overrides: options.overrides,
      template: options.templateMain,
      templateIntro: options.templateIntro,
      templateFields: options.templateFields,
      templateExamples: options.templateExamples,
      templateBloblang: options.templateBloblang,
      writeFullDrafts: false,
      includeBloblang: !!options.includeBloblang,
      csvMetadata
    })
    partialsWritten = result.partialsWritten
    partialFiles = result.partialFiles
  } catch (err) {
    console.error(`Error: Failed to generate partials: ${err.message}`)
    process.exit(1)
  }

  let oldIndex = {}
  let oldVersion = null
  if (options.oldData && fs.existsSync(options.oldData)) {
    // Load with platform metadata intact for accurate diff
    oldIndex = JSON.parse(fs.readFileSync(options.oldData, 'utf8'))
    const m = options.oldData.match(/connect-([\d.]+)\.json$/)
    if (m) oldVersion = m[1]
  } else {
    const existingVersions = fs.readdirSync(dataDir)
      .filter(f => /^connect-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.json$/.test(f))
      .filter(f => f !== path.basename(dataFile))
      .map(f => {
        const match = f.match(/^connect-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.json$/)
        return match ? match[1] : null
      })
      .filter(v => v && semver.valid(v))

    if (existingVersions.length > 0) {
      const sortedVersions = semver.rsort(existingVersions)
      oldVersion = sortedVersions[0]
      const oldFile = `connect-${oldVersion}.json`
      const oldPath = path.join(dataDir, oldFile)
      // Load with platform metadata intact for accurate diff
      oldIndex = JSON.parse(fs.readFileSync(oldPath, 'utf8'))
      console.log(`📋 Using old version data: ${oldFile}`)
    } else {
      oldVersion = getAntoraValue('asciidoc.attributes.latest-connect-version')
      if (oldVersion) {
        const oldPath = path.join(dataDir, `connect-${oldVersion}.json`)
        if (fs.existsSync(oldPath)) {
          // Load with platform metadata intact for accurate diff
          oldIndex = JSON.parse(fs.readFileSync(oldPath, 'utf8'))
        }
      }
    }
  }

  // Load with platform metadata intact for accurate diff
  let newIndex = JSON.parse(fs.readFileSync(dataFile, 'utf8'))

  // Save a clean copy of OSS data for binary analysis
  // Binary analyzer needs pure OSS data without augmented CGO/cloud connectors
  const cleanOssDataPath = path.join(dataDir, `._connect-${newVersion}-clean.json`)

  // Create clean version by removing augmented connectors
  const cleanData = JSON.parse(JSON.stringify(newIndex))
  const connectorTypes = ['inputs', 'outputs', 'processors', 'caches', 'rate_limits',
    'buffers', 'metrics', 'scanners', 'tracers']

  for (const type of connectorTypes) {
    if (Array.isArray(cleanData[type])) {
      // Keep only connectors from OSS rpk (have config/fields)
      // Remove augmentation-only connectors (added by previous binary analysis)
      cleanData[type] = cleanData[type].filter(c => c.config || c.fields)

      // Remove platform metadata from remaining connectors
      stripPlatformMetadata(cleanData[type])
    }
  }

  fs.writeFileSync(cleanOssDataPath, JSON.stringify(cleanData, null, 2), 'utf8')

  const versionsMatch = oldVersion && newVersion && oldVersion === newVersion
  if (versionsMatch) {
    console.log(`\n✓ Already at version ${newVersion}`)
    console.log('  Skipping diff generation, but will run binary analysis.\n')
  }

  // Binary analysis
  let oldBinaryAnalysis = null

  if (oldVersion) {
    const standalonePath = path.join(dataDir, `binary-analysis-${oldVersion}.json`)
    if (fs.existsSync(standalonePath)) {
      try {
        oldBinaryAnalysis = JSON.parse(fs.readFileSync(standalonePath, 'utf8'))
        console.log(`✓ Loaded old binary analysis from: binary-analysis-${oldVersion}.json`)
      } catch (err) {
        console.warn(`Warning: Failed to load ${standalonePath}: ${err.message}`)
      }
    }

    if (!oldBinaryAnalysis) {
      const diffFiles = fs.readdirSync(dataDir)
        .filter(f => f.startsWith('connect-diff-') && f.endsWith(`_to_${oldVersion}.json`))
        .sort()
        .reverse()

      for (const file of diffFiles) {
        const diffPath = path.join(dataDir, file)
        try {
          const oldDiff = JSON.parse(fs.readFileSync(diffPath, 'utf8'))
          if (oldDiff.binaryAnalysis) {
            oldBinaryAnalysis = {
              comparison: {
                inCloud: oldDiff.binaryAnalysis.details?.cloudSupported || [],
                notInCloud: oldDiff.binaryAnalysis.details?.selfHostedOnly || []
              },
              cgoOnly: oldDiff.binaryAnalysis.details?.cgoOnly || []
            }
            console.log(`✓ Loaded old binary analysis from: ${file}`)
            break
          }
        } catch {
          // Continue to next file
        }
      }
    }
  }

  // Always use clean OSS data for comparison
  // Temporarily rename the file so the analyzer finds it
  const expectedPath = path.join(dataDir, `connect-${newVersion}.json`)
  let tempRenamed = false

  try {
    console.log('\nAnalyzing connector binaries...')

    if (fs.existsSync(cleanOssDataPath)) {
      if (fs.existsSync(expectedPath)) {
        fs.renameSync(expectedPath, path.join(dataDir, `._connect-${newVersion}-augmented.json.tmp`))
        tempRenamed = true
      }
      fs.copyFileSync(cleanOssDataPath, expectedPath)
    }

    const analysisOptions = {
      skipCloud: false,
      skipCgo: false,
      cgoVersion: options.cgoVersion || null
    }

    binaryAnalysis = await analyzeAllBinaries(
      newVersion,
      options.cloudVersion || null,
      dataDir,
      analysisOptions
    )

    console.log('Done: Binary analysis complete:')
    console.log(`   • OSS version: ${binaryAnalysis.ossVersion}`)

    if (binaryAnalysis.cloudVersion) {
      console.log(`   • Cloud version: ${binaryAnalysis.cloudVersion}`)
    }

    if (binaryAnalysis.comparison) {
      console.log(`   • Connectors in cloud: ${binaryAnalysis.comparison.inCloud.length}`)
      console.log(`   • Self-hosted only: ${binaryAnalysis.comparison.notInCloud.length}`)
      if (binaryAnalysis.comparison.cloudOnly && binaryAnalysis.comparison.cloudOnly.length > 0) {
        console.log(`   • Cloud-only connectors: ${binaryAnalysis.comparison.cloudOnly.length}`)
      }
    }

    if (binaryAnalysis.cgoOnly && binaryAnalysis.cgoOnly.length > 0) {
      console.log(`   • cgo-only connectors: ${binaryAnalysis.cgoOnly.length}`)
    }
  } catch (err) {
    console.error(`Warning: Binary analysis failed: ${err.message}`)
    console.error('   Continuing without binary analysis data...')
  } finally {
    // Restore the augmented file regardless of success or failure
    if (tempRenamed) {
      if (fs.existsSync(expectedPath)) {
        fs.unlinkSync(expectedPath)
      }
      const tmpPath = path.join(dataDir, `._connect-${newVersion}-augmented.json.tmp`)
      if (fs.existsSync(tmpPath)) {
        fs.renameSync(tmpPath, expectedPath)
      }
    }
  }

  // Augment data file
  if (needsAugmentation && binaryAnalysis) {
    try {
      console.log('\nAugmenting connector data with cloud/cgo fields...')

      const connectorData = JSON.parse(fs.readFileSync(dataFile, 'utf8'))

      const { augmentedData, augmentedCount, addedCgoCount, addedCloudOnlyCount } =
        augmentConnectorData(connectorData, binaryAnalysis)

      fs.writeFileSync(dataFile, JSON.stringify(augmentedData, null, 2), 'utf8')
      console.log(`Done: Augmented ${augmentedCount} connectors with cloud/cgo fields`)
      if (addedCgoCount > 0) {
        console.log(`   • Added ${addedCgoCount} cgo-only connectors to data file`)
      }
      if (addedCloudOnlyCount > 0) {
        console.log(`   • Added ${addedCloudOnlyCount} cloud-only connectors to data file`)
      }

      // Keep only the latest 2 versions (by semver), delete all others
      const dataVersions = fs.readdirSync(dataDir)
        .filter(f => /^connect-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.json$/.test(f))
        .map(f => {
          const match = f.match(/^connect-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.json$/)
          return match ? match[1] : null
        })
        .filter(v => v && semver.valid(v))
        .sort((a, b) => semver.rcompare(a, b)) // Sort descending (newest first)

      // Keep only the latest 2 versions
      const versionsToKeep = new Set(dataVersions.slice(0, 2))

      // Delete all older versions
      for (const version of dataVersions) {
        if (!versionsToKeep.has(version)) {
          const dataFile = `connect-${version}.json`
          const dataPath = path.join(dataDir, dataFile);
          fs.unlinkSync(dataPath);
          console.log(`🧹 Deleted old version from docs-data: ${dataFile}`);
        }
      }

      // Also clean up old CSV files (keep latest 2)
      const csvVersions = fs.readdirSync(dataDir)
        .filter(f => /^connect-info-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.csv$/.test(f))
        .map(f => {
          const match = f.match(/^connect-info-(\d+\.\d+\.\d+(?:-[0-9A-Za-z-.]+)?)\.csv$/)
          return match ? match[1] : null
        })
        .filter(v => v && semver.valid(v))
        .sort((a, b) => semver.rcompare(a, b))

      const csvToKeep = new Set(csvVersions.slice(0, 2))
      for (const version of csvVersions) {
        if (!csvToKeep.has(version)) {
          const csvFile = `connect-info-${version}.csv`
          const csvPath = path.join(dataDir, csvFile);
          fs.unlinkSync(csvPath);
          console.log(`🧹 Deleted old CSV from docs-data: ${csvFile}`);
        }
      }

      // IMPORTANT: Reload newIndex with augmented data for unified diff
      // The unified diff approach compares platform metadata to detect transitions
      newIndex = augmentedData
      console.log(`✓ Reloaded newIndex with augmented data for diff comparison`)
    } catch (err) {
      console.error(`Warning: Failed to augment data file: ${err.message}`)
    }
  }

  // Publish merged version to attachments
  // IMPORTANT: This must run AFTER binary analysis and