@redpanda-data/docs-extensions-and-macros/tools/redpanda-connect/report-delta.js

Antora extensions and macros developed for Redpanda documentation.

const { execSync } = require('child_process');

/**
 * Generate a JSON diff report between two connector index objects.
 * Includes platform metadata (CGO, cloud-only) to detect transitions.
 * @param {object} oldIndex - Previous version connector index
 * @param {object} newIndex - Current version connector index
 * @param {object} opts - { oldVersion, newVersion, timestamp, binaryAnalysis, oldBinaryAnalysis }
 * @returns {object} JSON diff report
 */
function generateConnectorDiffJson(oldIndex, newIndex, opts = {}) {
  const oldMap = buildComponentMap(oldIndex);
  const newMap = buildComponentMap(newIndex);

  // New components (include platform metadata)
  const newComponentKeys = Object.keys(newMap).filter(k => !(k in oldMap));
  const newComponents = newComponentKeys.map(key => {
    const [type, name] = key.split(':');
    const raw = newMap[key].raw;
    const metadata = newMap[key].metadata || {};
    return {
      name,
      type,
      status: raw.status || raw.type || '',
      version: raw.version || raw.introducedInVersion || '',
      description: raw.description || '',
      requiresCgo: metadata.requiresCgo || false,
      cloudOnly: metadata.cloudOnly || false,
      cloudSupported: metadata.cloudSupported || false
    };
  });

  // Removed components (include platform metadata to understand why removed)
  const removedComponentKeys = Object.keys(oldMap).filter(k => !(k in newMap));
  const removedComponents = removedComponentKeys.map(key => {
    const [type, name] = key.split(':');
    const raw = oldMap[key].raw;
    const metadata = oldMap[key].metadata || {};
    return {
      name,
      type,
      status: raw.status || raw.type || '',
      version: raw.version || raw.introducedInVersion || '',
      description: raw.description || '',
      requiresCgo: metadata.requiresCgo || false,
      cloudOnly: metadata.cloudOnly || false,
      cloudSupported: metadata.cloudSupported || false
    };
  });

  // New fields under existing components
  const newFields = [];
  Object.keys(newMap).forEach(cKey => {
    if (!(cKey in oldMap)) return;
    const oldFields = new Set(oldMap[cKey].fields || []);
    const newFieldsArr = newMap[cKey].fields || [];
    newFieldsArr.forEach(fName => {
      if (!oldFields.has(fName)) {
        const [type, compName] = cKey.split(':');
        let rawFieldObj = null;
        if (type === 'config') {
          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
        } else {
          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
        }
        newFields.push({
          component: cKey,
          field: fName,
          introducedIn: rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version),
          description: rawFieldObj && rawFieldObj.description
        });
      }
    });
  });

  // Removed fields under existing components
  const removedFields = [];
  Object.keys(oldMap).forEach(cKey => {
    if (!(cKey in newMap)) return;
    const newFieldsSet = new Set(newMap[cKey].fields || []);
    const oldFieldsArr = oldMap[cKey].fields || [];
    oldFieldsArr.forEach(fName => {
      if (!newFieldsSet.has(fName)) {
        removedFields.push({
          component: cKey,
          field: fName
        });
      }
    });
  });

  // Newly deprecated components (exist in both versions but became deprecated)
  const deprecatedComponents = [];
  Object.keys(newMap).forEach(cKey => {
    if (!(cKey in oldMap)) return;
    const oldStatus = (oldMap[cKey].raw.status || '').toLowerCase();
    const newStatus = (newMap[cKey].raw.status || '').toLowerCase();
    if (oldStatus !== 'deprecated' && newStatus === 'deprecated') {
      const [type, name] = cKey.split(':');
      const raw = newMap[cKey].raw;
      deprecatedComponents.push({
        name,
        type,
        status: raw.status || raw.type || '',
        version: raw.version || raw.introducedInVersion || '',
        description: raw.description || ''
      });
    }
  });

  // Platform transitions (CGO, cloud support changes)
  const platformTransitions = [];
  Object.keys(newMap).forEach(cKey => {
    if (!(cKey in oldMap)) return;

    const oldMeta = oldMap[cKey].metadata || {};
    const newMeta = newMap[cKey].metadata || {};
    const [type, name] = cKey.split(':');

    const transitions = [];

    // CGO requirement changes
    if (!oldMeta.requiresCgo && newMeta.requiresCgo) {
      transitions.push('became_cgo_only');
    } else if (oldMeta.requiresCgo && !newMeta.requiresCgo) {
      transitions.push('no_longer_cgo_only');
    }

    // Cloud support changes
    if (!oldMeta.cloudSupported && newMeta.cloudSupported) {
      transitions.push('added_cloud_support');
    } else if (oldMeta.cloudSupported && !newMeta.cloudSupported) {
      transitions.push('removed_cloud_support');
    }

    // Cloud-only status changes
    if (!oldMeta.cloudOnly && newMeta.cloudOnly) {
      transitions.push('became_cloud_only');
    } else if (oldMeta.cloudOnly && !newMeta.cloudOnly) {
      transitions.push('no_longer_cloud_only');
    }

    if (transitions.length > 0) {
      platformTransitions.push({
        name,
        type,
        transitions,
        oldPlatform: {
          requiresCgo: oldMeta.requiresCgo || false,
          cloudSupported: oldMeta.cloudSupported || false,
          cloudOnly: oldMeta.cloudOnly || false
        },
        newPlatform: {
          requiresCgo: newMeta.requiresCgo || false,
          cloudSupported: newMeta.cloudSupported || false,
          cloudOnly: newMeta.cloudOnly || false
        }
      });
    }
  });

  // Newly deprecated fields (exist in both versions but became deprecated)
  const deprecatedFields = [];
  // Changed default values
  const changedDefaults = [];

  Object.keys(newMap).forEach(cKey => {
    if (!(cKey in oldMap)) return;
    const oldFieldsArr = oldMap[cKey].fields || [];
    const newFieldsArr = newMap[cKey].fields || [];

    // Check fields that exist in both versions
    const commonFields = oldFieldsArr.filter(f => newFieldsArr.includes(f));
    commonFields.forEach(fName => {
      const [type, compName] = cKey.split(':');

      // Get old field object
      let oldFieldObj = null;
      if (type === 'config') {
        oldFieldObj = (oldMap[cKey].raw.children || []).find(f => f.name === fName);
      } else {
        oldFieldObj = (oldMap[cKey].raw.config?.children || []).find(f => f.name === fName);
      }

      // Get new field object
      let newFieldObj = null;
      if (type === 'config') {
        newFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
      } else {
        newFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
      }

      const oldDeprecated = oldFieldObj && (oldFieldObj.is_deprecated === true || oldFieldObj.deprecated === true || (oldFieldObj.status || '').toLowerCase() === 'deprecated');
      const newDeprecated = newFieldObj && (newFieldObj.is_deprecated === true || newFieldObj.deprecated === true || (newFieldObj.status || '').toLowerCase() === 'deprecated');

      if (!oldDeprecated && newDeprecated) {
        deprecatedFields.push({
          component: cKey,
          field: fName,
          description: newFieldObj && newFieldObj.description
        });
      }

      // Check for changed default values
      if (oldFieldObj && newFieldObj) {
        const oldDefault = oldFieldObj.default;
        const newDefault = newFieldObj.default;

        // Compare defaults using JSON stringification to handle objects/arrays
        const oldDefaultStr = JSON.stringify(oldDefault);
        const newDefaultStr = JSON.stringify(newDefault);

        if (oldDefaultStr !== newDefaultStr) {
          changedDefaults.push({
            component: cKey,
            field: fName,
            oldDefault: oldDefault,
            newDefault: newDefault,
            description: newFieldObj && newFieldObj.description
          });
        }
      }
    });
  });

  // Detect new/removed Bloblang methods and functions
  const oldMethods = new Set((oldIndex['bloblang-methods'] || []).filter(Boolean).map(m => m.name).filter(Boolean));
  const newMethods = new Set((newIndex['bloblang-methods'] || []).filter(Boolean).map(m => m.name).filter(Boolean));
  const oldFunctions = new Set((oldIndex['bloblang-functions'] || []).filter(Boolean).map(f => f.name).filter(Boolean));
  const newFunctions = new Set((newIndex['bloblang-functions'] || []).filter(Boolean).map(f => f.name).filter(Boolean));

  const newBloblangMethods = Array.from(newMethods).filter(m => !oldMethods.has(m)).sort();
  const removedBloblangMethods = Array.from(oldMethods).filter(m => !newMethods.has(m)).sort();
  const newBloblangFunctions = Array.from(newFunctions).filter(f => !oldFunctions.has(f)).sort();
  const removedBloblangFunctions = Array.from(oldFunctions).filter(f => !newFunctions.has(f)).sort();

  // Detect deprecated Bloblang methods and functions
  const deprecatedBloblangMethods = [];
  const deprecatedBloblangFunctions = [];

  const oldMethodsMap = new Map((oldIndex['bloblang-methods'] || []).filter(Boolean).filter(m => m.name).map(m => [m.name, m]));
  const newMethodsMap = new Map((newIndex['bloblang-methods'] || []).filter(Boolean).filter(m => m.name).map(m => [m.name, m]));
  const oldFunctionsMap = new Map((oldIndex['bloblang-functions'] || []).filter(Boolean).filter(f => f.name).map(f => [f.name, f]));
  const newFunctionsMap = new Map((newIndex['bloblang-functions'] || []).filter(Boolean).filter(f => f.name).map(f => [f.name, f]));

  // Check methods for newly deprecated status
  newMethodsMap.forEach((newMethod, name) => {
    const oldMethod = oldMethodsMap.get(name);
    if (oldMethod) {
      const oldStatus = (oldMethod.status || '').toLowerCase();
      const newStatus = (newMethod.status || '').toLowerCase();
      if (oldStatus !== 'deprecated' && newStatus === 'deprecated') {
        deprecatedBloblangMethods.push(name);
      }
    }
  });

  // Check functions for newly deprecated status
  newFunctionsMap.forEach((newFunction, name) => {
    const oldFunction = oldFunctionsMap.get(name);
    if (oldFunction) {
      const oldStatus = (oldFunction.status || '').toLowerCase();
      const newStatus = (newFunction.status || '').toLowerCase();
      if (oldStatus !== 'deprecated' && newStatus === 'deprecated') {
        deprecatedBloblangFunctions.push(name);
      }
    }
  });

  const result = {
    comparison: {
      oldVersion: opts.oldVersion || '',
      newVersion: opts.newVersion || '',
      timestamp: opts.timestamp || new Date().toISOString()
    },
    summary: {
      newComponents: newComponents.length,
      removedComponents: removedComponents.length,
      newFields: newFields.length,
      removedFields: removedFields.length,
      deprecatedComponents: deprecatedComponents.length,
      deprecatedFields: deprecatedFields.length,
      changedDefaults: changedDefaults.length,
      platformTransitions: platformTransitions.length,
      newBloblangMethods: newBloblangMethods.length,
      removedBloblangMethods: removedBloblangMethods.length,
      newBloblangFunctions: newBloblangFunctions.length,
      removedBloblangFunctions: removedBloblangFunctions.length,
      deprecatedBloblangMethods: deprecatedBloblangMethods.length,
      deprecatedBloblangFunctions: deprecatedBloblangFunctions.length
    },
    details: {
      newComponents,
      removedComponents,
      newFields,
      removedFields,
      deprecatedComponents,
      deprecatedFields,
      changedDefaults,
      platformTransitions,
      newBloblangMethods,
      removedBloblangMethods,
      newBloblangFunctions,
      removedBloblangFunctions,
      deprecatedBloblangMethods,
      deprecatedBloblangFunctions
    }
  };

  // Include binary analysis data if provided
  if (opts.binaryAnalysis) {
    const ba = opts.binaryAnalysis;
    const oldBa = opts.oldBinaryAnalysis || {};

    result.binaryAnalysis = {
      versions: {
        oss: ba.ossVersion,
        cloud: ba.cloudVersion || null,
        cgo: ba.cgoVersion || null
      },
      current: {
        cloudSupported: ba.comparison?.inCloud?.length || 0,
        selfHostedOnly: ba.comparison?.notInCloud?.length || 0,
        cgoOnly: ba.cgoOnly?.length || 0
      },
      changes: {}
    };

    // Calculate cloud support changes
    if (oldBa.comparison && ba.comparison) {
      const oldCloudSet = new Set(oldBa.comparison.inCloud?.map(c => `${c.type}:${c.name}`) || []);
      const newCloudSet = new Set(ba.comparison.inCloud?.map(c => `${c.type}:${c.name}`) || []);

      const addedToCloud = ba.comparison.inCloud?.filter(c =>
        !oldCloudSet.has(`${c.type}:${c.name}`)
      ) || [];

      const removedFromCloud = oldBa.comparison.inCloud?.filter(c =>
        !newCloudSet.has(`${c.type}:${c.name}`)
      ) || [];

      result.binaryAnalysis.changes.cloud = {
        added: addedToCloud.map(c => ({ type: c.type, name: c.name, status: c.status })),
        removed: removedFromCloud.map(c => ({ type: c.type, name: c.name, status: c.status }))
      };
    }

    // Calculate cgo-only changes
    if (oldBa.cgoOnly && ba.cgoOnly) {
      const oldCgoSet = new Set(oldBa.cgoOnly.map(c => `${c.type}:${c.name}`));
      const newCgoSet = new Set(ba.cgoOnly.map(c => `${c.type}:${c.name}`));

      const newCgoOnly = ba.cgoOnly.filter(c =>
        !oldCgoSet.has(`${c.type}:${c.name}`)
      );

      const removedCgoOnly = oldBa.cgoOnly.filter(c =>
        !newCgoSet.has(`${c.type}:${c.name}`)
      );

      result.binaryAnalysis.changes.cgo = {
        newCgoOnly: newCgoOnly.map(c => ({ type: c.type, name: c.name, status: c.status })),
        removedCgoOnly: removedCgoOnly.map(c => ({ type: c.type, name: c.name, status: c.status }))
      };
    }

    // Include full lists for reference
    result.binaryAnalysis.details = {
      cloudSupported: ba.comparison?.inCloud?.map(c => ({ type: c.type, name: c.name, status: c.status })) || [],
      selfHostedOnly: ba.comparison?.notInCloud?.map(c => ({ type: c.type, name: c.name, status: c.status })) || [],
      cloudOnly: ba.comparison?.cloudOnly?.map(c => ({ type: c.type, name: c.name, status: c.status })) || [],
      cgoOnly: ba.cgoOnly?.map(c => ({ type: c.type, name: c.name, status: c.status })) || []
    };
  }

  return result;
}

function discoverComponentKeys(obj) {
  return Object.keys(obj).filter(key => Array.isArray(obj[key]));
}

function buildComponentMap(indexObj) {
  const map = {};
  const types = discoverComponentKeys(indexObj);

  types.forEach(type => {
    (indexObj[type] || []).forEach(component => {
      const name = component.name;
      if (!name) return;

      const lookupKey = `${type}:${name}`;
      let childArray = [];

      if (type === 'config') {
        if (Array.isArray(component.children)) {
          childArray = component.children;
        }
      } else {
        if (component.config && Array.isArray(component.config.children)) {
          childArray = component.config.children;
        }
      }

      const fieldNames = childArray.map(f => f.name);

      // Preserve platform metadata for accurate diff comparison
      const metadata = {
        requiresCgo: component.requiresCgo || false,
        cloudSupported: component.cloudSupported || false,
        cloudOnly: component.cloudOnly || false
      };

      map[lookupKey] = {
        raw: component,
        fields: fieldNames,
        metadata: metadata
      };
    });
  });

  return map;
}

function getRpkConnectVersion() {
  try {
    // Make sure the connect plugin is upgraded first (silent)
    execSync('rpk connect upgrade', { stdio: 'ignore' });

    // Now capture the --version output
    const raw = execSync('rpk connect --version', {
      stdio: ['ignore', 'pipe', 'ignore'],
    })
      .toString()
      .trim();

    // raw looks like:
    //   Version: 4.53.0
    //   Date:    2025-04-18T17:49:53Z
    // We want to extract “4.53.0”
    const match = raw.match(/^Version:\s*(.+)$/m);
    if (!match) {
      throw new Error(`Unexpected format from "rpk connect --version":\n${raw}`);
    }
    return match[1];
  } catch (err) {
    throw new Error(`Unable to run "rpk connect --version": ${err.message}`);
  }
}

/**
 * Given two “index objects” (parsed from connect.json), produce a console summary of:
 *  • which connectors/components are brand-new
 *  • which new fields appeared under existing connectors (including “config” entries)
 *  • for each new component/field, if the raw object contains “version” or “introducedInVersion” or “requiresVersion” metadata, print it
 */
function printDeltaReport(oldIndex, newIndex) {
  const oldMap = buildComponentMap(oldIndex);
  const newMap = buildComponentMap(newIndex);

  // 1) brand-new components
  const newComponentKeys = Object.keys(newMap).filter(k => !(k in oldMap));

  // 2) brand-new fields under shared components
  const newFields = [];
  Object.keys(newMap).forEach(cKey => {
    if (!(cKey in oldMap)) return; // skip brand-new components here
    const oldFields = new Set(oldMap[cKey].fields || []);
    const newFieldsArr = newMap[cKey].fields || [];
    newFieldsArr.forEach(fName => {
      if (!oldFields.has(fName)) {
        // fetch raw field metadata if available
        const [type, compName] = cKey.split(':');
        let rawFieldObj = null;
        if (type === 'config') {
          rawFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
        } else {
          rawFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
        }

        let introducedIn = rawFieldObj && (rawFieldObj.introducedInVersion || rawFieldObj.version);
        let requiresVer = rawFieldObj && rawFieldObj.requiresVersion;

        newFields.push({
          component: cKey,
          field: fName,
          introducedIn,
          requiresVersion: requiresVer,
        });
      }
    });
  });

  // Newly deprecated components
  const deprecatedComponentKeys = [];
  Object.keys(newMap).forEach(cKey => {
    if (!(cKey in oldMap)) return;
    const oldStatus = (oldMap[cKey].raw.status || '').toLowerCase();
    const newStatus = (newMap[cKey].raw.status || '').toLowerCase();
    if (oldStatus !== 'deprecated' && newStatus === 'deprecated') {
      deprecatedComponentKeys.push(cKey);
    }
  });

  // Newly deprecated fields
  const deprecatedFieldsList = [];
  // Changed default values
  const changedDefaultsList = [];

  Object.keys(newMap).forEach(cKey => {
    if (!(cKey in oldMap)) return;
    const oldFieldsArr = oldMap[cKey].fields || [];
    const newFieldsArr = newMap[cKey].fields || [];
    const commonFields = oldFieldsArr.filter(f => newFieldsArr.includes(f));

    commonFields.forEach(fName => {
      const [type, compName] = cKey.split(':');
      let oldFieldObj = null;
      if (type === 'config') {
        oldFieldObj = (oldMap[cKey].raw.children || []).find(f => f.name === fName);
      } else {
        oldFieldObj = (oldMap[cKey].raw.config?.children || []).find(f => f.name === fName);
      }
      let newFieldObj = null;
      if (type === 'config') {
        newFieldObj = (newMap[cKey].raw.children || []).find(f => f.name === fName);
      } else {
        newFieldObj = (newMap[cKey].raw.config?.children || []).find(f => f.name === fName);
      }
      const oldDeprecated = oldFieldObj && (oldFieldObj.is_deprecated === true || oldFieldObj.deprecated === true || (oldFieldObj.status || '').toLowerCase() === 'deprecated');
      const newDeprecated = newFieldObj && (newFieldObj.is_deprecated === true || newFieldObj.deprecated === true || (newFieldObj.status || '').toLowerCase() === 'deprecated');
      if (!oldDeprecated && newDeprecated) {
        deprecatedFieldsList.push({ component: cKey, field: fName });
      }

      // Check for changed default values
      if (oldFieldObj && newFieldObj) {
        const oldDefault = oldFieldObj.default;
        const newDefault = newFieldObj.default;

        // Compare defaults using JSON stringification to handle objects/arrays
        const oldDefaultStr = JSON.stringify(oldDefault);
        const newDefaultStr = JSON.stringify(newDefault);

        if (oldDefaultStr !== newDefaultStr) {
          changedDefaultsList.push({
            component: cKey,
            field: fName,
            oldDefault: oldDefault,
            newDefault: newDefault
          });
        }
      }
    });
  });

  console.log('\n📋 RPCN Connector Delta Report\n');

  if (newComponentKeys.length) {
    console.log('➤ Newly added components:');
    newComponentKeys.forEach(key => {
      const [type, name] = key.split(':');
      const raw = newMap[key].raw;
      const status = raw.status || raw.type || '';
      const version = raw.version || raw.introducedInVersion || '';
      console.log(
        `   • ${type}/${name}${
          status ? ` (${status})` : ''
        }${version ? ` — introduced in ${version}` : ''}`
      );
    });
    console.log('');
  } else {
    console.log('➤ No newly added components.\n');
  }

  if (newFields.length) {
    console.log('➤ Newly added fields:');
    newFields.forEach(entry => {
      const { component, field, introducedIn, requiresVersion } = entry;
      process.stdout.write(`   • ${component} → ${field}`);
      if (introducedIn) process.stdout.write(` (introducedIn: ${introducedIn})`);
      if (requiresVersion) process.stdout.write(` (requiresVersion: ${requiresVersion})`);
      console.log('');
    });
    console.log('');
  } else {
    console.log('➤ No newly added fields.\n');
  }

  if (deprecatedComponentKeys.length) {
    console.log('➤ Newly deprecated components:');
    deprecatedComponentKeys.forEach(key => {
      const [type, name] = key.split(':');
      const raw = newMap[key].raw;
      console.log(`   • ${type}/${name}`);
    });
    console.log('');
  } else {
    console.log('➤ No newly deprecated components.\n');
  }

  if (deprecatedFieldsList.length) {
    console.log('➤ Newly deprecated fields:');
    deprecatedFieldsList.forEach(entry => {
      const { component, field } = entry;
      console.log(`   • ${component} → ${field}`);
    });
    console.log('');
  } else {
    console.log('➤ No newly deprecated fields.\n');
  }

  if (changedDefaultsList.length) {
    console.log('➤ Changed default values:');
    changedDefaultsList.forEach(entry => {
      const { component, field, oldDefault, newDefault } = entry;
      const oldStr = JSON.stringify(oldDefault);
      const newStr = JSON.stringify(newDefault);
      console.log(`   • ${component} → ${field}`);
      console.log(`     Old: ${oldStr}`);
      console.log(`     New: ${newStr}`);
    });
    console.log('');
  } else {
    console.log('➤ No changed default values.\n');
  }
}

module.exports = {
  discoverComponentKeys,
  buildComponentMap,
  getRpkConnectVersion,
  printDeltaReport,
  generateConnectorDiffJson,
};