@apistudio/apim-cli/node_modules/@apic/studio-build/src/build-project-assets.ts

CLI for API Management Products

/**
 * Copyright IBM Corp. 2024, 2025
 */
import { ReferenceValidationResult, ReferenceValidationResultMap } from './model/interface.js';
import { GatewaysJson, loadYaml } from '@apic/studio-shared';
import { ProjectAssetValidator } from './validator/asset-validator.js';
import JSZip from 'jszip';
import yaml from 'js-yaml';
import {
  checkFileExtension,
  convertNumberToString,
  isValidAsset,
  updateMapWithMetadata,
  updateRefs,
} from './utils.js';
import path from 'path';
import { DataPowerAdapter } from './adapter/datapower-adapter.js';
import { Logger } from '@apic/studio-shared';
import { errorsArray } from './utils.js';
import { AssetModelKindConstants } from '@apic/studio-client-model';
import { YamlContent } from '@apic/studio-shared';

export class BuildProjectAssets {
  public async loadZipFromBuffer(fileBuffer: Buffer): Promise<JSZip> {
    Logger.info('Loading ZIP from buffer');
    const zip = new JSZip();
    return zip.loadAsync(fileBuffer);
  }

  private async validate(fileBuffer: Buffer): Promise<ReferenceValidationResultMap> {
    Logger.info('Validating ZIP file');
    const AssetValidator = new ProjectAssetValidator();
    const folderNames = new Set<string>();
    const filePathsInFolder = new Set<string>();

    await this.extractFolderNamesAndPaths(fileBuffer, folderNames, filePathsInFolder);
    const validationPromises = Array.from(folderNames).map(async (folderName) => {
      return this.validateFolder(
        fileBuffer,
        folderName,
        AssetValidator,
        filePathsInFolder,
        folderNames
      );
    });

    const validationResults = await Promise.all(validationPromises);
    const assetUniqueness = await AssetValidator.validateAssetUniqueness(fileBuffer);
    const isValid = assetUniqueness && validationResults.every((result) => result.isValid);

    const allYamlErrors: string[] = [];

    // create a map with folderName and it's refMap
    const allRefMaps = new Map<string, Map<string, boolean>>();
    validationResults.forEach((result, index) => {
      const folderName = Array.from(folderNames)[index];
      if (folderName) {
        allRefMaps.set(folderName, result.refMap);
      }

      if (result.errors && result.errors.length > 0) {
        allYamlErrors.push(...result.errors);
      }
    });

    return { isValid, allRefMaps, errors: allYamlErrors };
  }

  private async extractFolderNamesAndPaths(
    fileBuffer: Buffer,
    folderNames: Set<string>,
    filePathsInFolder: Set<string>
  ): Promise<void> {
    Logger.info('Extracting folder names and paths');
    const zipContent = await this.loadZipFromBuffer(fileBuffer);
    for (const fileName in zipContent.files) {
      const folderName = fileName.split(path.sep)[0];
      if (folderName && folderName !== 'dependencies') {
        folderNames.add(folderName);
        filePathsInFolder.add(fileName);
      }
    }
  }

  private async validateFolder(
    fileBuffer: Buffer,
    folderName: string,
    AssetValidator: ProjectAssetValidator,
    filePathsInFolder: Set<string>,
    allFolderNames: Set<string>
  ): Promise<ReferenceValidationResult> {
    //reseting this to 0 as it has older errors as it is.
    errorsArray.length = 0;

    const assetReferenceValid = await AssetValidator.validateProjectAssetReference(
      fileBuffer,
      folderName,
      allFolderNames
    );
    const pathReferenceValid = await AssetValidator.validateProjectPathReference(
      fileBuffer,
      folderName,
      filePathsInFolder
    );
    const minimumAssetsValid = await AssetValidator.validateProjectHasMinimumAssets(fileBuffer);
    const apiSpecVariableValid = await AssetValidator.validateProjectApiSpecVariable(
      fileBuffer,
      folderName
    );
    const soapGatewayValid = await AssetValidator.validateSoapApiGatewayRestriction(fileBuffer);
    // const yamlValidationResult = await AssetValidator.validateYaml(fileBuffer);
    // const yamlValid = yamlValidationResult.isValid;

    // return isvalid true if all the validations are true
    // return refMap to consolidate assets later
    return {
      isValid:
        assetReferenceValid.isValid &&
        pathReferenceValid &&
        minimumAssetsValid &&
        apiSpecVariableValid &&
        soapGatewayValid,
      refMap: assetReferenceValid.refMap,
      errors: errorsArray.map((e) => e.description),
    };
  }

  private async getFileFromZip(fileBuffer: Buffer, filePath: string): Promise<string | null> {
    const zipContent = await this.loadZipFromBuffer(fileBuffer);
    const file = zipContent.file(filePath);
    if (file) {
      return file.async('string');
    }

    return null;
  }

  private async createProjectBuildZip(
    buffer: Buffer,
    allRefMaps: Map<string, Map<string, boolean>>,
    mode: string
  ): Promise<JSZip> {
    Logger.info('Creating project build ZIP');
    const buildZip = new JSZip();
    await this.loadZipFromBuffer(buffer);
    const folderNames = new Set<string>();
    const filePathsInFolder = new Set<string>();
    let specToContentMap = new Map();

    await this.extractFolderNamesAndPaths(buffer, folderNames, filePathsInFolder);
    specToContentMap = await this.adaptToDataPower(buffer, specToContentMap);
    await this.addConsolidatedYAMLs(buildZip, buffer, folderNames, allRefMaps);
    await this.addReferencedFiles(buildZip, buffer, folderNames, specToContentMap, mode);

    return buildZip;
  }

  async adaptToDataPower(fileBuffer: Buffer, specToContentMap: Map<string, string>) {
    const DPAdapter = new DataPowerAdapter();
    const isDataPower = await DPAdapter.checkForDataPowerAssembly(fileBuffer);
    if (isDataPower) {
      specToContentMap = await DPAdapter.getDataPowerAssemblyContent(fileBuffer);
    }
    return specToContentMap;
  }

  private async addConsolidatedYAMLs(
    buildZip: JSZip,
    buffer: Buffer,
    folderNames: Set<string>,
    allRefMaps: Map<string, Map<string, boolean>>
  ): Promise<void> {
    Logger.info('Adding consolidated YAMLs to build ZIP');
    for (const folderName of folderNames) {
      const consolidatedYaml = await this.createConsolidatedYaml(buffer, folderName, allRefMaps);
      buildZip.file(`${folderName}.yaml`, consolidatedYaml);
    }
  }

  private async findMatchingApiMetadataForSpecFile(
    buffer: Buffer,
    specFileName: string
  ): Promise<{ namespace: string; name: string; version: string } | null> {
    const zip = await JSZip.loadAsync(buffer);

    for (const fileName of Object.keys(zip.files)) {
      const zipEntry = zip.file(fileName);
      if (!zipEntry) continue;

      try {
        const fileHandle = await zipEntry.async('string');
        const parsed: any = loadYaml(fileHandle);

        //checks for api file to read the apispecpath in the api file
        if (parsed?.kind === AssetModelKindConstants.API && parsed?.spec?.['api-spec']?.['$path']) {
          let apiSpecPath;
          if (parsed?.metadata?.type == 'SOAP' && parsed?.spec?.['rest-def']?.['$path']) {
            apiSpecPath = parsed.spec['rest-def']['$path'];
          } else {
            apiSpecPath = parsed.spec['api-spec']['$path'];
          }

          //compares api-spec file name and apispecpath in api file and returns api metadata on match
          if (path.basename(apiSpecPath) === path.basename(specFileName)) {
            const metadata = parsed.metadata || {};
            return {
              namespace: metadata.namespace || '',
              name: metadata.name || '',
              version: metadata.version || '',
            };
          }
        }
      } catch (err) {
        console.warn(`Failed to parse ${fileName}:`, err);
        continue;
      }
    }

    return null;
  }

  private async addReferencedFiles(
    buildZip: JSZip,
    buffer: Buffer,
    folderNames: Set<string>,
    specToContentMap: Map<string, string> | undefined,
    _mode: string
  ): Promise<void> {
    Logger.info('Adding referenced files to build ZIP');
    const AssetValidator = new ProjectAssetValidator();

    for (const folderName of folderNames) {
      const refMap = await AssetValidator.createProjectPathReferenceMap(buffer, folderName);
      const promises = Array.from(refMap.keys()).map(async (key) => {
        let file = await this.getFileFromZip(buffer, path.normalize(`${folderName}/${key}`));
        if (file !== null) {
          if (specToContentMap && specToContentMap.get(key) != undefined) {
            let existingSpec, dataPowerAssemblySpec;
            //yaml spec
            if (checkFileExtension(key)) {
              existingSpec = yaml.load(file) as any;
              // get the api metadata from the corresponding api file for the spec file
              const matchingApiInfo = await this.findMatchingApiMetadataForSpecFile(buffer, key);
              if (matchingApiInfo) {
                existingSpec.info['x-ibm-name'] = `${matchingApiInfo.name}`;
              } else {
                existingSpec.info['x-ibm-name'] = existingSpec.info.title;
              }
              dataPowerAssemblySpec = yaml.load(
                specToContentMap.get(key) ? yaml.dump(specToContentMap.get(key)) || '' : ''
              ) as any;
              const mergedSpec = { ...existingSpec, ...dataPowerAssemblySpec };
              file = yaml.dump(mergedSpec, { indent: 2 });
            } else {
              //json spec
              existingSpec = JSON.parse(file);
              // get the api metadata from the corresponding api file for the spec file
              const matchingApiInfo = await this.findMatchingApiMetadataForSpecFile(buffer, key);
              if (matchingApiInfo) {
                existingSpec.info['x-ibm-name'] = `${matchingApiInfo.name}`;
              } else {
                existingSpec.info['x-ibm-name'] = existingSpec.info.title;
              }
              dataPowerAssemblySpec = JSON.parse(
                specToContentMap.get(key) ? JSON.stringify(specToContentMap.get(key)) || '' : ''
              );
              const mergedSpec = { ...existingSpec, ...dataPowerAssemblySpec };
              file = JSON.stringify(mergedSpec, null, 2);
            }
          }
          buildZip.file(path.normalize(`resources/${folderName}/${key}`), file);
        }
      });
      await Promise.all(promises);
    }
  }

  private async createConsolidatedYaml(
    buffer: Buffer,
    folderName: string,
    allRefMaps: Map<string, Map<string, boolean>>
  ): Promise<string> {
    let consolidatedYaml = '';
    const visitedAsset = new Set<string>();
    try {
      const zipContent = await this.loadZipFromBuffer(buffer);
      const versionMap = await this.createVersionProcessingMap(buffer);
      consolidatedYaml += await this.processYamlFiles(
        zipContent,
        folderName,
        visitedAsset,
        versionMap
      );
      consolidatedYaml += await this.processDependencyFiles(zipContent, visitedAsset, versionMap);
      consolidatedYaml += await this.processDependenciesInOtherFolders(
        zipContent,
        visitedAsset,
        versionMap,
        folderName,
        allRefMaps
      );
    } catch (err) {
      Logger.error(
        'Error creating consolidated YAML',
        err instanceof Error ? err : new Error(String(err))
      );
    }

    return consolidatedYaml;
  }

  private async processYamlFiles(
    zipContent: JSZip,
    folderName: string,
    visitedAsset: Set<string>,
    versionMap: Map<string, boolean>
  ): Promise<string> {
    let consolidatedYaml = '';

    for (const fileName in zipContent.files) {
      const entry = zipContent.files[fileName];
      if (
        entry &&
        !entry.dir &&
        fileName.startsWith(folderName + path.sep) &&
        (fileName.endsWith('.yaml') || fileName.endsWith('.yml'))
      ) {
        try {
          const content = await entry.async('string');
          consolidatedYaml += await this.processYamlContent(content, visitedAsset, versionMap);
        } catch (err) {
          Logger.error(
            'Error processing YAML files',
            err instanceof Error ? err : new Error(String(err))
          );
        }
      }
    }

    return consolidatedYaml;
  }

  private async processDependenciesInOtherFolders(
    zipContent: JSZip,
    visitedAsset: Set<string>,
    versionMap: Map<string, boolean>,
    folderName: string,
    allRefMaps: Map<string, Map<string, boolean>>
  ): Promise<string> {
    Logger.info('Processing dependency YAML files');
    let consolidatedYaml = '';

    // retrieve the refMap of the current folderName and process it
    for (const [refFolderName, refMap] of allRefMaps) {
      if (refFolderName === folderName) {
        for (const fileName in zipContent.files) {
          const entry = zipContent.files[fileName];
          if (entry && this.shouldProcessFilesInOtherFolders(entry, fileName, folderName)) {
            try {
              const content = await entry.async('string');
              consolidatedYaml += await this.processYamlContentForOtherFolders(
                content,
                visitedAsset,
                versionMap,
                refMap
              );
            } catch (err) {
              Logger.error(
                'Error parsing dependency YAML file',
                err instanceof Error ? err : new Error(String(err))
              );
            }
          }
        }
      }
    }
    return consolidatedYaml;
  }

  private async processDependencyFiles(
    zipContent: JSZip,
    visitedAsset: Set<string>,
    versionMap: Map<string, boolean>
  ): Promise<string> {
    Logger.info('Processing dependency YAML files');
    let consolidatedYaml = '';

    for (const fileName in zipContent.files) {
      const entry = zipContent.files[fileName];
      if (
        entry &&
        !entry.dir &&
        fileName.startsWith('dependencies' + path.sep) &&
        (fileName.endsWith('.yaml') || fileName.endsWith('.yml'))
      ) {
        try {
          const content = await entry.async('string');
          consolidatedYaml += await this.processYamlContent(content, visitedAsset, versionMap);
        } catch (err) {
          Logger.error(
            'Error parsing dependency YAML file',
            err instanceof Error ? err : new Error(String(err))
          );
        }
      }
    }

    return consolidatedYaml;
  }

  private shouldProcessFile(fileName: string): boolean {
    return (
      (fileName.endsWith('.yaml') || fileName.endsWith('.yml')) && !fileName.includes('resources')
    );
  }

  private shouldProcessFilesInOtherFolders(
    entry: JSZip.JSZipObject,
    fileName: string,
    folderName: string
  ): boolean {
    return (
      !entry.dir &&
      !fileName.startsWith(folderName + path.sep) &&
      (fileName.endsWith('.yaml') || fileName.endsWith('.yml'))
    );
  }

  public async createVersionProcessingMap(buffer: Buffer) {
    const refMap = new Map<string, boolean>();
    try {
      const zipContent = await this.loadZipFromBuffer(buffer);
      for (const fileName in zipContent.files) {
        const entry = zipContent.files[fileName];
        if (entry && this.shouldProcessFile(fileName) && !entry.dir) {
          await this.processFileContent(entry, refMap);
        }
      }
    } catch (err) {
      Logger.error('Error loading ZIP', err instanceof Error ? err : new Error(String(err)), {
        code: '0013',
      });
    }
    return refMap;
  }

  private async processFileContent(entry: any, refMap: Map<string, boolean>) {
    try {
      const content = await entry.async('string');
      const yamlContents = this.parseYaml(content);
      this.processYamlContents(yamlContents, refMap);
    } catch (err) {
      Logger.error('Error parsing YAML', err instanceof Error ? err : new Error(String(err)), {
        code: '0013',
      });
    }
  }

  private parseYaml(content: string): YamlContent[] {
    try {
      return yaml.loadAll(content) as YamlContent[];
    } catch (err) {
      Logger.error('Error parsing YAML', err instanceof Error ? err : new Error(String(err)), {
        code: '0013',
      });
      return [];
    }
  }

  private processYamlContents(yamlContents: YamlContent[], refMap: Map<string, boolean>) {
    for (const yamlContent of yamlContents) {
      if (isValidAsset(yamlContent)) {
        const metadata = yamlContent['metadata'];
        if (typeof metadata.version === 'string') {
          updateMapWithMetadata(yamlContent, refMap);
        }
      }
    }
  }

  private async processYamlContent(
    content: string,
    visitedAsset: Set<string>,
    versionMap: Map<string, boolean>
  ): Promise<string> {
    let yamlResult = '';

    try {
      const yamlContents = yaml.loadAll(content) as YamlContent[];
      for (const yamlContent of yamlContents) {
        if (isValidAsset(yamlContent)) {
          const metadata = yamlContent.metadata;
          const contentString = `${metadata.namespace ? metadata.namespace : ''}:${metadata.name}:${convertNumberToString(metadata.version)}`;
          if (visitedAsset.has(contentString)) {
            Logger.info(`Skipping already visited asset: ${contentString}`);
            continue;
          }
          yamlContent.metadata.version = convertNumberToString(metadata.version);
          visitedAsset.add(contentString);
          const processedYamlContent = updateRefs(yamlContent, versionMap);
          yamlResult += '---' + '\n' + yaml.dump(processedYamlContent) + '\n';
        }
      }
    } catch (err) {
      Logger.error(
        'Error processing YAML content',
        err instanceof Error ? err : new Error(String(err))
      );
    }

    return yamlResult;
  }

  private async processYamlContentForOtherFolders(
    content: string,
    visitedAsset: Set<string>,
    versionMap: Map<string, boolean>,
    refMap: Map<string, boolean>
  ): Promise<string> {
    let yamlResult = '';

    try {
      const yamlContents = yaml.loadAll(content) as YamlContent[];
      for (const yamlContent of yamlContents) {
        if (isValidAsset(yamlContent)) {
          const metadata = yamlContent.metadata;
          const contentString = `${metadata.namespace ? metadata.namespace : ''}:${metadata.name}:${convertNumberToString(metadata.version)}`;
          // If the current file metadata is present in refMap and the metadata value is set to true, then it needs to be added
          if (refMap.has(contentString) && refMap.get(contentString) === true) {
            if (visitedAsset.has(contentString)) {
              // console.log(`Skipping already visited asset: ${contentString}`);
              continue;
            }
            yamlContent.metadata.version = convertNumberToString(metadata.version);
            visitedAsset.add(contentString);
            const processedYamlContent = updateRefs(yamlContent, versionMap);
            yamlResult += '---' + '\n' + yaml.dump(processedYamlContent) + '\n';
          }
        }
      }
    } catch (err) {
      Logger.error(
        'Error processing YAML content',
        err instanceof Error ? err : new Error(String(err))
      );
    }

    return yamlResult;
  }

  async processProjectZip(
    fileBuffer: Buffer,
    mode: string
  ): Promise<{ zip: JSZip | null; errors: string[] }> {
    Logger.info('Processing project ZIP');
    const validatedFileBuffer = await this.validate(fileBuffer);
    if (!validatedFileBuffer.isValid) {
      Logger.info('Project ZIP validation failed');
      //Collecting yaml errors
      return { zip: null, errors: validatedFileBuffer.errors };
    }
    const zip = await this.createProjectBuildZip(fileBuffer, validatedFileBuffer.allRefMaps, mode);
    return { zip, errors: [] };
  }

  async extractGatewaysJson(buffer: Buffer): Promise<GatewaysJson> {
    Logger.info('Extracting gateways.json');
    const zipContent = await this.loadZipFromBuffer(buffer);
    return this.extractGatewaysJsonFromZip(zipContent);
  }

  private async extractGatewaysJsonFromZip(zipContent: JSZip): Promise<GatewaysJson> {
    Logger.info('Extracting gateways.json from ZIP');
    let gatewaysJsonContent: GatewaysJson = {} as GatewaysJson;

    try {
      const gatewaysJsonFile = this.findGatewaysJsonFile(zipContent);
      if (gatewaysJsonFile) {
        gatewaysJsonContent = await this.parseJsonContent(gatewaysJsonFile);
      } else {
        Logger.info('gateways.json file not found in ZIP');
      }
    } catch (err) {
      Logger.error(
        'Error extracting gateways.json',
        err instanceof Error ? err : new Error(String(err))
      );
    }

    return gatewaysJsonContent;
  }

  private findGatewaysJsonFile(zipContent: JSZip): JSZip.JSZipObject | null {
    Logger.info('Finding gateways.json file in ZIP');
    for (const fileName in zipContent.files) {
      const entry = zipContent.files[fileName];
      if (entry && !entry.dir && fileName.includes('gateways.json')) {
        return entry;
      }
    }
    return null;
  }

  private async parseJsonContent(file: JSZip.JSZipObject): Promise<GatewaysJson> {
    Logger.info('Parsing JSON content');
    let jsonContent: GatewaysJson = {} as GatewaysJson;

    try {
      const content = await file.async('string');
      jsonContent = JSON.parse(content);
    } catch (err) {
      Logger.error(
        'Error parsing JSON content',
        err instanceof Error ? err : new Error(String(err))
      );
    }

    return jsonContent;
  }
}