@apistudio/apim-cli/node_modules/@apic/studio-build/src/build-project-assets.ts

CLI for API Management Products

/**
* Copyright Super iPaaS Integration LLC, an IBM Company 2024
*/
import {  Metadata_Ref, YamlContent, ReferenceValidationResult, ReferenceValidationResultMap} from './model/interface.js';
import { GatewaysJson } from '@apic/studio-shared';
import { ProjectAssetValidator } from './validator/asset-validator.js';
import JSZip from 'jszip';
import yaml from 'js-yaml';
import {convertNumberToString, isValidAsset, updateMapWithMetadata, updateRefs} from './utils.js';
import path from 'path';

export class BuildProjectAssets {
	private async loadZipFromBuffer(fileBuffer: Buffer): Promise<JSZip> {
		console.log('Loading ZIP from buffer');
		const zip = new JSZip();
		return zip.loadAsync(fileBuffer);
	}

	private async validate(fileBuffer: Buffer): Promise<ReferenceValidationResultMap> {
		console.log('Validating ZIP file');
		const AssetValidator = new ProjectAssetValidator();
		const folderNames = new Set<string>();
		const filePathsInFolder = new Set<string>();

		await this.extractFolderNamesAndPaths(fileBuffer, folderNames, filePathsInFolder);
		const validationPromises = Array.from(folderNames).map(async (folderName) => {
			return this.validateFolder(fileBuffer, folderName, AssetValidator, filePathsInFolder, folderNames);
		});

		const validationResults = await Promise.all(validationPromises);
		const isValid = validationResults.every(result => result.isValid);

		// create a map with folderName and it's refMap
		const allRefMaps = new Map<string, Map<string, boolean>>();
		validationResults.forEach((result, index) => {
			const folderName = Array.from(folderNames)[index];
			allRefMaps.set(folderName, result.refMap);
		});

		return { isValid, allRefMaps };
	}

	private async extractFolderNamesAndPaths(fileBuffer: Buffer, folderNames: Set<string>, filePathsInFolder: Set<string>): Promise<void> {
		console.log('Extracting folder names and paths');
		const zipContent = await this.loadZipFromBuffer( fileBuffer );
		for (const fileName in zipContent.files) {
			const folderName = fileName.split(path.sep)[0];
			if (folderName !== 'dependencies') {
				folderNames.add(folderName);
				filePathsInFolder.add(fileName);
			}
		}
		console.log(Array.from(folderNames).join(', '));
	}

	private async validateFolder(
		fileBuffer: Buffer,
		folderName: string,
		AssetValidator: ProjectAssetValidator,
		filePathsInFolder: Set<string>, allFolderNames: Set<string>
	): Promise<ReferenceValidationResult> {

		console.log(`folder: ${folderName}`);
		const assetReferenceValid = await AssetValidator.validateProjectAssetReference(fileBuffer, folderName, allFolderNames);
		const pathReferenceValid = await AssetValidator.validateProjectPathReference(fileBuffer, folderName, filePathsInFolder);
		const minimumAssetsValid = await AssetValidator.validateProjectHasMinimumAssets(fileBuffer);
		const apiSpecVariableValid = await AssetValidator.validateProjectApiSpecVariable(fileBuffer, folderName);
		const yamlValid = await AssetValidator.validateYaml(fileBuffer);

		// return isvalid true if all the validations are true
		// return refMap to consolidate assets later
		return {isValid: assetReferenceValid.isValid && pathReferenceValid && minimumAssetsValid && apiSpecVariableValid && yamlValid,
			refMap: assetReferenceValid.refMap
		};
	}

	private async getFileFromZip(fileBuffer: Buffer, filePath: string): Promise<string | null> {
		console.log(filePath);
		const zipContent = await this.loadZipFromBuffer(fileBuffer);
		const file = zipContent.file( filePath );
		if (file) {
			return file.async('string');
		}
		console.log(filePath);
		return null;
	}

	private async createProjectBuildZip(buffer: Buffer, allRefMaps: Map<string, Map<string, boolean>>): Promise<JSZip> {
		console.log('Creating project build ZIP');
		const buildZip = new JSZip();
		await this.loadZipFromBuffer(buffer);
		const folderNames = new Set<string>();
		const filePathsInFolder = new Set<string>();

		await this.extractFolderNamesAndPaths(buffer, folderNames, filePathsInFolder);

		await this.addConsolidatedYAMLs(buildZip, buffer, folderNames, allRefMaps);
		await this.addReferencedFiles(buildZip, buffer, folderNames);

		return buildZip;
	}

	private async addConsolidatedYAMLs(buildZip: JSZip, buffer: Buffer, folderNames: Set<string>, allRefMaps: Map<string, Map<string, boolean>>): Promise<void> {
		console.log('Adding consolidated YAMLs to build ZIP');
		for (const folderName of folderNames) {
			const consolidatedYaml = await this.createConsolidatedYaml(buffer, folderName, allRefMaps);
			buildZip.file(`${folderName}.yaml`, consolidatedYaml);
		}
	}

	private async addReferencedFiles(buildZip: JSZip, buffer: Buffer, folderNames: Set<string>): Promise<void> {
		console.log('Adding referenced files to build ZIP');
		const AssetValidator = new ProjectAssetValidator();

		for (const folderName of folderNames) {
			const refMap = await AssetValidator.createProjectPathReferenceMap(buffer, folderName);
			const promises = Array.from(refMap.keys()).map(async (key) => {
				const file = await this.getFileFromZip(buffer, path.normalize(`${folderName}/${key}`));
				if (file !== null) {
					buildZip.file(path.normalize(`resources/${folderName}/${key}`), file);
				} else {
					console.log(key);
				}
			});
			await Promise.all(promises);
		}
	}

	private async createConsolidatedYaml(buffer: Buffer, folderName: string, allRefMaps: Map<string, Map<string, boolean>>): Promise<string> {
		
		console.log(folderName);
		const zipContent = await this.loadZipFromBuffer(buffer);
		const visitedAsset = new Set<string>();
		let consolidatedYaml = '';

		try {
			const versionMap = await this.createVersionProcessingMap(buffer);
			consolidatedYaml += await this.processYamlFiles(zipContent, folderName, visitedAsset,versionMap);
			consolidatedYaml += await this.processDependencyFiles(zipContent, visitedAsset,versionMap);
			consolidatedYaml += await this.processDependenciesInOtherFolders(zipContent, visitedAsset, versionMap, folderName, allRefMaps);
		} catch (err) {
			console.log('processing ZIP', `${err}`);
		}

		return consolidatedYaml;
	}

	private async processYamlFiles(zipContent: JSZip, folderName: string, visitedAsset: Set<string>,versionMap:Map<string,boolean>): Promise<string> {
		let consolidatedYaml = '';

		for (const fileName in zipContent.files) {
			const entry = zipContent.files[fileName];
			if (
				!entry.dir &&
				fileName.startsWith(folderName + path.sep) &&
				(fileName.endsWith('.yaml') || fileName.endsWith('.yml'))
			) {
				try {
					const content = await entry.async('string');
					consolidatedYaml += await this.processYamlContent(content, visitedAsset,versionMap);
				} catch (err) {
					console.log('parsing YAML file', `${err}`);
				}
			}
		}

		return consolidatedYaml;
	}

	private async processDependenciesInOtherFolders(
		zipContent: JSZip,
		visitedAsset: Set<string>,
		versionMap:Map<string,boolean>,
		folderName: string,
		allRefMaps: Map<string, Map<string, boolean>>
	): Promise<string> {
		console.log('Processing dependency YAML files');
		let consolidatedYaml = '';

		// retrieve the refMap of the current folderName and process it
		for (const [refFolderName, refMap] of allRefMaps) {
			if (refFolderName === folderName) {

				for (const fileName in zipContent.files) {
					const entry = zipContent.files[fileName];
					if (this.shouldProcessFilesInOtherFolders(entry, fileName, folderName)) {
						try {
							const content = await entry.async('string');
							consolidatedYaml += await this.processYamlContentForOtherFolders(content, visitedAsset,versionMap, refMap);
						} catch (err) {
							console.log('parsing dependency YAML file', `${err}`);
						}
					}
				}

			}
		}
		return consolidatedYaml;
	}

	private async processDependencyFiles(zipContent: JSZip, visitedAsset: Set<string>,versionMap:Map<string,boolean>): Promise<string> {
		console.log('Processing dependency YAML files');
		let consolidatedYaml = '';

		for (const fileName in zipContent.files) {
			const entry = zipContent.files[fileName];
			if (
				!entry.dir &&
				fileName.startsWith('dependencies' + path.sep) &&
				(fileName.endsWith('.yaml') || fileName.endsWith('.yml'))
			) {
				try {
					const content = await entry.async('string');
					consolidatedYaml += await this.processYamlContent(content, visitedAsset,versionMap);
				} catch (err) {
					console.log('parsing dependency YAML file', `${err}`);
				}
			}
		}

		return consolidatedYaml;
	}

	private shouldProcessFile(fileName: string): boolean {
		return  (fileName.endsWith('.yaml') || fileName.endsWith('.yml')) && !fileName.includes('resources');
	}
	
	private shouldProcessFilesInOtherFolders(entry: JSZip.JSZipObject, fileName: string, folderName: string): boolean {
		return !entry.dir &&
			!fileName.startsWith(folderName + path.sep) &&
			(fileName.endsWith('.yaml') || fileName.endsWith('.yml'));
	}

	public async createVersionProcessingMap(buffer: Buffer) {
		const refMap = new Map<string, boolean>();
		try {
			const zipContent = await this.loadZipFromBuffer(buffer);
			for (const fileName in zipContent.files) {
				const entry = zipContent.files[fileName];
				if (this.shouldProcessFile(fileName) && !entry.dir) {
					await this.processFileContent(entry, refMap);
				}
			}
		} catch (err) {
			console.log(`Error loading ZIP: ${err}`);
		}
		return refMap;
	}
	// eslint-disable-next-line @typescript-eslint/no-explicit-any
	private async processFileContent(entry: any, refMap: Map<string, boolean>) {
		try {
			const content = await entry.async('string');
			const yamlContents = this.parseYaml(content);
			this.processYamlContents(yamlContents, refMap);
		} catch (err) {
			console.log(`Error parsing YAML: ${err}`);
		}
	}

	private parseYaml(content: string): YamlContent[] {
		try {
			return yaml.loadAll(content) as YamlContent[];
		} catch (err) {
			console.log(`Error parsing YAML: ${err}`);
			return [];
		}
	}

	private processYamlContents(yamlContents: YamlContent[], refMap: Map<string, boolean>) {
		for (const yamlContent of yamlContents) {
			if (isValidAsset(yamlContent)) {
				const metadata = yamlContent['metadata'] as Metadata_Ref;
				if (typeof metadata.version === 'string') {
					updateMapWithMetadata(yamlContent, refMap);
				}
			}
		}
	}


	private async processYamlContent(content: string, visitedAsset: Set<string>,versionMap:Map<string,boolean>): Promise<string> {
		let yamlResult = '';

		try {
			const yamlContents = yaml.loadAll(content) as YamlContent[];
			for (const yamlContent of yamlContents) {
				if (isValidAsset(yamlContent)) {
					const metadata = yamlContent.metadata as Metadata_Ref;
					const contentString = `${metadata.namespace ? metadata.namespace : ''}:${metadata.name}:${convertNumberToString(metadata.version)}`;
					if (visitedAsset.has(contentString)) {
						console.log(`Skipping already visited asset: ${contentString}`);
						continue;
					}
					yamlContent.metadata.version = convertNumberToString(metadata.version);
					visitedAsset.add(contentString);
					const processedYamlContent= updateRefs(yamlContent,versionMap);
					yamlResult += '---' + '\n' + yaml.dump(processedYamlContent) + '\n';
				}
			}
		} catch (err) {
			console.log('processing YAML content', `${err}`);
		}

		return yamlResult;
	}

	private async processYamlContentForOtherFolders(content: string, visitedAsset: Set<string>,versionMap:Map<string,boolean>, refMap: Map<string, boolean>): Promise<string> {
		let yamlResult = '';

		try {
			const yamlContents = yaml.loadAll(content) as YamlContent[];
			for (const yamlContent of yamlContents) {
				if (isValidAsset(yamlContent)) {
					const metadata = yamlContent.metadata as Metadata_Ref;
					const contentString = `${metadata.namespace ? metadata.namespace : ''}:${metadata.name}:${convertNumberToString(metadata.version)}`;
					// If the current file metadata is present in refMap and the metadata value is set to true, then it needs to be added
					if (refMap.has(contentString) && refMap.get(contentString) === true) {
						if (visitedAsset.has(contentString)) {
							console.log(`Skipping already visited asset: ${contentString}`);
							continue;
						}
						yamlContent.metadata.version = convertNumberToString(metadata.version);
						visitedAsset.add(contentString);
						const processedYamlContent= updateRefs(yamlContent,versionMap);
						yamlResult += '---' + '\n' + yaml.dump(processedYamlContent) + '\n';
					}
				}
			}
		} catch (err) {
			console.log('processing YAML content', `${err}`);
		}

		return yamlResult;
	}

	async processProjectZip(fileBuffer: Buffer): Promise<JSZip | null> {
		console.log('Processing project ZIP');
		const validatedFileBuffer = await this.validate(fileBuffer);
		if (!validatedFileBuffer.isValid) {
			console.log('Project ZIP validation failed');
			return null;
		}
		return this.createProjectBuildZip(fileBuffer, validatedFileBuffer.allRefMaps);
	}

	async extractGatewaysJson(buffer: Buffer): Promise<GatewaysJson> {
		console.log('Extracting gateways.json');
		const zipContent = await this.loadZipFromBuffer(buffer);
		return this.extractGatewaysJsonFromZip(zipContent);
	}

	private async extractGatewaysJsonFromZip(zipContent: JSZip): Promise<GatewaysJson> {
		console.log('Extracting gateways.json from ZIP');
		let gatewaysJsonContent: GatewaysJson = {} as GatewaysJson;

		try {
			const gatewaysJsonFile = this.findGatewaysJsonFile(zipContent);
			if (gatewaysJsonFile) {
				gatewaysJsonContent = await this.parseJsonContent(gatewaysJsonFile);
			} else {
				console.log('gateways.json file not found in ZIP');
			}
		} catch (err) {
			console.log('extracting gateways.json', `${err}`);
		}

		return gatewaysJsonContent;
	}

	private findGatewaysJsonFile(zipContent: JSZip): JSZip.JSZipObject | null {
		console.log('Finding gateways.json file in ZIP');
		for (const fileName in zipContent.files) {
			const entry = zipContent.files[fileName];
			if (!entry.dir && fileName.includes('gateways.json')) {
				return entry;
			}
		}
		return null;
	}

	private async parseJsonContent(file: JSZip.JSZipObject): Promise<GatewaysJson> {
		console.log('Parsing JSON content');
		let jsonContent: GatewaysJson = {} as GatewaysJson;

		try {
			const content = await file.async('string');
			jsonContent = JSON.parse(content);
		} catch (err) {
			console.log('parsing JSON content', `${err}`);
		}

		return jsonContent;
	}
}