@apistudio/apim-cli/node_modules/@apic/studio-build/src/build-asset.ts

CLI for API Management Products

/**
* Copyright Super iPaaS Integration LLC, an IBM Company 2024
*/
import yaml from 'js-yaml';
import { Api_Spec_Ref,  Metadata_Ref, YamlContent } from './model/interface.js';
import { GatewaysJson } from '@apic/studio-shared';
import { AppConstants } from './constants/app.constants.js';
import {
	addErrorToResponse,
	convertNumberToString,
	createAssetReferenceMap,
	createPathReferenceMap,
	isValidAsset,
	updatePathRefMap,
	validateMinAssets,
	validateYamlFiles
} from './utils.js';
import JSZip from 'jszip';
// import { // } from './service/log-wrapper.js';

export class ZipProcessor {
	buffer: Buffer;
	constructor(buffer: Buffer) {
		this.buffer = buffer;
	}

	async validate(): Promise<boolean> {
		//.logInfo('0003', 'Starting validation process for ZIP buffer'); // Starting validation process
		if (!await this.validateReferences()) {
			//.logDebug('0360', 'references');
			return false;
		}
		if (!await this.validatePaths()) {
			//.logDebug('0360', 'paths');
			return false;
		}
		if (!await this.validateZip()) {
			//.logDebug('0360', 'zip');
			return false;
		}
		if (!await this.validateApiFiles()) {
			//.logDebug('0360', 'API files');
			return false;
		}
		if (!await this.validateYamlStructure()) {
			//.logDebug('0360', AppConstants.YAMLStructure);
			return false;
		}
		return this.validateYamlFiles();
	}

	private async validateYamlStructure(): Promise<boolean> {
		//.logDebug('0361', AppConstants.YAMLStructure);
		if (!await validateYamlFiles(this.buffer)) {
			//.logError('0255', 'Invalid YAML files detected in buffer');
			return false;
		}
		//.logDebug('0362', AppConstants.YAMLStructure);
		return true;
	}

	private async validateApiFiles(): Promise<boolean> {
		//.logDebug('0361', 'API files in zip');
		const zip = new JSZip();
		try {
			const zipContent = await zip.loadAsync(this.buffer);
			//.logDebug('0362', 'API files in zip');
			return await this.validateZipContent(zipContent);
		} catch (err) {
			//.logError('0013', 'loading ZIP', `${err}`);
			return false;
		}
	}

	private async validateZipContent(zipContent: JSZip): Promise<boolean> {
		//.logDebug('0361', 'ZIP content');
		for (const fileName in zipContent.files) {
			const entry = zipContent.files[fileName];
			if (this.isYamlFile(entry, fileName)) {
				//.logDebug('0357', AppConstants.YAMLContent, fileName);
				const content = await entry.async('string');
				const isValid = await this.checkYamlContent(content, fileName);
				if (!isValid) {
					//.logError('0359', AppConstants.YAMLContent, fileName);
					return false;
				}
			}
		}
		//.logDebug('0362', 'ZIP content');
		return true;
	}

	private async checkYamlContent(content: string, fileName: string): Promise<boolean> {
		try {
			const yamlContents = yaml.loadAll(content) as YamlContent[];
			for (const yamlContent of yamlContents) {
				if (yamlContent.kind?.toLowerCase() === 'api') {
					const specObj = JSON.stringify(yamlContent.spec);
					const apiSpec = yaml.load(specObj) as Api_Spec_Ref;
					if (this.isInvalidApiSpec(apiSpec)) {
						addErrorToResponse(AppConstants.VALIDATION_ERROR_CODE, fileName, `Validation failed for api-spec field in file ${fileName}`);
						//.logError('0359', 'API spec', fileName);
						return false;
					}
				}
			}
		} catch (err) {
			//.logError('0013', `parsing YAML content in ${fileName}`, `${err}`);
			return false;
		}
		//.logDebug('0358', AppConstants.YAMLContent, fileName);
		return true;
	}

	private isInvalidApiSpec(apiSpec: Api_Spec_Ref): boolean {
		//.logDebug('0361', 'API Specification');
		const apiSpecField = AppConstants.apiSpec;
		const apiSpecPathLength = apiSpec[apiSpecField]?.$path?.length ?? 0;
		return !apiSpec ||
			!apiSpec[apiSpecField] ||
			!apiSpec[apiSpecField].$path ||
			(apiSpecPathLength <= 0);
	}

	private async validateReferences(): Promise<boolean> {
		//.logDebug('0361', 'references');
		const refMap = await createAssetReferenceMap(this.buffer);
		const allRefsValid = Array.from(refMap.entries()).every(([key, value]) => {
			if (!value) {
				addErrorToResponse(AppConstants.VALIDATION_ERROR_CODE, key, `Validation failed for Reference ${key}`);
				//.logError('0359', 'reference', key);
			}
			return value;
		});
		if (!allRefsValid) {
			//.logError('0255', 'Some references are not valid');
			return false;
		}
		return true;
	}

	private async validatePaths(): Promise<boolean> {
		//.logDebug('0361', 'paths');
		const pathMap = await createPathReferenceMap(this.buffer);
		await updatePathRefMap(this.buffer, pathMap);
		const allPathsValid = Array.from(pathMap.entries()).every(([key, value]) => {
			if (!value) {
				addErrorToResponse(AppConstants.VALIDATION_ERROR_CODE, key, `Validation failed for Reference ${key}`);
			}
			return value;
		});
		if (!allPathsValid) {
			//.logError('0255', 'Some references are not valid');
			return false;
		}
		//.logDebug('0362', 'paths');
		return true;
	}

	private async validateZip(): Promise<boolean> {
		//.logDebug('0361', 'ZIP');
		if (!await validateMinAssets(this.buffer)) {
			//.logError('0255', 'Zip does not contain minimum assets');
			return false;
		}
		//.logDebug('0362', 'ZIP');
		return true;
	}

	private async validateYamlFiles(): Promise<boolean> {
		//.logDebug('0361', 'YAML files in zip');
		const zip = new JSZip();
		const zipContent = await zip.loadAsync(this.buffer);

		for (const fileName in zipContent.files) {
			const entry = zipContent.files[fileName];
			if (this.isYamlFile(entry, fileName)) {
				if (!(await this.validateYamlFileContent(entry, fileName))) {
					return false;
				}
			}
		}
		//.logDebug('0362', 'YAML files in zip');
		return true;
	}

	private async validateYamlFileContent(entry: JSZip.JSZipObject, fileName: string): Promise<boolean> {
		//.logDebug('0357', AppConstants.YAMLContent, fileName);
		try {
			const content = await entry.async('string');
			const yamlContents = yaml.loadAll(content) as YamlContent[];
			for (const yamlContent of yamlContents) {
				if (!isValidAsset(yamlContent)) {
					//.logError('0359', 'asset', fileName);
					addErrorToResponse(AppConstants.VALIDATION_ERROR_CODE, 'YAML_FILE', `Invalid YAML in file ${fileName}`);
					return false;
				}
			}
		} catch (err) {
			//.logError('0013', `parsing YAML in ${fileName}`, `${err}`);
			return false;
		}
		//.logDebug('0358', AppConstants.YAMLContent, fileName);
		return true;
	}

	async processZip(): Promise<{ buildZip: JSZip } | null> {
		//.logInfo('0003', 'Creating build ZIP from buffer');
		if (!await this.validate()) {
			//.logDebug('0003', 'ZIP validation failed during build process');
			return null;
		}
		//.logDebug('0003', 'Validation passed, extracting kind groups');
		const kindGroups = await this.extractKindGroups(this.buffer);
		//.logDebug('0003', 'Extracted kind groups, creating build ZIP');
		const buildZip = await this.createBuildZip(this.buffer, kindGroups);
		//.logInfo('0003', 'Build ZIP creation completed');
		return { buildZip };
	}

	async extractGatewaysJson(buffer: Buffer): Promise<GatewaysJson> {
		//.logDebug('0003', 'Extracting gateways.json from ZIP');
		const zip = new JSZip();
		let gatewaysJsonContent: GatewaysJson = {} as GatewaysJson;
		try {
			const zipContent = await zip.loadAsync(buffer);
			for (const fileName in zipContent.files) {
				const entry = zipContent.files[fileName];
				if (!entry.dir && fileName.includes('gateways.json')) {
					try {
						const content = await entry.async('string');
						gatewaysJsonContent = JSON.parse(content);
					} catch (err) {
						//.logError('0013', 'parsing gateways.json', `${err}`);
					}
				}
			}
		} catch (err) {
			//.logError('0013', 'loading ZIP', `${err}`);
		}
		return gatewaysJsonContent;
	}

	private async extractKindGroups(buffer: Buffer): Promise<{ [key: string]: string[] }> {
		const zip = new JSZip();
		const kindGroups: { [key: string]: string[] } = {};
		const kindGroupsSet: Set<string> = new Set<string>();
		try {
			const zipContent = await zip.loadAsync(buffer);
			await this.processZipContent(zipContent, kindGroups, kindGroupsSet);
		} catch (err) {
			//.logError('0013', 'extracting kind groups', `${err}`);
		}
		return kindGroups;
	}

	private async processZipContent(zipContent: JSZip, kindGroups: { [key: string]: string[] }, kindGroupsSet: Set<string>): Promise<void> {
		for (const fileName in zipContent.files) {
			const entry = zipContent.files[fileName];
			if (this.isYamlFile(entry, fileName)) {
				//.logDebug('0353', fileName);
				await this.processYamlEntry(entry, fileName, kindGroups, kindGroupsSet);
			}
		}
	}

	private isYamlFile(entry: JSZip.JSZipObject, fileName: string): boolean {
		return !entry.dir && (fileName.endsWith('.yaml') || fileName.endsWith('.yml')) && !fileName.startsWith('resources');
	}

	private async processYamlEntry(entry: JSZip.JSZipObject, fileName: string, kindGroups: { [key: string]: string[] }, kindGroupsSet: Set<string>): Promise<void> {
		try {
			const content = await entry.async('string');
			const yamlContents = yaml.loadAll(content) as YamlContent[];
			for (const yamlContent of yamlContents) {
				if (yamlContent && yamlContent.kind) {
					await this.processYamlContent(yamlContent, kindGroups, kindGroupsSet);
				}
			}
		} catch (err) {
			//.logError('0013', 'parsing YAML', `${err}`);
		}
	}

	private async processYamlContent(yamlContent: YamlContent, kindGroups: { [key: string]: string[] }, kindGroupsSet: Set<string>): Promise<void> {
		const kind = yamlContent.kind;
		if (!kindGroups[kind]) {
			kindGroups[kind] = [];
		}
		const metadata = yamlContent.metadata as Metadata_Ref;
		const contentString = `${metadata.namespace?metadata.namespace:''}:${metadata.name}:${convertNumberToString(metadata.version)}`;
		if (!kindGroupsSet.has(contentString)) {
			yamlContent.metadata.version = convertNumberToString(metadata.version);
			const yamlString = yaml.dump(yamlContent);
			const contentWithYaml = await this.zipYamlContent(yamlString);
			kindGroups[kind].push(contentWithYaml);
			kindGroupsSet.add(contentString);
		}
	}

	private async zipYamlContent(yamlString: string): Promise<string> {
		//.logDebug('0003', 'Zipping YAML content');
		const tempZip = new JSZip();
		tempZip.file('file.yaml', yamlString);
		const tempZipBuffer = await tempZip.generateAsync({ type: 'nodebuffer' });
		const zipWithYaml = new JSZip();
		const yamlZipContent = await zipWithYaml.loadAsync(tempZipBuffer);
		if (yamlZipContent.files) {
			return yamlZipContent.files['file.yaml'].async('string');
		}
		//.logError('0004', 'Failed to zip YAML content');
		throw new Error('Failed to zip YAML content');
	}

	private async createBuildZip(buffer: Buffer, kindGroups: { [key: string]: string[] }): Promise<JSZip> {
		const jszip = new JSZip();
		const zip = await JSZip.loadAsync(buffer);
		const resourcesFolder = jszip.folder('resources');
		for (const fileName in zip.files) {
			const entry = zip.files[fileName];
			if (!entry.dir) {
				const content = await entry.async('nodebuffer');
				if (fileName.includes('resources/')) {
					//.logDebug('0003', `Adding file to resources folder: ${fileName}`);
					resourcesFolder?.file(fileName.replace('resources/', ''), content);
				}
			}
		}
		const allYamlContent = this.combineYamlContent(kindGroups);
		jszip.file('gw_build.yaml', Buffer.from(allYamlContent.join('\n---\n') + '\n', 'utf-8'));
		//.logDebug('0003', 'Build ZIP creation completed');
		return jszip;
	}

	private combineYamlContent(kindGroups: { [key: string]: string[] }): string[] {
		//.logDebug('0003', 'Combining YAML content from kind groups');
		const allYamlContent: string[] = [];
		for (const kind in kindGroups) {
			const combinedYaml = kindGroups[kind].join('\n---\n');
			allYamlContent.push(combinedYaml);
		}
		//.logDebug('0003', 'YAML content combined');
		return allYamlContent;
	}
}