@event-inc/pipelines/src/apis/utilities.ts

Event Inc is a fully managed event bus lets you send and receive data across mission-critical cloud apps, databases and warehouses.

import {
	Extractor,
	Settings,
	Transformation,
	PipelineMiddleware,
	Pipeline,
} from '@event-inc/types';
import {
	BError,
	handleBuildableError,
	isAlphanumericUnderscore,
	isValidUrl,
} from '@event-inc/utils';

export const getTransformers = (transformation: Transformation) => {
	return {
		_type: 'transformer',
		language: transformation.type,
		code: String(transformation.func),
	};
};

const hasDuplicate = (arr: Extractor[]) => {
	const values = arr.map((extractor) => extractor.label);
	return new Set(values).size !== values.length;
};

export const getExtractors = (extractors: Extractor[]) => {
	if (extractors?.length > 0 && hasDuplicate(extractors)) {
		handleBuildableError(
			BError(
				false,
				'extractor_4000',
				'Extractors cannot have duplicate labels',
				'client-middleware',
				false
			)
		);
	}
	return (
		extractors?.map((extractor) => {
			if (extractor.type === 'http') {
				if (!isValidUrl(extractor.config.url)) {
					handleBuildableError(
						BError(
							false,
							'extractor_4000',
							'Extractor url is invalid',
							'client-middleware',
							false
						)
					);
				}

				if (!isAlphanumericUnderscore(extractor.label)) {
					handleBuildableError(
						BError(
							false,
							'extractor_4000',
							'The label for the extractor can only contain alphanumeric characters and underscores',
							'client-middleware',
							false
						)
					);
				}

				return {
					key: extractor.label,
					_type: 'extractor::http',
					url: extractor.config.url,
					headers: extractor.config.headers
						? JSON.stringify(extractor.config.headers)
						: '{}',
					method: extractor.config.method,
					data: extractor.config.data
						? JSON.stringify(extractor.config.data)
						: '{}',
					...getConfig(extractor.settings),
				};
			}
		}) ?? []
	);
};

export const getMiddleware = (
	extractors: Extractor[],
	transformation: Transformation,
	middleware?: PipelineMiddleware[]
) => {
	let finalMiddleware = middleware ? [...middleware] : [];
	if (extractors) {
		const extractorList = getExtractors(extractors);
		const middlewareWithoutExtractors = finalMiddleware?.filter(
			(middleware) => middleware._type !== 'extractor::http'
		);
		finalMiddleware = [...middlewareWithoutExtractors, ...extractorList];
	}
	if (transformation) {
		const middlewareWithoutTransformation = finalMiddleware?.filter(
			(middleware) => middleware._type !== 'transformer'
		);
		finalMiddleware = [
			...middlewareWithoutTransformation,
			getTransformers(transformation),
		];
	}
	return finalMiddleware;
};

export const getConfig = (settings: Settings, pipeline?: Pipeline) => {
	const config = {
		startToCloseTimeout: settings?.startToCloseTimeout || pipeline?.config?.startToCloseTimeout || '1 minute',
		policies: {
			retry: {
				maximumAttempts: settings?.retry?.maximumAttempts || pipeline?.config?.policies?.retry?.maximumAttempts || 1,
				initialInterval: '1 second',
			},
		},
	};

	return config;
};