@forzalabs/remora/engines/validation/Validator.js

A powerful CLI tool for seamless data translation.

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Affirm_1 = __importDefault(require("../../core/Affirm"));
const Algo_1 = __importDefault(require("../../core/Algo"));
const ConsumerManager_1 = __importDefault(require("../consumer/ConsumerManager"));
const Environment_1 = __importDefault(require("../Environment"));
const ExecutionPlanner_1 = __importDefault(require("../execution/ExecutionPlanner"));
class ValidatorClass {
    constructor() {
        this.validateSources = (sources) => {
            (0, Affirm_1.default)(sources, 'Invalid sources');
            const errors = [];
            try {
                const dupes = Algo_1.default.duplicatesObject(sources, 'name');
                if (dupes.length > 0)
                    errors.push(`Duplicate name(s) found in sources: "${dupes.map(x => x.name).join(', ')}"`);
                for (let i = 0; i < sources.length; i++) {
                    const source = sources[i];
                    if (source.engine === 'local' && !source.authentication.path)
                        errors.push(`For source ${source.name}, the path has not been configured`);
                }
            }
            catch (e) {
                if (errors.length === 0)
                    errors.push(`There was an error in the validation Sources. (error: ${e})`);
            }
            return errors;
        };
        this.validateProducers = (producers) => {
            (0, Affirm_1.default)(producers, 'Invalid producers');
            const errors = [];
            try {
                const dupes = Algo_1.default.duplicatesObject(producers, 'name');
                if (dupes.length > 0)
                    errors.push(`Duplicate name(s) found in producers: "${dupes.map(x => x.name).join(', ')}"`);
            }
            catch (e) {
                if (errors.length === 0)
                    errors.push(`There was an error in the validation Producers. (error: ${e})`);
            }
            return errors;
        };
        this.validateProducer = (producer) => {
            (0, Affirm_1.default)(producer, 'Invalid producer');
            const errors = [];
            try {
                if (!producer.source || producer.source.length === 0)
                    errors.push(`Missing parameter "source" in producer`);
                if (producer.dimensions.some(x => x.name.includes('{') || x.name.includes('[')))
                    errors.push(`Invalid dimension name found in producer "${producer.name}": can't use characters "{" or "[" in dimension names`);
            }
            catch (e) {
                if (errors.length === 0)
                    errors.push(`There was an error in the validation Producer. (error: ${e})`);
            }
            return errors;
        };
        this.validateConsumers = (consumers) => {
            (0, Affirm_1.default)(consumers, 'Invalid consumers');
            const errors = [];
            try {
                const dupes = Algo_1.default.duplicatesObject(consumers, 'name');
                if (dupes.length > 0)
                    errors.push(`Duplicate name(s) found in consumers: "${dupes.map(x => x.name).join(', ')}"`);
            }
            catch (e) {
                if (errors.length === 0)
                    errors.push(`There was an error in the validation Consumers. (error: ${e})`);
            }
            return errors;
        };
        this.validateConsumer = (consumer) => {
            (0, Affirm_1.default)(consumer, 'Invalid consumer');
            const errors = [];
            try {
                // TODO: check that a consumer doesn't consume hitself
                const allFieldsWithNoFrom = consumer.fields.filter(x => x.key === '*' && !x.from);
                if (allFieldsWithNoFrom.length > 0 && consumer.producers.length > 1)
                    errors.push(`Field with key "*" was used without specifying the "from" producer and multiple producers were found.`);
                if (consumer.fields.some(x => x.key === '*' && x.grouping))
                    errors.push(`Field with key "*" can't be used for "grouping". Either remove the grouping or change the key.`);
                // Validation on producers
                if (consumer.producers.length === 0)
                    errors.push(`Consumer must have at least 1 producer.`);
                const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
                if (producers.length === 0)
                    errors.push('No producers found');
                if (producers.some(x => !x))
                    errors.push(`Invalid producer found in consumer "${consumer.name}"`);
                // Validation on sources
                const sources = producers.map(x => Environment_1.default.getSource(x.source));
                if (sources.length === 0)
                    errors.push('No sources found');
                if (sources.some(x => !x))
                    errors.push(`Invalid source found in consumer "${consumer.name}"`);
                // For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
                const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
                if (uniqEngines.length !== 1)
                    errors.push(`Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
                // For now we also only support consumers that have producers ALL having the same exact source
                const uniqNames = Algo_1.default.uniqBy(sources, 'name');
                if (uniqNames.length !== 1)
                    errors.push(`Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
                if (consumer.filters && consumer.filters.length > 0) {
                    if (consumer.filters.some(x => x.sql && x.rule))
                        errors.push(`A single consumer can't have both filters based on SQL and filters based on rules.`);
                    const [source] = ConsumerManager_1.default.getSource(consumer);
                    const engineClass = ExecutionPlanner_1.default.getEngineClass(source.engine);
                    if (engineClass === 'file' && consumer.filters.some(x => x.sql))
                        errors.push(`Filters based on SQL are only valid for SQL based sources. (source: ${source.name})`);
                    if (engineClass === 'sql' && consumer.filters.some(x => x.rule))
                        errors.push(`Filters based on rules are only valid for non-SQL based sources. (source: ${source.name})`);
                }
                // Validation on fields
                const validateGroupingLevels = (fields, level = 0) => {
                    let errors = [];
                    const groupingFields = fields.filter(x => x.grouping);
                    if (groupingFields.length > 1)
                        errors.push(`There can't be 2 fields with grouping defined at the same level (${groupingFields.map(x => x.key).join(', ')}). Level: ${level}`);
                    groupingFields.forEach(field => {
                        if (field.grouping)
                            errors = [...errors, ...validateGroupingLevels(field.grouping.subFields, level + 1)];
                    });
                    return errors;
                };
                const validateTransformations = (fields) => {
                    const errors = [];
                    const trxsFields = fields.filter(x => x.transform);
                    for (const field of trxsFields) {
                        const trxToValidate = [];
                        if (Array.isArray(field.transform))
                            trxToValidate.push(...field.transform);
                        else
                            trxToValidate.push(field.transform);
                        for (const trans of trxToValidate) {
                            const trxKeys = Object.keys(trans);
                            if (trxKeys.length !== 1)
                                errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}`);
                        }
                    }
                    return errors;
                };
                errors.push(...validateGroupingLevels(consumer.fields));
                errors.push(...validateTransformations(consumer.fields));
                // Validation outputs
                const duplicatesOutputs = Algo_1.default.duplicatesObject(consumer.outputs, 'format');
                if (duplicatesOutputs.length > 0) {
                    const duplicatesTypes = Algo_1.default.uniq(duplicatesOutputs.map(x => x.format));
                    errors.push(`There are outputs with the same type. (duplicates type: ${duplicatesTypes.join(' and ')})`);
                }
                for (const output of consumer.outputs) {
                    const format = output.format.toUpperCase();
                    if (format === 'SQL' && output.accellerated && output.direct)
                        errors.push(`An output SQL cannot be both direct and accelerated (output: ${format})`);
                    if ((format === 'CSV' || format === 'JSON' || format === 'PARQUET')) {
                        if (!output.exportDestination)
                            errors.push(`A static file output must have an export destination set (${format})`);
                        else if (!Environment_1.default.getSource(output.exportDestination))
                            errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
                    }
                }
            }
            catch (e) {
                if (errors.length === 0)
                    errors.push(`There was an error in the validation Consumer. (error: ${e})`);
            }
            return errors;
        };
    }
}
const Validator = new ValidatorClass();
exports.default = Validator;