@forzalabs/remora
Version:
A powerful CLI tool for seamless data translation.
156 lines (155 loc) • 8.67 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Affirm_1 = __importDefault(require("../../core/Affirm"));
const Algo_1 = __importDefault(require("../../core/Algo"));
const Environment_1 = __importDefault(require("../Environment"));
class ConsumerManagerClass {
constructor() {
this.getConsumerFlatFields = (consumer) => {
(0, Affirm_1.default)(consumer, 'Invalid consumer');
return this.getFlatFields(consumer.fields);
};
this.getFlatFields = (list) => {
let result = [...list];
for (let i = 0; i < list.length; i++) {
const field = list[i];
if (field.grouping && field.grouping.subFields && field.grouping.subFields.length > 0)
result = [...result, ...this.getFlatFields(field.grouping.subFields)];
}
return result;
};
/**
* Returns the full list of fields that are used by a consumer, while keeping the nested structure of fields.
* If there are *, then replace them with the actual fields found in the underlying producer/consumer
*/
this.getExpandedFields = (consumer) => {
(0, Affirm_1.default)(consumer, 'Invalid consumer');
const availableColumns = this.getAvailableColumns(consumer);
const convertedFields = this.convertFields(consumer.fields);
const expandedFields = convertedFields.flatMap(x => this.expandField(consumer, x, availableColumns));
return expandedFields;
};
this.convertFields = (fieldsToConvert) => {
(0, Affirm_1.default)(fieldsToConvert, 'Invalid fields');
const convertedFields = fieldsToConvert.map(x => ({
cField: x
}));
return convertedFields;
};
/**
* Return all the available columns (dimensions and measures) to the consumer given its producers
*/
this.getAvailableColumns = (consumer) => {
const availableColumns = consumer.producers.flatMap(cProd => {
var _a, _b;
const producer = Environment_1.default.getProducer(cProd.name);
if (!producer) {
const subConsumer = Environment_1.default.getConsumer(cProd.name);
(0, Affirm_1.default)(subConsumer, `No producer found with name "${cProd.name}"`);
return this.getAvailableColumns(subConsumer);
}
else {
const dims = producer.dimensions.map(x => ({
consumerAlias: null,
consumerKey: null,
nameInProducer: x.name,
aliasInProducer: x.alias,
dimension: x,
owner: cProd.name
}));
const meas = (_b = (_a = producer.measures) === null || _a === void 0 ? void 0 : _a.map(x => ({
consumerAlias: null,
consumerKey: null,
nameInProducer: x.name,
aliasInProducer: x.name,
measure: x,
owner: cProd.name
}))) !== null && _b !== void 0 ? _b : [];
return [...dims, ...meas];
}
});
return availableColumns;
};
/**
* If the field is '*' then replace them with the actual fields found in the underlying producer/consumer
*/
this.expandField = (consumer, field, availableColumns) => {
var _a;
(0, Affirm_1.default)(consumer, 'Invalid consumer');
(0, Affirm_1.default)(field, 'Invalid consumer field');
const expandedFields = [];
if (field.cField.key === '*') {
const from = (_a = field.cField.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
availableColumns.filter(x => x.owner === from).forEach(col => {
expandedFields.push({
cField: {
key: col.nameInProducer,
alias: col.nameInProducer,
from: col.owner
},
dimension: col.dimension,
measure: col.measure
});
});
}
else if (field.cField.grouping) {
expandedFields.push({
cField: {
key: field.cField.key,
alias: field.cField.alias,
from: field.cField.from,
grouping: {
groupingKey: field.cField.grouping.groupingKey,
subFields: field.cField.grouping.subFields.flatMap(x => this.expandField(consumer, { cField: x }, availableColumns)).map(x => x.cField)
}
},
dimension: field.dimension,
measure: field.measure
});
}
else {
const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
(0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
expandedFields.push(Object.assign(Object.assign({}, field), { dimension: col.dimension, measure: col.measure }));
}
return expandedFields;
};
this.searchFieldInColumns = (field, columns, consumer) => {
(0, Affirm_1.default)(field, 'Invalid field');
(0, Affirm_1.default)(columns, 'Invalid columns');
(0, Affirm_1.default)(consumer, 'Invalid consumer');
if (field.from) {
return columns.find(x => x.owner === field.from && x.nameInProducer === field.key);
}
else if (consumer.producers.length === 1 && !field.from) {
return columns.find(x => x.nameInProducer === field.key);
}
else {
const matches = columns.filter(x => x.nameInProducer === field.key);
(0, Affirm_1.default)(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map(x => x.name).join(', ')})`);
(0, Affirm_1.default)(matches.length === 1, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is ambiguos between the fields with same name from the producers: ${matches.map(x => x.owner).join(', ')}`);
return matches[0];
}
};
this.getSource = (consumer) => {
const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
(0, Affirm_1.default)(producers.length > 0, 'No producers found');
(0, Affirm_1.default)(producers.every(x => x), `Invalid producer found in consumer "${consumer.name}"`);
const sources = producers.map(x => Environment_1.default.getSource(x.source));
(0, Affirm_1.default)(sources.length > 0, 'No sources found');
(0, Affirm_1.default)(sources.every(x => x), `Invalid source found in consumer "${consumer.name}"`);
// For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
(0, Affirm_1.default)(uniqEngines.length === 1, `Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
// For now we also only support consumers that have producers ALL having the same exact source
const uniqNames = Algo_1.default.uniqBy(sources, 'name');
(0, Affirm_1.default)(uniqNames.length === 1, `Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
return [sources[0], producers[0]];
};
}
}
const ConsumerManager = new ConsumerManagerClass();
exports.default = ConsumerManager;