@forzalabs/remora
Version:
A powerful CLI tool for seamless data translation.
219 lines (218 loc) • 12.8 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Affirm_1 = __importDefault(require("../../core/Affirm"));
const Algo_1 = __importDefault(require("../../core/Algo"));
const Environment_1 = __importDefault(require("../Environment"));
const ProducerManager_1 = __importDefault(require("../producer/ProducerManager"));
class ConsumerManagerClass {
constructor() {
/**
* Returns the full list of fields that are used by a consumer, while keeping the nested structure of fields.
* If there are *, then replace them with the actual fields found in the underlying producer/consumer
*/
this.getExpandedFields = (consumer) => {
(0, Affirm_1.default)(consumer, 'Invalid consumer');
const availableColumns = this.getAvailableColumns(consumer);
const convertedFields = consumer.fields.map(x => {
var _a;
return ({
cField: x,
finalKey: (_a = x.alias) !== null && _a !== void 0 ? _a : x.key
});
});
const expandedFields = convertedFields.flatMap(x => this.expandField(consumer, x, availableColumns));
return expandedFields;
};
/**
* Return all the available columns (dimensions and measures) to the consumer given its producers
*/
this.getAvailableColumns = (consumer) => {
const availableColumns = consumer.producers.flatMap(cProd => {
var _a, _b;
const producer = Environment_1.default.getProducer(cProd.name);
if (!producer) {
const subConsumer = Environment_1.default.getConsumer(cProd.name);
(0, Affirm_1.default)(subConsumer, `No producer found with name "${cProd.name}"`);
return this.getAvailableColumns(subConsumer);
}
else {
const dims = producer.dimensions.map(x => ({
consumerAlias: null,
consumerKey: null,
nameInProducer: x.name,
aliasInProducer: x.alias,
dimension: x,
owner: cProd.name
}));
const meas = (_b = (_a = producer.measures) === null || _a === void 0 ? void 0 : _a.map(x => ({
consumerAlias: null,
consumerKey: null,
nameInProducer: x.name,
aliasInProducer: x.name,
measure: x,
owner: cProd.name
}))) !== null && _b !== void 0 ? _b : [];
return [...dims, ...meas];
}
});
return availableColumns;
};
/**
* If the field is '*' then replace them with the actual fields found in the underlying producer/consumer
*/
this.expandField = (consumer, field, availableColumns) => {
var _a;
(0, Affirm_1.default)(consumer, 'Invalid consumer');
(0, Affirm_1.default)(field, 'Invalid consumer field');
const expandedFields = [];
if (field.cField.key === '*') {
// If the producers are "union" then they have the same dimensions, meaning that I can use the "*"
if (consumer.producers.length > 1 && consumer.producers.every(x => x.union)) {
const first = consumer.producers[0];
const firstColumns = availableColumns.filter(x => x.owner === first.name);
expandedFields.push(...firstColumns.map(x => ({
cField: {
key: x.nameInProducer,
alias: x.nameInProducer,
from: x.owner
},
finalKey: x.nameInProducer,
dimension: x.dimension,
measure: x.measure
})));
}
else {
const from = (_a = field.cField.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
availableColumns.filter(x => x.owner === from).forEach(col => {
expandedFields.push({
cField: {
key: col.nameInProducer,
alias: col.nameInProducer,
from: col.owner
},
finalKey: col.nameInProducer,
dimension: col.dimension,
measure: col.measure
});
});
}
}
else {
const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
(0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
expandedFields.push(Object.assign(Object.assign({}, field), { dimension: col.dimension, measure: col.measure }));
}
return expandedFields;
};
this.searchFieldInColumns = (field, columns, consumer) => {
var _a, _b;
(0, Affirm_1.default)(field, 'Invalid field');
(0, Affirm_1.default)(columns, 'Invalid columns');
(0, Affirm_1.default)(consumer, 'Invalid consumer');
let column = null;
if (field.from) {
column = columns.find(x => x.owner === field.from && x.nameInProducer === field.key);
}
else if (consumer.producers.length === 1 && !field.from) {
column = columns.find(x => x.nameInProducer === field.key);
}
else if (!field.fixed) {
const matches = columns.filter(x => x.nameInProducer === field.key);
(0, Affirm_1.default)(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map(x => x.name).join(', ')})`);
if (matches.length === 1) {
// Need to check if the producers have "union" if they do, I don't care about this check
const cProd = consumer.producers.find(x => x.name === matches[0].owner);
if (!cProd.union)
(0, Affirm_1.default)(matches.length === 1, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is ambiguos between the fields with same name from the producers: ${matches.map(x => x.owner).join(', ')}`);
}
column = matches[0];
}
if (!column) {
// If the consumer doesn't find the field in the producer but has a default value AND set_default onError
// then instead of failing, create a placeholder column for the producer
if (field.fixed === true && Algo_1.default.hasVal(field.default)) {
column = {
aliasInProducer: field.key,
nameInProducer: (_a = field.alias) !== null && _a !== void 0 ? _a : field.key,
consumerAlias: (_b = field.alias) !== null && _b !== void 0 ? _b : field.key,
consumerKey: field.key,
owner: field.from,
dimension: { name: field.key, type: typeof field.default }
};
}
}
return column;
};
this.getSource = (consumer) => {
const producers = consumer.producers.map(x => Environment_1.default.getProducer(x.name));
(0, Affirm_1.default)(producers.length > 0, 'No producers found');
(0, Affirm_1.default)(producers.every(x => x), `Invalid producer found in consumer "${consumer.name}"`);
const sources = producers.map(x => Environment_1.default.getSource(x.source));
(0, Affirm_1.default)(sources.length > 0, 'No sources found');
(0, Affirm_1.default)(sources.every(x => x), `Invalid source found in consumer "${consumer.name}"`);
// For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
const uniqEngines = Algo_1.default.uniqBy(sources, 'engine');
(0, Affirm_1.default)(uniqEngines.length === 1, `Sources with different engines were used in the consumer "${consumer.name}" (${uniqEngines.join(', ')})`);
// For now we also only support consumers that have producers ALL having the same exact source
const uniqNames = Algo_1.default.uniqBy(sources, 'name');
(0, Affirm_1.default)(uniqNames.length === 1, `Producers with different sources were used in the consumer "${consumer.name}" (${uniqNames.join(', ')})`);
return [sources[0], producers[0]];
};
this.compile = (consumer) => {
var _a, _b;
(0, Affirm_1.default)(consumer, `Invalid consumer`);
const availableColumns = this.getAvailableColumns(consumer);
const selectedColumns = [];
const flat = consumer.fields;
for (let i = 0; i < flat.length; i++) {
const field = flat[i];
// TODO: replace with the new funcitons in the consumermanager to reduce diplicate code
if (field.key === '*') {
const from = (_a = field.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
availableColumns.filter(x => x.owner === from).forEach(col => {
col.consumerKey = col.nameInProducer;
col.consumerAlias = col.nameInProducer;
selectedColumns.push(col);
});
}
else {
const col = ConsumerManager.searchFieldInColumns(field, availableColumns, consumer);
(0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
col.consumerKey = field.key;
col.consumerAlias = (_b = field.alias) !== null && _b !== void 0 ? _b : field.key;
selectedColumns.push(col);
}
}
const columnsWithNoAlias = selectedColumns.filter(x => !x.consumerAlias || !x.consumerKey);
(0, Affirm_1.default)(columnsWithNoAlias.length === 0, `Consumer "${consumer.name}" compilation error: some selected fields don't have a correct alias or key (${columnsWithNoAlias.map(x => x.nameInProducer).join(', ')})`);
return selectedColumns;
};
this.getOutputShape = (consumer) => {
(0, Affirm_1.default)(consumer, `Invalid consumer`);
const compiled = this.compile(consumer);
const outDimensions = compiled.map(x => {
var _a, _b, _c, _d, _e, _f, _g;
return ({
name: (_a = x.consumerAlias) !== null && _a !== void 0 ? _a : x.consumerKey,
type: (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type,
classification: (_c = x.dimension) === null || _c === void 0 ? void 0 : _c.classification,
description: (_e = (_d = x.dimension) === null || _d === void 0 ? void 0 : _d.description) !== null && _e !== void 0 ? _e : (_f = x.measure) === null || _f === void 0 ? void 0 : _f.description,
mask: ProducerManager_1.default.getMask(x.dimension),
pk: (_g = x.dimension) === null || _g === void 0 ? void 0 : _g.pk
});
});
return {
_version: consumer._version,
name: consumer.name,
description: consumer.description,
metadata: consumer.metadata,
dimensions: outDimensions
};
};
}
}
const ConsumerManager = new ConsumerManagerClass();
exports.default = ConsumerManager;