@forzalabs/remora
Version:
A powerful CLI tool for seamless data translation.
168 lines (167 loc) • 9.97 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Affirm_1 = __importDefault(require("../../core/Affirm"));
const Algo_1 = __importDefault(require("../../core/Algo"));
const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
const Environment_1 = __importDefault(require("../Environment"));
const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
const TypeCaster_1 = __importDefault(require("../transform/TypeCaster"));
const ConsumerManager_1 = __importDefault(require("./ConsumerManager"));
class PostProcessorClass {
constructor() {
/**
* Maps an array of objects and projects it to another array of objects but with different shape:
* - grouping (creates nested fields)
* - type casting
* - default field values
*/
this.doProjection = (consumer, data) => {
(0, Affirm_1.default)(consumer, 'Invalid consumer');
(0, Affirm_1.default)(data, 'Invalid data');
const fields = ConsumerManager_1.default.getExpandedFields(consumer);
const subProjection = (allFields, items) => {
(0, Affirm_1.default)(allFields, 'Invalid fields');
(0, Affirm_1.default)(Array.isArray(allFields), 'Invalid fields type, must be an array');
(0, Affirm_1.default)(items, 'Invalid items');
(0, Affirm_1.default)(Array.isArray(items), 'Invalid items type, must be an array');
if (allFields.some(x => x.cField.grouping)) {
(0, Affirm_1.default)(allFields.filter(x => x.cField.grouping).length === 1, `Mutliple fields at the same level were found having "grouping" attributes.`);
const groupingRule = allFields.find(x => x.cField.grouping).cField.grouping;
const groups = Algo_1.default.groupBy(items, groupingRule.groupingKey);
const projections = [];
groups.forEach(gItems => {
var _a, _b, _c;
const projected = {};
const first = gItems[0];
for (const field of allFields) {
const { key, alias, grouping } = field.cField;
const fieldKey = alias !== null && alias !== void 0 ? alias : key;
const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
if (!field.cField.grouping) {
projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, first[fieldKey], fieldType);
}
else {
const { subFields } = grouping;
const convertedSubFields = ConsumerManager_1.default.convertFields(subFields);
projected[fieldKey] = subProjection(convertedSubFields, gItems);
}
}
projections.push(projected);
});
return projections;
}
else {
return items.map(x => {
var _a, _b, _c;
const projected = {};
for (const field of allFields) {
const { key, alias } = field.cField;
const fieldKey = alias !== null && alias !== void 0 ? alias : key;
const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
const fieldValue = this._getFieldValue(x, field);
if (Algo_1.default.hasVal(maskType))
projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, fieldValue, fieldType);
else
projected[fieldKey] = TypeCaster_1.default.cast(fieldValue, fieldType);
}
return projected;
});
}
};
return subProjection(fields, data);
};
/**
* Gets an array of objects (with potentially nested fields) and unpacks them to an array of objects with no nested fields
* If some nested keys are lists, then a logic similar to a SQL JOIN is used and rows are duplicated
*/
this.unpack = (data, producer) => {
(0, Affirm_1.default)(data, 'Invalid data');
(0, Affirm_1.default)(Array.isArray(data), 'Invalid data type, must be an array');
(0, Affirm_1.default)(producer, 'Invalid producer');
const source = Environment_1.default.getSource(producer.source);
(0, Affirm_1.default)(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
const columns = FileCompiler_1.default.compileProducer(producer, source);
(0, Affirm_1.default)(columns, `Invalid columns from compilation for producer "${producer.name}"`);
const unpackDimension = (item, dimension) => {
var _a, _b, _c;
const { nameInProducer, aliasInProducer } = dimension;
const maskType = (_a = dimension.dimension.mask) !== null && _a !== void 0 ? _a : undefined;
const fieldType = (_c = (_b = dimension.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
const keys = aliasInProducer.split('.');
let prevValue = item;
for (const key of keys) {
if (key.includes('{')) {
const cleanedKey = key.replace('{', '').replace('}', '');
if (Array.isArray(prevValue))
prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue.map((x) => x[cleanedKey]);
else
prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue[cleanedKey];
}
else if (key.includes('[')) {
const cleanedKey = key.replace('[', '').replace(']', '');
if (Array.isArray(prevValue))
prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue.flatMap((x) => x[cleanedKey]);
else
prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue[cleanedKey];
}
else {
if (Array.isArray(prevValue))
prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue.map((x) => x[key]);
else
prevValue = prevValue === null || prevValue === void 0 ? void 0 : prevValue[key];
}
}
prevValue = CryptoEngine_1.default.hashValue(maskType, prevValue, fieldType);
const res = { [nameInProducer]: prevValue };
return res;
};
const splitArrayFields = (item) => {
const keysWithArrayValues = Object.keys(item).filter(key => Array.isArray(item[key]));
if (keysWithArrayValues.length === 0)
return [item];
const key = keysWithArrayValues[0];
const values = item[key];
const remainingItem = Object.assign({}, item);
delete remainingItem[key];
const splitRemaining = splitArrayFields(remainingItem);
return values.flatMap((value) => {
return splitRemaining.map((remaining) => {
return Object.assign(Object.assign({}, remaining), { [key]: value });
});
});
};
const unpackSingle = (item) => {
const unpackedItem = {};
for (const column of columns) {
const value = unpackDimension(item, column);
Object.assign(unpackedItem, value);
}
return splitArrayFields(unpackedItem);
};
const unpackedData = data.flatMap(x => unpackSingle(x));
return unpackedData;
};
this._getFieldValue = (record, field) => {
var _a, _b, _c;
const fieldValue = record[field.cField.key];
if (Algo_1.default.hasVal(fieldValue) && !isNaN(fieldValue)) {
const fieldType = (_b = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.type) !== null && _b !== void 0 ? _b : 'string';
if (fieldType === 'number' && typeof fieldValue === 'string' && fieldValue.length === 0)
return (_c = field.cField.default) !== null && _c !== void 0 ? _c : fieldValue;
else
return fieldValue;
}
else if ((!Algo_1.default.hasVal(fieldValue) || isNaN(fieldValue)) && Algo_1.default.hasVal(field.cField.default))
return field.cField.default;
else
return fieldValue;
};
}
}
const PostProcessor = new PostProcessorClass();
exports.default = PostProcessor;