@forzalabs/remora
Version:
A powerful CLI tool for seamless data translation.
205 lines (204 loc) • 12.9 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Affirm_1 = __importDefault(require("../../core/Affirm"));
const Algo_1 = __importDefault(require("../../core/Algo"));
const DriverFactory_1 = __importDefault(require("../../drivers/DriverFactory"));
const Helper_1 = __importDefault(require("../../helper/Helper"));
const DeploymentPlanner_1 = __importDefault(require("../deployment/DeploymentPlanner"));
const Environment_1 = __importDefault(require("../Environment"));
const ExecutionEnvironment_1 = __importDefault(require("../execution/ExecutionEnvironment"));
const SQLCompiler_1 = __importDefault(require("../sql/SQLCompiler"));
const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
const UsageManager_1 = __importDefault(require("../UsageManager"));
const ConsumerManager_1 = __importDefault(require("./ConsumerManager"));
class ConsumerEngineClass {
constructor() {
this.compile = (consumer) => {
var _a, _b;
(0, Affirm_1.default)(consumer, `Invalid consumer`);
const availableColumns = consumer.producers.flatMap(cProd => {
var _a, _b;
const producer = Environment_1.default.getProducer(cProd.name);
if (!producer) {
const subConsumer = Environment_1.default.getConsumer(cProd.name);
(0, Affirm_1.default)(subConsumer, `No producer found with name "${cProd.name}"`);
return this.compile(subConsumer);
}
else {
const dims = producer.dimensions.map(x => ({
consumerAlias: null,
consumerKey: null,
nameInProducer: x.name,
aliasInProducer: x.alias,
dimension: x,
owner: cProd.name
}));
const meas = (_b = (_a = producer.measures) === null || _a === void 0 ? void 0 : _a.map(x => ({
consumerAlias: null,
consumerKey: null,
nameInProducer: x.name,
aliasInProducer: x.name,
measure: x,
owner: cProd.name
}))) !== null && _b !== void 0 ? _b : [];
return [...dims, ...meas];
}
});
const selectedColumns = [];
const flat = ConsumerManager_1.default.getConsumerFlatFields(consumer);
for (let i = 0; i < flat.length; i++) {
const field = flat[i];
// TODO: replace with the new funcitons in the consumermanager to reduce diplicate code
if (field.key === '*') {
const from = (_a = field.from) !== null && _a !== void 0 ? _a : (consumer.producers.length === 1 ? consumer.producers[0].name : null);
availableColumns.filter(x => x.owner === from).forEach(col => {
col.consumerKey = col.nameInProducer;
col.consumerAlias = col.nameInProducer;
selectedColumns.push(col);
});
}
else if (field.grouping) {
// This field should be ignored since it is only created when building the output for supported formats (json)
continue;
}
else {
const col = ConsumerManager_1.default.searchFieldInColumns(field, availableColumns, consumer);
(0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
col.consumerKey = field.key;
col.consumerAlias = (_b = field.alias) !== null && _b !== void 0 ? _b : field.key;
selectedColumns.push(col);
}
}
const columnsWithNoAlias = selectedColumns.filter(x => !x.consumerAlias || !x.consumerKey);
(0, Affirm_1.default)(columnsWithNoAlias.length === 0, `Consumer "${consumer.name}" compilation error: some selected fields don't have a correct alias or key (${columnsWithNoAlias.map(x => x.nameInProducer).join(', ')})`);
return selectedColumns;
};
this.deploy = (consumer) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(consumer, `Invalid consumer`);
const firstProd = Environment_1.default.getFirstProducer(consumer.producers[0].name);
(0, Affirm_1.default)(firstProd, `Missing producer in consumer "${consumer.name}"`);
// For now we only support connecting producers of the same engine type to a consumer, so we give an error if we detect different ones
const allSources = consumer.producers.map(x => Environment_1.default.getSource(Environment_1.default.getProducer(x.name).source));
const uniqEngines = Algo_1.default.uniqBy(allSources, 'engine');
(0, Affirm_1.default)(uniqEngines.length === 1, `Sources with different engines were used in a single consumer (${uniqEngines.join(', ')})`);
// For now we also only support consumers that have producers ALL having the same exact source
const uniqNames = Algo_1.default.uniqBy(allSources, 'name');
(0, Affirm_1.default)(uniqNames.length === 1, `Producers with different sources were used in a single consumer (${uniqNames.join(', ')})`);
const source = Environment_1.default.getSource(firstProd.source);
const driver = yield DriverFactory_1.default.instantiateSource(source);
const plan = DeploymentPlanner_1.default.planConsumer(consumer);
for (const planStep of plan) {
switch (planStep.type) {
case 'create-materialized-view': {
const sql = SQLCompiler_1.default.compileConsumer(consumer);
(0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for consumer "${consumer.name}"`);
const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
(0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${consumer.name}" wanting to create a view)`);
// TODO When I want to update a materialize view there is no way except killing it and recreating it. The problem is that: 1) it is not said that it can be deleted since that materialize view could have some dependencies 2) we should find a way to update it without it going completely offline.
const mvSQL = `
DROP MATERIALIZED VIEW IF EXISTS "${internalSchema}"."${SQLUtils_1.default.acceleratedViewName(consumer.name)}";
CREATE MATERIALIZED VIEW "${internalSchema}"."${SQLUtils_1.default.acceleratedViewName(consumer.name)}" AS ${sql}`;
yield driver.execute(mvSQL);
break;
}
case 'create-view': {
const sql = SQLCompiler_1.default.compileConsumer(consumer);
(0, Affirm_1.default)(sql, `Invalid SQL from deployment compilation for consumer "${consumer.name}"`);
const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
(0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${consumer.name}" wanting to create a view)`);
const vSQL = `CREATE OR REPLACE VIEW "${internalSchema}"."${SQLUtils_1.default.sanitizeName(consumer.name)}" AS ${sql}`;
yield driver.execute(vSQL);
break;
}
default: throw new Error(`Invalid execution consumer plan step type "${planStep.type}"`);
}
}
});
this.execute = (consumer, options, user) => __awaiter(this, void 0, void 0, function* () {
(0, Affirm_1.default)(consumer, `Invalid consumer`);
(0, Affirm_1.default)(options, `Invalid execute consume options`);
const { usageId } = UsageManager_1.default.startUsage(consumer, user);
try {
const execution = new ExecutionEnvironment_1.default(consumer);
const result = yield execution.run(options);
UsageManager_1.default.endUsage(usageId, result.data.length);
return result;
}
catch (error) {
UsageManager_1.default.failUsage(usageId, Helper_1.default.asError(error).message);
throw error;
}
});
this.getOutputShape = (consumer) => {
(0, Affirm_1.default)(consumer, `Invalid consumer`);
const compiled = this.compile(consumer);
const outDimensions = compiled.map(x => {
var _a, _b, _c, _d, _e, _f, _g, _h;
return ({
name: (_a = x.consumerAlias) !== null && _a !== void 0 ? _a : x.consumerKey,
type: (_b = x.dimension) === null || _b === void 0 ? void 0 : _b.type,
classification: (_c = x.dimension) === null || _c === void 0 ? void 0 : _c.classification,
description: (_e = (_d = x.dimension) === null || _d === void 0 ? void 0 : _d.description) !== null && _e !== void 0 ? _e : (_f = x.measure) === null || _f === void 0 ? void 0 : _f.description,
mask: (_g = x.dimension) === null || _g === void 0 ? void 0 : _g.mask,
pk: (_h = x.dimension) === null || _h === void 0 ? void 0 : _h.pk
});
});
return {
_version: consumer._version,
name: consumer.name,
description: consumer.description,
metadata: consumer.metadata,
dimensions: outDimensions
};
};
/**
* Given a consumer, create the entire dependency chain of all the sub-consumers, producers and finally sources that are used by this consumer
*/
this.getDependencyChain = (consumer, depth = 0) => {
(0, Affirm_1.default)(consumer, `Invalid consumer`);
const chain = [];
for (let i = 0; i < consumer.producers.length; i++) {
const cProd = consumer.producers[i];
const producer = Environment_1.default.getProducer(cProd.name);
if (!producer) {
const subConsumer = Environment_1.default.getConsumer(cProd.name);
(0, Affirm_1.default)(subConsumer, `No producer found with name "${cProd.name}"`);
chain.push({
depth: depth,
from: { name: consumer.name, type: 'consumer' },
to: { name: subConsumer.name, type: 'consumer' }
});
if (subConsumer.producers && subConsumer.producers.length > 0)
return [...chain, ...this.getDependencyChain(subConsumer, depth + 1)];
}
else {
chain.push({
depth: depth,
from: { name: consumer.name, type: 'consumer' },
to: { name: producer.name, type: 'producer' }
});
chain.push({
depth: depth + 1,
from: { name: producer.name, type: 'producer' },
to: { name: producer.source, type: 'source' }
});
}
}
return chain.sort((a, b) => a.depth - b.depth);
};
}
}
const ConsumerEngine = new ConsumerEngineClass();
exports.default = ConsumerEngine;