@forzalabs/remora/engines/sql/SQLCompiler.js

A powerful CLI tool for seamless data translation.

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const Affirm_1 = __importDefault(require("../../core/Affirm"));
const Algo_1 = __importDefault(require("../../core/Algo"));
const ConsumerEngine_1 = __importDefault(require("../consumer/ConsumerEngine"));
const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
const Environment_1 = __importDefault(require("../Environment"));
const SQLUtils_1 = __importDefault(require("../sql/SQLUtils"));
class SQLCompilerClass {
    constructor() {
        this.compileProducer = (producer, source) => {
            var _a, _b;
            (0, Affirm_1.default)(producer, `Invalid producer`);
            (0, Affirm_1.default)(source, `Invalid source`);
            if (Algo_1.default.hasVal(producer.settings.sql)) {
                const libraryItem = Environment_1.default.getSqlInLibrary(producer.settings.sql);
                (0, Affirm_1.default)(libraryItem, `Misconfiguration for producer "${producer.name}": SQL library item is missing. Required SQL item named "${producer.settings.sql}" was not found.`);
                return libraryItem.sql;
            }
            (0, Affirm_1.default)(producer.settings.sqlTable, `Misconfiguration for producer "${producer.name}": SQL table name is missing. Required since no override to the SQL was used.`);
            const dimensions = producer.dimensions.map(x => {
                var _a;
                const maskType = x.mask;
                const columnReference = (_a = x.alias) !== null && _a !== void 0 ? _a : x.name;
                const fieldReference = `"${producer.settings.sqlTable}"."${columnReference}"`;
                if (x.mask && x.mask === 'hash')
                    return CryptoEngine_1.default.hashQuery(maskType, fieldReference, x.name);
                else
                    return `${fieldReference} AS "${x.name}"`;
            });
            const measures = (_b = (_a = producer.measures) === null || _a === void 0 ? void 0 : _a.map(measure => {
                const thisProducer = `"${producer.settings.sqlTable}"`;
                const sanitized = measure.sql.replace('P', thisProducer).replace('PROD', thisProducer).replace('PRODUCER', thisProducer)
                    .replace('${', '').replace('}', '');
                return `${sanitized} AS "${measure.name}"`;
            })) !== null && _b !== void 0 ? _b : [];
            const columns = dimensions.concat(measures);
            const sql = `SELECT ${columns.join(', ')} FROM "${source.authentication['schema']}"."${producer.settings.sqlTable}"`;
            if (measures.length > 0) {
                const groupBys = `GROUP BY ${dimensions.map((_, i) => i + 1).join(', ')}`;
                return `${sql} ${groupBys}`;
            }
            return sql;
        };
        this.deployProducer = (producer, source) => {
            (0, Affirm_1.default)(producer, `Invalid producer`);
            (0, Affirm_1.default)(source, `Invalid source`);
            const sql = this.compileProducer(producer, source);
            if (producer.settings.direct) {
                return sql;
            }
            else {
                const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
                (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${producer.name}" wanting to create a view)`);
                return `CREATE OR REPLACE VIEW "${internalSchema}"."${producer.name}" AS ${sql}`;
            }
        };
        /**
         * Returns the SQL reference to this producer, used in FROM when constructing SQL statements of consumers (and others).
         * This might easily just return a reference to a view or the underlying SQL of a producer if the view is not available.
         */
        this.getProducerReference = (producer) => {
            (0, Affirm_1.default)(producer, 'Invalid producer');
            const source = Environment_1.default.getSource(producer.source);
            (0, Affirm_1.default)(source, `No source found for producer "${producer.name}"`);
            if (producer.settings.direct) {
                return this.compileProducer(producer, source);
            }
            else {
                const internalSchema = Environment_1.default.get('REMORA_SCHEMA');
                (0, Affirm_1.default)(internalSchema, `Missing "REMORA_SCHEMA" on project settings (needed due to "${producer.name}" wanting to create a view)`);
                return `SELECT * FROM "${internalSchema}"."${producer.name}"`;
            }
        };
        this.getConsumerReference = (consumer) => {
            (0, Affirm_1.default)(consumer, 'Invalid consumer');
            if (consumer.outputs.some(x => x.format === 'SQL' && x.accellerated))
                return `SELECT * FROM "av_remora_${SQLUtils_1.default.sanitizeName(consumer.name)}"`;
            if (consumer.outputs.some(x => x.format === 'SQL' && !x.direct))
                return `SELECT * FROM "v_remora_${SQLUtils_1.default.sanitizeName(consumer.name)}"`;
            return `SELECT * FROM (${this.compileConsumer(consumer)})`;
        };
        this.compileConsumer = (consumer) => {
            var _a;
            (0, Affirm_1.default)(consumer, `Invalid consumer`);
            (0, Affirm_1.default)(consumer.producers.length > 0, `Consumer has no producers to draw data from ("${consumer.name}")`);
            const subqueries = consumer.producers.map(cProd => {
                const producer = Environment_1.default.getProducer(cProd.name);
                if (!producer) {
                    const consumer = Environment_1.default.getConsumer(cProd.name);
                    (0, Affirm_1.default)(consumer, `No producer found for consumer "${consumer.name}" with name "${cProd.name}"`);
                    return `"${cProd.name}" AS (${this.getConsumerReference(consumer)})`;
                }
                return `"${cProd.name}" AS (${this.getProducerReference(producer)})`;
            });
            const columns = ConsumerEngine_1.default.compile(consumer);
            const sqlColumns = columns.map(x => { var _a, _b, _c; return `"${x.owner}"."${(_b = (_a = x.dimension) === null || _a === void 0 ? void 0 : _a.name) !== null && _b !== void 0 ? _b : (_c = x.measure) === null || _c === void 0 ? void 0 : _c.name}" AS "${x.consumerAlias}"`; });
            const columnQualifiers = sqlColumns.map(x => Algo_1.default.replaceAll(x.split('.')[1], '"', ''));
            const dupes = Algo_1.default.duplicates(columnQualifiers);
            (0, Affirm_1.default)(dupes.length === 0, `Wrong consumer configuration: duplicate columns where found for consumer "${consumer.name}" ("${dupes.join(', ')}")`);
            const joins = consumer.producers.filter(x => x.joins && x.joins.length > 0).flatMap(cProd => {
                return cProd.joins.map(x => {
                    const otherProd = consumer.producers.find(k => k.name === x.otherName);
                    (0, Affirm_1.default)(otherProd, `Invalid JOIN relantionship: the producer "${cProd.name}" is asking for a join with "${x.otherName}", but this one doesn't exists.`);
                    const starts = Algo_1.default.locations(x.sql, '${');
                    const ends = Algo_1.default.locations(x.sql, '}');
                    (0, Affirm_1.default)(starts.length === ends.length, `Invalid JOIN SQL: number of condition parameters does not match on consumer "${consumer.name}" producer "${cProd.name}" ("${x.sql}")`);
                    const params = starts.map((start, index) => x.sql.substring(start, ends[index] + 1));
                    const thisProducer = `"${cProd.name}"`;
                    const sqlParams = params.map(param => param.replace('P', thisProducer).replace('PROD', thisProducer).replace('PRODUCER', thisProducer)
                        .replace('${', '').replace('}', ''));
                    let sqlCondition = x.sql;
                    params.forEach((p, i) => sqlCondition = sqlCondition.replace(p, sqlParams[i]));
                    return `LEFT JOIN "${otherProd.name}" ON ${sqlCondition}`;
                });
            });
            const filters = (_a = consumer.filters) === null || _a === void 0 ? void 0 : _a.map(x => {
                const starts = Algo_1.default.locations(x.sql, '${');
                const ends = Algo_1.default.locations(x.sql, '}');
                (0, Affirm_1.default)(starts.length === ends.length, `Invalid filter SQL: number of condition parameters does not match on consumer "${consumer.name}" ("${x.sql}")`);
                const params = starts.map((start, index) => x.sql.substring(start, ends[index] + 1));
                const sqlParams = params.map(param => SQLUtils_1.default.findDimension(columns, param));
                let sqlFilter = x.sql;
                params.forEach((p, i) => sqlFilter = sqlFilter.replace(p, sqlParams[i].consumerAlias));
                return sqlFilter;
            });
            let sql = `WITH ${subqueries.join(',\n')}\n\nSELECT ${sqlColumns.join(', ')} FROM "${consumer.producers[0].name}"`;
            if (joins && joins.length > 0)
                sql += ` ${joins.join(', ')}`;
            if (filters && filters.length > 0)
                sql += ` WHERE ${filters.join(', ')}`;
            return sql;
        };
    }
}
const SQLCompiler = new SQLCompilerClass();
exports.default = SQLCompiler;