persistanz/lib/DslResolver.js

Object relational mapping (ORM) library with unique features.

"use strict";
var ModelMeta = require("./ModelMeta.js");
var helper = require("./helper.js");

class Alias {
  constructor (prefix, number, modelMeta, parentAliasObject, bridgeField, pers) {
    this.prefix = prefix;
    this.name = "#" + number;
    this.escaped = pers.escapeId(this.name);
    this.bridgeField = bridgeField;
    this.parent = parentAliasObject;
    this.modelMeta = modelMeta;
  }
}

module.exports = class DslResolver {
  constructor (persQuery) {
    this.pq = persQuery;
    this.pers = this.pq.pers;
    this.aliasCount = 0;
    this.objectAliasMap = {}; //indexed by their prefixes
    this.sqlClauses = {
      selects: [],
      discriminator: null, //{sql, value}
      where: null, //{sql, values},
      from: null,
      limit: null, //{sql, values}
      orderBy: null,
      groupBy: null,
      having: null, //{sql, values}
      joins: null, //{sql, values}
      queryOptions: null,
      mainQuery: null, //{sql, values}
      subQueries: [],
      index: null, //{type: field/callback, callback, sql, aliasName, fieldName}
      limitlessQuery: null, //{sql, values}
      toManyIdsIn: null, //{sql, values}
    };

    this.dsl = { //these are mostly input by the user by calling methods.
      from: null,
      where: null,
      limit: null,
      limitValues: [],
      orderBy: null,
      groupBy: null,
      selects: [],
      having: null,
      havingValues: [],
      withSelects: [], /*{withPart, fieldList}*/
      aliasSelects: [],
      distinct: false,
      calcLimitless: false,
      queryOptions: null,
      whereValues:[], //entered by user to replace ?.
      index: null,
      toManyIdsIn: null, //someTableId IN (?, ?, ?...)
      toManyIdsInValues: null,  //[value, value, value]
    };
  }

  setDsl (what, value) {
    if (this.dsl[what] === undefined) throw new Error ("Internal error / setDsl : " + what);
    this.dsl[what] = value;
  }

  addDsl (what, value) {
    if (!Array.isArray(this.dsl[what])) throw new Error ("Internal error / addDsl : " + what);
    this.dsl[what].push(value);
  }

  generateJoins() {

    //a foreign key definition looks likes this:
    /*
    { name: 'test2',
      fkColumn: {
        name: 'test2Id',
        fkInfo: { toTable: 'Test2', toColumn: 'id' }
      },
      isAutoGenerated: true,
      modelName: 'Test2' }
    */

    var joins = [], values = [];
    for (var prefix in this.objectAliasMap) {
      if (prefix === '') continue; //don't produce a join for the FROM table.
      var aliasObject = this.objectAliasMap[prefix];
      var eTableName = this.pers.escapeId(aliasObject.modelMeta.table.name);
      var eAlias = aliasObject.escaped;
      var eParentAlias = aliasObject.parent.escaped;
      var bfc = aliasObject.bridgeField.fkColumn;
      var eFkTo = this.pers.escapeId(bfc.fkInfo.toColumn);
      var eFkFrom = this.pers.escapeId(bfc.name);
      var j = `LEFT JOIN ${eTableName} ${eAlias} ON ${eAlias}.${eFkTo} = ${eParentAlias}.${eFkFrom}`;
      var discCondition = "";
      //check if it has a discriminator column, and use it if so:
      if (aliasObject.modelMeta.discriminator != null) {
        var eDisc = this.pers.escapeId(aliasObject.modelMeta.discriminator);
        discCondition = ` AND ${eAlias}.${eDisc} = ?`
        values.push(aliasObject.modelMeta.name);
        j += discCondition;
      }
      joins.push(j);
    }

    if (!joins.length) return this.sqlClauses.joins = null;

    return this.sqlClauses.joins = {
      sql: joins.join("\n"),
      values
    }
  }

  createAlias (prefix, modelMeta, parentAliasObject, bridgeField) {
    if (!this.objectAliasMap[prefix]) {
      var number = this.aliasCount++;
      this.objectAliasMap[prefix] =
        new Alias(prefix, number, modelMeta, parentAliasObject, bridgeField, this.pers);
    }
    return this.objectAliasMap[prefix];
  }

  //in:  "1+1, SUM({test2.id}) as total, AVG({test2.count}) as avg"
  //out: [ '1+1, SUM("#1"."id") as total, AVG("#1"."title") as avg' ]
  resolveCurlyExpression (expression, forWhat) {
    var r=/\{([^\}]+)\}/g, regexResults, replaced = expression;
    while ((regexResults = r.exec(expression)) !== null) {
      var curly = regexResults[0].trim();
      var exp = regexResults[1].trim();
      var resolvedList = this.resolveFieldExpression(exp, forWhat);
      replaced = replaced.replace(curly, this.generateSqlFromField(resolvedList[0]));
    }
    return replaced;
  }

  build () {

    if (!this.sqlClauses.from)
      throw new Error(".from() is never called, from clause is missing.");

    this.processAliasSelect();
    this.expandSelectWithToSelects();
    this.processIndex();
    this.processSelect();
    this.processWhere();
    this.processToManyIdsIn();
    this.processGroupBy();
    this.processHaving();
    this.processLimit();
    this.processOrderBy();
    this.processQOptions(); //handles distinct too.
    this.generateJoins();

    if (!this.sqlClauses.selects.length)
      throw new Error("No fields specified to be used in select clause for the query.");

    var c = this.sqlClauses;
    var select = c.selects.join(', ');
    var query = "SELECT ";
    if (c.queryOptions) query += `${c.queryOptions} `;
    query += `${select}\n`;

    query += `FROM ${c.from}\n`;

    if (c.joins != null) query += `${c.joins.sql}\n`;

    if (c.where != null || c.discriminator != null || c.toManyIdsIn != null) {
      var whereParts = [];
      if (c.where != null) whereParts.push(c.where.sql);
      if (c.discriminator != null) whereParts.push(c.discriminator.sql);
      if (c.toManyIdsIn != null) whereParts.push(c.toManyIdsIn.sql);

      if (c.where != null && whereParts.length > 1)
        whereParts[0] = "( " + whereParts[0] + " )";

      query += "WHERE " + whereParts.join(" AND\n") + "\n";
    }

    if (c.groupBy)  query += `GROUP BY ${c.groupBy}\n`;
    if (c.having)   query += `HAVING ${c.having.sql}\n`;

    var limitlessQuery = this.dsl.calcLimitless ? query : '';

    if (c.orderBy)  query += `ORDER BY ${c.orderBy}\n`;
    if (c.limit) query += `LIMIT ${c.limit.sql}\n`;

    //prepare values to be escaped.
    //order is joins, where, discriminator, toManyIdsIn, having, limit,
    var values = [];
    if (c.joins != null) values = values.concat(c.joins.values);
    if (c.where != null && c.where.values) values = values.concat(c.where.values);
    if (c.discriminator != null) values.push(c.discriminator.value);
    if (c.toManyIdsIn != null) values = values.concat(c.toManyIdsIn.values);
    if (c.having != null) values = values.concat(c.having.values);
    //limitlessQuery values are all values except that of limit clause.
    //so simply copy the array:
    var limitlessValues = limitlessQuery === '' ? null : values.slice(0);
    if (c.limit != null) values = values.concat(c.limit.values);

    c.mainQuery = {
      sql: query,
      values,
    };

    if (this.dsl.calcLimitless) {
      var eCount = this.pers.escapeId("count");
      var subAlias = this.pers.escapeId("#SOME_TABLE_ALIAS");

      this.sqlClauses.limitlessQuery = {
        sql:  `SELECT COUNT(*) AS ${eCount}
              FROM (
                ${limitlessQuery}
              ) ${subAlias}`,
        values: limitlessValues,
      }
    }

    return this;
  }

  mapRows (rows) {
    var indexed = this.sqlClauses.index != null;
    var results = indexed ? new Map() : [];

    //Unroll loops, it may speed up to remove a conditional:
    if (! indexed) {
      for (var row of rows)
        results.push(this.mapRow(row).object);
    } else { //indexed
      for (var row of rows) {
        var mappedRow = this.mapRow(row);
        results.set(mappedRow.index, mappedRow.object);
      }
    }

    return results;
  }

  mapRow (row) {
    //create empty objects based on the map, keyed to aliases:
    var objects = {};

    for (var prefix in this.objectAliasMap) {
      var aliasObject = this.objectAliasMap[prefix];
      var object = new (aliasObject.modelMeta.model)();
      objects[aliasObject.name] = object;
    }

    //extract columns:
    //TODO: This needs to be optimized. We are in the inner loop, so
    //try to avoid split, shift and join by analysing field mapping
    //before rows.forEach, and then only loop to assign properties.
    for (var aliasedColumn in row) {
      var parts = aliasedColumn.split('.');
      var tableAlias = parts.shift(); //e.g: #1
      var fieldPart = parts.join('.'); //e.g: title
      if (objects[tableAlias]) {
        //normal fields:
        var meta = ModelMeta.getByObject(this.pers, objects[tableAlias]);
        var value = meta.deserialize(fieldPart, row[aliasedColumn]);
        objects[tableAlias][fieldPart] = value;
      }
      else //.selectAlias exps don't map, they are attached to object #0.
        objects["#0"][aliasedColumn] = row[aliasedColumn];
    }

    //connect objects to each other.
    for (var prefix in this.objectAliasMap) {
      var oMap = this.objectAliasMap[prefix];

      //map tomany fields:
      if (this.pq.toManyQueries && this.pq.toManyQueries.size) {
        this.pq.toManyQueries.forEach( (toManyQuery) => {
          var alias = this.objectAliasMap[toManyQuery.prefixToParent];
          var pkName = this.objectAliasMap[toManyQuery.prefixToParent].modelMeta.table.pks[0];
          objects[alias.name][toManyQuery.toManyField.name] = toManyQuery.groupedRows.get(objects[alias.name][pkName]) || [];
        });
      }

      //attach objects to each other:
      if (oMap.parent) { //root object doesn't have it, so we only deal with bridge fields.
        //we want bridgeFields to be null if the result brought a null id.
        var pkName = oMap.modelMeta.table.pks[0];
        var parentObject = objects[oMap.parent.name];
        parentObject[oMap.bridgeField.name] = objects[oMap.name][pkName] != null
          ? objects[oMap.name]
          : null;
        //if the user didn't exclusively wanted the pk, remove it.
        //don't attempt to delete properties from the fields that are nulled above:
        ///if (removeFields[anAlias] && objects[oMap.bindToAlias][oMap.propName])
          ///delete objects[oMap.bindToAlias][oMap.propName][pkName];
        if (typeof objects[oMap.name].afterLoad === 'function')
          helper.polycallBasedOnSignatureLength(objects[oMap.name].afterLoad, objects[oMap.name], [this.pq.options.tx], 2);
      }
    }

    //root object's afterLoad is called after all the bridge fields' objects.
    if (typeof objects["#0"].afterLoad === 'function')
      helper.polycallBasedOnSignatureLength(objects["#0"].afterLoad, objects["#0"], [this.pq.options.tx], 2);

    var index = null;

    if (this.sqlClauses.index) {
      var indexInfo = this.sqlClauses.index;
      if (indexInfo.type === 'field') {
        index = objects[indexInfo.aliasName][indexInfo.fieldName];
      } else { //callback;
        index = indexInfo.callback(objects["#0"]);
      }
    }

    return {object: objects["#0"], index};
  }

  processFrom() {
    var meta = ModelMeta.getByName(this.pers, this.dsl.from);
    var alias = this.createAlias("", meta, null, null);

    //okay, if discriminator, handle that too:
    if (meta.discriminator != null) {
      this.sqlClauses.discriminator = {
        sql: this.resolveCurlyExpression(`{${meta.discriminator}} = ?`, "discriminator"),
        value: meta.name,
      };
    }

    var escapedTableName = this.pers.escapeId(meta.table.name);
    return this.sqlClauses.from = `${escapedTableName} ${alias.escaped}`;
  }

  expandSelectWithToSelects () {
    //collect all blocks, if block is string, split by ',':
    var fieldList = this.dsl.withSelects.map(wsObject => {
      var withPart = wsObject.withPart.trim();
      var listPart = wsObject.fieldList;
      var list = Array.isArray(listPart) ? listPart : listPart.split(',');
      return list.map(part => {
        var trimmed = part.trim();
        //put the ! to the beginning as it is the regular syntax.
        return trimmed.substr(0,1) === '!'
          ? `!${withPart}${this.pers.separator}${trimmed.substring(1)}`
          : `${withPart}${this.pers.separator}${trimmed}`;
      });
    });

    //merge these new select expressions into the selects list:
    this.dsl.selects = this.dsl.selects.concat([].concat.apply([], fieldList));
  }

  //generates parts that go into the SELECT clause while creating necessary
  //aliases etc.
  processSelect () {
    //- simple fields directly connected to the table mentioned in the from: "name, age"
    //- *: "*",
    //- fields from connecting table "author.name, author.wife.age, author.*, author.specs.*"
    //- exclude fields: "!name, author.wife.*, !author.wife.age"
    //- to many: "author.pets.*, author.cars.*, !author.cars.year, author.pets.friends.*",
    //- select with .sw("author.wife","name,age,pet.*,!pet.vetName)"

    //handle empty select and resolve it to "*".
    if (! this.sqlClauses.selects.length && ! this.dsl.selects.length)
      this.dsl.selects.push("*");

    //an item dsl.selects can be an array or string. if string split by ','.
    var expList = [], fieldDefs = [];
    for (var list of this.dsl.selects) {
      var asArray = Array.isArray(list) ? list : list.split(',');
      expList = expList.concat(asArray.map(p => p.trim()));
      for (var exp of expList) {
        var foundFieldDefs = this.resolveFieldExpression(exp, "select");
        fieldDefs = fieldDefs.concat(foundFieldDefs);
      }
    }

    //filter out exclude fields:
    //separate the list into 2 sets, unique them and diff them:
    var excludeFields = fieldDefs.filter(f => f.isExclude);
    var includeFields = fieldDefs.filter(f => ! f.isExclude);
    var finalFields = {};

    //make sure child query has parent's foreign key, otherwise select list may
    //come empty.
    if (this.pq.parentQuery) {
      var fkColumnName = this.pq.toManyField.fkColumn.name;
      var fkDef = this.createFieldDef([], fkColumnName, fkColumnName, false, true);
      includeFields.push(fkDef);
    }

    //filter out exclude fields. autoadded fields must not be filtered.
    for (var i of includeFields) {
      var toInclude = true;
      for (var e of excludeFields) { //filter out if excluded and not autoAdded
        if (i.canonical === e.canonical && ! i.autoAdded) {
          toInclude = false;
          break;
        }
      }
      if (!toInclude) continue;
      if (finalFields[i.canonical]) { //if already added, prioritise non-autoAdded
        if (! i.autoAdded || ! finalFields[i.canonical].autoAdded)
          finalFields[i.canonical].autoAdded = false;
      } else {
        finalFields[i.canonical] = i;
      }
    }

    var clauseParts = [];
    for (var canonical in finalFields)
      clauseParts.push(this.generateSqlFromField(finalFields[canonical], true));

    return this.sqlClauses.selects = this.sqlClauses.selects.concat(clauseParts);
  }

  createFieldDef (prefixParts, columnName, alias, isExclude, autoAdded) {
    var parts = prefixParts.slice(0); //create new array as it is a ref, causing bugs.
    return  {
      prefixParts: parts,
      columnName,
      sqlAsAlias: alias,
      isExclude,
      autoAdded,
      canonical: parts.concat([alias]).join('.'),
    }
  }

  generateSqlFromField (field, forSelect) {
    var aliasName = this.objectAliasMap[field.prefixParts.join('.')].name;
    var columnName = this.pers.escapeId(aliasName) + "." + this.pers.escapeId(field.columnName);
    if (! forSelect) return columnName;
    var columnAlias = this.pers.escapeId(`${aliasName}.${field.sqlAsAlias}`);
    return `${columnName} AS ${columnAlias}`;
  }

  //analyses a field expression in the form of a.b.c, !a.b.c, a.b.*
  //generates necessary join information as well as toMany queries on the way.
  //toMany() requires all of the "hidden" toMany queries to be resolved in an
  //expression so that the last toMany query is returned. Normally this information
  //is not available until exec() called, so if called by toMany() we recursively
  //analyse all the remaining parts of an expression to generate all toMany queries,
  //and return the last one found.
  //if called from processSelect() we need to return fieldDefs so that it can
  //further analyse exclude fields etc.
  //returns array of field defs, or a toManyQuery if called from toMany().
  resolveFieldExpression (fieldExp, forWhat) {
    var exp = fieldExp.trim();
    var isExclude = exp.substr(0, 1) === '!';
    if (isExclude) exp = exp.substr(1);
    var parts = helper.split(exp, this.pers.separatorRegex);

    //handle special cases:
    if (isExclude) {
      if (forWhat !== 'select')
        throw new Error(`Expression ${fieldExp} is invalid as '!' operator cannot be used outside of a 'select' context.`);
      if (parts[parts.length - 1].trim() === '*')
        throw new Error(`Expression '${fieldExp}' is invalid as '!' operator cannot be used with '*'.`);
    }

    var partsSoFar = [], prefixPartsSoFar = [];
    var lastMeta = this.objectAliasMap[""].modelMeta;
    var requiredPkDefs = [], separator = this.pers.separator;

    //loop the parts
    for (var index = 0; index < parts.length; index++) {
      var p = parts[index].trim();
      var pFieldName = p.replace(/\\/g, '');
      var remainingParts = parts.slice(index + 1);
      partsSoFar.push(p);

      //1. check if it is toMany
      if (lastMeta.toManyFields && lastMeta.toManyFields[pFieldName]) {
        if (forWhat === 'index')
          throw new Error(`'${fieldExp}' resolves to a toMany field, which is invalid in index().`);
        var toManyField = lastMeta.toManyFields[pFieldName];
        lastMeta = this.pers.modelMeta[toManyField.modelName];
        var toManyQuery = this.pq._createToManyQuery(partsSoFar, toManyField, lastMeta);
        if (remainingParts.length)
          toManyQuery.s( (isExclude ? '!' : '') + remainingParts.join(separator));
        if (forWhat === 'toMany')
          return ! remainingParts.length ?
            toManyQuery :
            toManyQuery.resolver.resolveFieldExpression(remainingParts.join(separator), "toMany");
        //requiredPkDefs will be lost when called from toMany(), but this function
        //will be called again by processSelects(), which will add this anyway.
        return requiredPkDefs;
      }

      //2. check if it is a bridgeField:
      if (lastMeta.bridgeFields[pFieldName]) {
        var bridgeField = lastMeta.bridgeFields[pFieldName];
        if (!remainingParts.length)
          if (forWhat != "toMany") //toMany() handles the return value and gives a better err msg.
            throw new Error(`Expression '${fieldExp}' resolves to a bridge field, a query needs columns to be added.`);

        lastMeta = this.pers.modelMeta[bridgeField.modelName];
        var parent = this.objectAliasMap[partsSoFar.slice(0, index).join(separator)];
        var columnName = bridgeField.fkColumn.fkInfo.toColumn;
        prefixPartsSoFar.push(p);
        //add the primary key of the foreign table, otherwise we may not know
        //if the join resulted in no columns or other fields are null. also
        //required because the query may result in no select fields.
        //also create the alias. All bridge field columns may be excluded by !
        //later on, but extra join doesn't doesn't cause any problems, so we go ahead.
        if (! isExclude){
          var reqPk = this.createFieldDef(prefixPartsSoFar, columnName, columnName, isExclude, true);
          requiredPkDefs.push(reqPk);
          this.createAlias(partsSoFar.join(separator), lastMeta, parent, bridgeField);
        }
        continue;
      }

      //3. if a * star field:
      if (p === '*') {
        if (remainingParts.length) throw new Error(`'${fieldExp}' cannot be resolved.`);
        if (forWhat === 'index') throw new Error("'*' is invalid in index.");
        var fields = Object.keys(lastMeta.table.columns)
          .map(c => this.createFieldDef(prefixPartsSoFar, c, c, false, false));
        return [].concat(fields, requiredPkDefs);
      }

      //4. if a column ?
      if (lastMeta.table.columns[pFieldName]) { //is a column ?
        if (remainingParts.length) throw new Error(`'${fieldExp}' cannot be resolved.`);
        var fields = [this.createFieldDef(prefixPartsSoFar, pFieldName, pFieldName, isExclude, false)];
        return [].concat(fields, requiredPkDefs);
      }

      //5. if an abstraction over affix ?
      for (var affixName in this.pq.abstractAffixes) {
        var affixObject = this.pq.abstractAffixes[affixName];
        var possibleFieldName = affixObject.type == 'suffix'
          ? `${pFieldName}${affixObject.affix}`
          : `${affixObject.affix}${pFieldName}` ;
        if (lastMeta.table.columns[possibleFieldName]) {
          if (remainingParts.length) throw new Error(`'${fieldExp}' cannot be resolved.`);
          var fields = [this.createFieldDef(prefixPartsSoFar, possibleFieldName, pFieldName, isExclude, false)];
          return [].concat(fields, requiredPkDefs);
        }
      }

      //nothing found:
      throw new Error(`'${fieldExp}' cannot be resolved.`);
    }
  }

  processIndex() {
    if (this.dsl.index == null) return;
    if (typeof this.dsl.index === "function")
      this.sqlClauses.index = {type: "callback", callback: this.dsl.index};
    else {
      this.resolveFieldExpression(this.dsl.index, "index"); //genarate bridges if necessary
      this.pq.select(this.dsl.index);
      var parts = helper.split(this.dsl.index, this.pers.separatorRegex).map(p => p.trim());
      var fieldName = parts.pop().replace(/\\/g, '');
      var prefix = parts.join(this.pers.separator);
      var aliasName = this.objectAliasMap[prefix].name;
      this.sqlClauses.index = {type: "field", aliasName, fieldName};
    }
  }

  processAliasSelect() {
    var results = this.dsl.aliasSelects.map(as => this.resolveCurlyExpression (as, "aliasSelect"))
    return this.sqlClauses.selects = this.sqlClauses.selects.concat(results);
  }

  processWhere() {
    if (this.dsl.where != null)
      return this.sqlClauses.where = {
        sql: this.resolveCurlyExpression(this.dsl.where, "where"),
        values: this.dsl.whereValues,
      };
  }

  processToManyIdsIn() {
    if (this.dsl.toManyIdsIn != null)
      return this.sqlClauses.toManyIdsIn = {
        sql: this.resolveCurlyExpression(this.dsl.toManyIdsIn, "toManyIdsIn"),
        values: this.dsl.toManyIdsInValues,
      };
  }

  processOrderBy() {
    if (this.dsl.orderBy != null)
      return this.sqlClauses.orderBy = this.resolveCurlyExpression(this.dsl.orderBy, "orderBy");
  }

  processHaving() {
    if (this.dsl.having != null)
      return this.sqlClauses.having = {
        sql: this.resolveCurlyExpression(this.dsl.having, "having"),
        values: this.dsl.havingValues,
      }
  }

  processGroupBy() {
    if (this.dsl.groupBy != null)
      return this.sqlClauses.groupBy = this.resolveCurlyExpression(this.dsl.groupBy, "groupBy");
  }

  processLimit() {
    if (this.dsl.limit != null)
      return this.sqlClauses.limit = {
        sql: this.dsl.limit,
        values: this.dsl.limitValues,
      };
  }

  processQOptions() {
    this.sqlClauses.queryOptions = this.dsl.distinct ? "DISTINCT" : '';
    if (this.dsl.queryOptions != null)
      this.sqlClauses.queryOptions += ` ${this.dsl.queryOptions}`;
    return this.sqlClauses.queryOptions;
  }
}