joola.io.engine/lib/objects/datatables.js

joola.io's Framework Engine

/**
 *  joola.io
 *
 *  Copyright Joola Smart Solutions, Ltd. <info@joo.la>
 *
 *  Licensed under GNU General Public License 3.0 or later.
 *  Some rights reserved. See LICENSE, AUTHORS.
 *
 *  @license GPL-3.0+ <http://spdx.org/licenses/GPL-3.0+>
 */


var
  joola = global.joola, //for jslint
  url = require('url'),
  ce = require('cloneextend'),
  _datasources = require('./datasources'),
  _dimensions = require('./dimensions'),
  _metrics = require('./metrics'),
  connector = require('../connectors/connector'),
  manager = require('../caching/manager');

var list = function (datasourceid) {
  var datasources;
  if (datasourceid)
    datasources = _datasources.get(datasourceid);
  else
    datasources = _datasources.list();
  var datatables = [];

  _.each(datasources, function (ds) {
    _.each(ds.datatables, function (dt) {
      dt.datasourceid = ds.id;
      datatables.push(dt);
    })
  });
  return ce.clone(datatables);
};

var get = function (datatableid) {
  var datatables = this.list();
  return ce.clone(_.find(datatables, function (dt) {
    try {
      return dt.id.toLowerCase() == datatableid.toLowerCase();
    }
    catch (ex) {
      return null;
    }
  }));
};

var minCacheDate = function (datatable, callback) {
  manager.minCacheDate(datatable, callback);
};

var maxCacheDate = function (datatable, callback) {
  manager.maxCacheDate(datatable, callback);
};

var flushCache = function (datatable, callback) {
  manager.flushCache(datatable, callback);
};

var basequery = function (datatable) {
  var _self = this;
  var result = {};
  result.sql = 'SELECT\r\n';

  if (!datatable.datasource) {
    datatable.datasource = _datasources.get(datatable.datasourceid);
  }
  var dimensions = ce.clone(_dimensions.list(datatable.datasource.id, datatable.id));

  var metrics = ce.clone(_metrics.list(datatable.datasource.id, datatable.id));

  result.datatable = datatable;
  result.dimensions = dimensions;
  result.refDimensions = [];
  result.metrics = metrics;

  result.sql += '\t' + datatable.name.clean() + '.`' + datatable.dates[0] + '` as `date`,\r\n';
  //result.sql += '\tcount(1) as "rowcount",\r\n';

  var groupBy = '\t' + datatable.name.clean() + '.`' + datatable.dates[0] + '`,\r\n';
  var dimensionsUsedInSelect = [];
  var referencedTables = [];

  //add dimensions from reference
  var recurReferences = function (references) {
    references.forEach(function (reference) {
      if ((reference.type && reference.type != 'restrict') || reference.type == null) {
        var refTable = _self.get(reference.datatableid, reference);
        refTable.dimensions.forEach(function (dimension) {
          if (!dimension.name || !dimension.datatableid) {
            var orig_column = dimension.column;
            dimension = _dimensions.get(dimension.id);
            dimension.dicColumn = dimension.column;
            dimension.column = orig_column;
          }
          if (!dimension.datatable)
            dimension.datatable = _self.get(dimension.datatableid);

          dimension.referenced = true;

          if (dimensionsUsedInSelect.indexOf(dimension.id) == -1) {
            _.each(dimensions, function (d, i) {
              if (dimensions.id == dimension.id)
                dimensions[i] = dimension;
            });

            var dimRefTable = _self.get(dimension.datatableid);
            if (dimRefTable == null)
              joola.logger.error('Failed to locate datatable for [' + dimension.name + ':' + dimension.datatableid + ']');
            //add dimensions according to their dictionaries
            if (dimension.virtual) {
              if (dimension.alias) {
                result.sql += '\tCOALESCE(' + dimension.column + ', \'(not set)\'), \'(not set)\') as `' + dimension.alias + '`,\r\n';
                groupBy += '\tCOALESCE(' + dimension.column + ',\r\n';
              }
              else {
                result.sql += '\tCOALESCE(' + dimRefTable.name.clean() + '.`' + dimension.column + '`, \'(not set)\') as `' + dimension.name + '`,\r\n';
                groupBy += '\tCOALESCE(' + dimRefTable.name.clean() + '.`' + dimension.column + '`, \'(not set)\'),\r\n';
              }
            }
            else {
              result.sql += '\tCOALESCE(' + dimRefTable.name.clean() + '.`' + dimension.dicColumn + '`, \'(not set)\') as `' + dimension.name + '`,\r\n';
              groupBy += '\tCOALESCE(' + dimRefTable.name.clean() + '.`' + dimension.dicColumn + '`, \'(not set)\'),\r\n';
            }

            dimensionsUsedInSelect.push(dimension.id);
          }
        });

        //referencedTables.push(refTable);

        if (refTable.references) {
          recurReferences(refTable.references);
        }
      }
    });

  };

  dimensions.forEach(function (dimension, index) {
    if (!dimension.name) {
      var orig_column = dimension.column;
      dimension = _dimensions.get(dimension.id);
      dimension.dicColumn = dimension.column;
      dimension.column = orig_column;
    }
    if (!dimension.datatable)
      dimension.datatable = _self.get(dimension.datatableid);

    if (dimensionsUsedInSelect.indexOf(dimension.id) == -1) {
      dimensions[index] = dimension;
      //add dimensions according to their dictionaries
      if (dimension.virtual) {
        if (dimension.alias) {
          result.sql += '\tCOALESCE(' + dimension.column + ', \'(not set)\') as `' + dimension.alias + '`,\r\n';
          groupBy += '\tCOALESCE(' + dimension.column + ', \'(not set)\'),\r\n';
        }
        else {
          if (dimension.allownulls) {
            result.sql += '\t' + dimension.datatable.name.clean() + '.`' + dimension.column + '` as `' + dimension.name + '`,\r\n';
            groupBy += '\t' + dimension.datatable.name.clean() + '.`' + dimension.column + '`,\r\n';
          } else {
            result.sql += '\tCOALESCE(' + dimension.datatable.name.clean() + '.`' + dimension.column + '`, \'(not set)\') as `' + dimension.name + '`,\r\n';
            groupBy += '\tCOALESCE(' + dimension.datatable.name.clean() + '.`' + dimension.column + '`, \'(not set)\'),\r\n';
          }
        }
      }
      else {
        result.sql += '\tCOALESCE(' + dimension.datatable.name.clean() + '.`' + dimension.dicColumn + '`, \'(not set)\') as `' + dimension.name + '`,\r\n';
        groupBy += '\tCOALESCE(' + dimension.datatable.name.clean() + '.`' + dimension.dicColumn + '`, \'(not set)\'),\r\n';
      }

      dimensionsUsedInSelect.push(dimension.id);

      if (dimension.datatable.references) {
        recurReferences(dimension.datatable.references);
      }
    }
  });

  if (datatable.references)
    recurReferences(datatable.references);

  _.each(metrics, function (metric) {
    if (metric.virtual) {
      result.sql += '\t' + metric.aggregation + '(' + metric.column + ') as `' + metric.name + '`,\r\n';
    }
    else {
      result.sql += '\t' + metric.aggregation + '("' + metric.column + '") as `' + metric.name + '`,\r\n';
    }
  });

  result.sql = result.sql.substring(0, result.sql.length - 3) + '\r\n';
  result.sql += 'FROM ' + datatable.name + ' AS ' + datatable.name.clean() + '\r\n';

  referencedTables.push(datatable.id);

  //let's grab the references
  var recurReferences = function (references, datatable) {
    references.forEach(function (reference) {

      var refTable = _self.get(reference.datatableid);
      if (referencedTables.indexOf(refTable.id) == -1) {

        result.sql += (reference.type == 'partial' ? 'LEFT ' : '') + 'JOIN ' + refTable.name + ' ' + refTable.name.clean() + ' ON ';
        reference.columns.forEach(function (columnPair) {
          if (reference.pipe)
            result.sql += refTable.name.clean() + '.' + columnPair.foreign + ' = ' + columnPair.base + ' AND ';
          else
            result.sql += refTable.name.clean() + '.' + columnPair.foreign + ' = ' + datatable.name.clean() + '.' + columnPair.base + ' AND ';
        });
        referencedTables.push(refTable.id);
        result.sql = result.sql.substring(0, result.sql.length - 5) + '\r\n';
        if (reference.type == null || (reference.type && reference.type != 'restrict')) {
          refTable.dimensions.forEach(function (dimension) {
            if (!dimension.name) {
              var orig_column = dimension.column;
              dimension = _dimensions.get(dimension.id);
              dimension.dicColumn = dimension.column;
              dimension.column = orig_column;
            }
            if (!dimension.datatable)
              dimension.datatable = _self.get(dimension.datatableid);

            dimension.referenced = true;
            if (_.find(dimensionsUsedInSelect, function (d) {
              return d == dimension.id
            })) {

              if (!dimension.datatableid)
                dimension = _dimensions.get(dimension.id);
              if (!dimension.datatable)
                dimension.datatable = _self.get(dimension.datatableid);

              var dimRefTable = _self.get(dimension.datatableid);
              if (referencedTables.indexOf(dimRefTable.id) == -1) {
                //add dimensions according to their dictionaries
                if (!dimension.virtual) {
                  result.sql += 'JOIN ' + dimRefTable.name + ' ' + dimRefTable.name.clean() + ' ON ' + dimRefTable.name.clean() + '.' + dimRefTable.primarykey + ' = ' + refTable.name.clean() + '.' + dimension.column + '\r\n';
                  referencedTables.push(dimRefTable.id);
                }
              }
            }
          });

          if (refTable.references) {
            recurReferences(refTable.references, refTable);
          }
        }
      }

    });
  };


  //join dictionaries for dimensions
  dimensions.forEach(function (dimension) {
    if (!dimension.referenced) {
      if (!dimension.virtual) {
        var dicTable = dimension.datatable;
        result.sql += 'JOIN ' + dicTable.name + ' ' + dicTable.name.clean() + ' ON ' + dicTable.name.clean() + '.' + dicTable.primarykey + ' = ' + datatable.name.clean() + '.' + dimension.column + '\r\n';
        referencedTables.push(dicTable.id);

        if (dimension.datatable.references) {
          recurReferences(dimension.datatable.references, dimension.datatable);
        }
      }
    }
  });

  recurReferences(datatable.references, datatable);
  /*
   datatable.references.forEach(function (reference) {
   var refTable = _self.get(reference.datatableid);
   result.sql += (reference.type == 'partial' ? 'LEFT ' : '') + 'JOIN ' + refTable.name + ' ' + refTable.name.clean() + ' ON ';// + refTable.name.clean() + '.' + refTable.primarykey + ' = ' + datatable.name + '.' + dimension.column + '\r\n';

   reference.columns.forEach(function (columnPair) {
   result.sql += refTable.name.clean() + '.' + columnPair.foreign + ' = ' + datatable.name + '.' + columnPair.base + ' AND ';
   });
   result.sql = result.sql.substring(0, result.sql.length - 5) + '\r\n';
   });
   */
  result.sql += "WHERE\r\n";
  result.sql += '\t`' + datatable.dates[0] + '` BETWEEN \'%STARTDATE%\' AND \'%ENDDATE%\'\r\n';

  if (datatable.filter) {
    result.sql += '\tAND ' + datatable.filter + '\r\n';
  }

  result.sql += 'GROUP BY\r\n' + groupBy;
  result.sql = result.sql.substring(0, result.sql.length - 3) + '\r\n';
  result.sql += 'ORDER BY 1' + '\r\n';

  _.each(dimensionsUsedInSelect, function (d, i) {
    dimensionsUsedInSelect[i] = _dimensions.get(d);
  });

  result.dimensionsUsed = dimensionsUsedInSelect;
  result.metricsUsed = metrics;

  return result;
};

exports.list = list;
exports.get = get;
exports.minCacheDate = minCacheDate;
exports.maxCacheDate = maxCacheDate;
exports.flushCache = flushCache;
exports.basequery = basequery;