@sap/dwf-generator/generator/generator.js

SAP HANA Data Warehousing Foundation - Generator

'use strict';

const coFs = require('fs');
const coLogging = require('@sap/logging');
const coPath = require('path');
const coAsync = require('async');
const coDefaultFolders = require('./const').defaultFolders;
const coGenerateFolders = require('./generate_folders');
const coEditDlmProfile = require('./dlm_profile/edit_dlm_profile');
const coGlobalNodes = require('./dlm_profile/global_nodes');
const coConnections = require('./dlm_profile/connections');
const coRuntime = require('./dlm_profile/runtime');
const coGeneratePruning = require('./generate/generate_pruning');
const coGenerateUnion = require('./generate/generate_union');
const coGenerateCommon = require('./generate/generate_common');
const coGenerateProcedure = require('./generate/generate_procedure');
const coGenerateTable = require('./generate/generate_table');
const coGenerateTaskChain = require('./generate/generate_task_chain');

function Generator(ioPaths) {
  this.relativeToRoot = ioPaths.relativeToRoot;
  this.srcTargetFolder = coPath.join(ioPaths.hdb, coDefaultFolders.src, this.relativeToRoot);
  this.cfgTargetFolder = coPath.join(ioPaths.hdb, coDefaultFolders.cfg, this.relativeToRoot);
  this.taskChainFolder = coPath.join(ioPaths.dwf, coDefaultFolders.dwfGenerated, this.relativeToRoot);
  this.namePrefix = coPath.basename(this.relativeToRoot);
  this.srcNamePrefix = coPath.join(this.srcTargetFolder, coPath.sep);
  this.cfgNamePrefix = coPath.join(this.cfgTargetFolder, coPath.sep);
  this.profilePath = coPath.join(ioPaths.dwf, coDefaultFolders.dwfSrc,
    this.relativeToRoot + '.dwfdlmprofile');

  // logs
  let appContext = coLogging.createAppContext();
  this.log = appContext.getLogger('/DLM-GENERATOR/log');
  // note generator version in log
  this.moduleVersion = 'v' + require('../package.json').version;
  this.log.info(`generator version: ${this.moduleVersion}`);
  // fetch dlm profile
  this.dlmProfile = new coEditDlmProfile(this.profilePath, this.log);
  this.globalNodes = new coGlobalNodes(this.dlmProfile);
  // calculate namespace
  this.namespace;
  this.updateNamespace();
  // prepare generation
  // connect two nodes with a rule
  this.connections = coConnections.getConnections(this);
  // prepare name of views and procedures
  this.runtime = coRuntime.getRuntime(this.connections);
  // prepare paths of artefacts
  this.pathOfFiles = {};
  this.setPaths();
}

/**
 *
 *
 * @param {function} ifnCallback
 */
Generator.prototype.generateFiles = function () {

  function lfnCheckLastGeneration() {
    this.notOverwrite = !this.dlmProfile.get('generate.strategy.overwrite');
    // check whether the profile was modified since last generation
    try {
      let lsLastGenerationPath = this.dlmProfile.get('runtime.last_generation.target.path');
      if (coPath.relative(this.relativeToRoot, lsLastGenerationPath) != '' && !this.notOverwrite) {
        this.log.error(`The dwfdlmprofile ${
          this.profilePath} was modified, the last generation could not be overwrite`);
        throw new Error(`The dwfdlmprofile ${
          this.profilePath} was modified, the last generation could not be overwrite`);
      }
    } catch (ioError) {
      if (!ioError.message.match(/is not defined in the dlm profile/)) {
        this.log.error(ioError);
        throw ioError;
      }
    }
    arguments[arguments.length - 1](null);
  }

  function lfnCheckFolders() {
    coAsync.parallel([
      coAsync.asyncify(function () {
        // create folders in the target with the same name like the profile
        if (coGenerateFolders.checkFolder(this.srcTargetFolder, this.notOverwrite)) {
          throw new Error(`A folder in "src" for "${this.namePrefix
            }" already exists, this folder should not be overwritten`);
        }
      }.bind(this)),
      coAsync.asyncify(function () {
        if (coGenerateFolders.checkFolder(this.cfgTargetFolder, this.notOverwrite)) {
          throw new Error(`A folder in "cfg" for "${this.namePrefix
            }" already exists, this folder should not be overwritten`);
        }
      }.bind(this))
    ], arguments[arguments.length - 1]);
  }

  function lfnGenerate() {
    let laTasks = [];
    // create basic
    laTasks.push(coGenerateCommon.generateNameSpace.bind(this, this));
    if (typeof this.pathOfFiles.common.sparkAccess != 'undefined') {
      laTasks.push(coGenerateProcedure.prepareSparkAccess.bind(this, this));
    }
    // create artefacts for relocations
    for (let i in this.connections) {
      if (typeof this.connections[i].pathOfFiles.view.pruning != 'undefined') {
        // create pruning view
        laTasks.push(coGeneratePruning.generatePruningView.bind(this, this, this.connections[i]));
        // // create pruning table
        laTasks.push(coGeneratePruning.generatePruningTable.bind(this, this, this.connections[i]));
      }
      laTasks.push(coGenerateCommon.generateExecuteTable.bind(this, this, this.connections[i]));
      // create virtual tables or synonym
      laTasks.push(coGenerateTable.generateTables.bind(this, this, this.connections[i]));
      // create union view
      laTasks.push(coGenerateUnion.generateUnionView.bind(this, this, this.connections[i]));
      // create relocation procedure
      laTasks.push(coGenerateProcedure.generateRelocationProcedures.bind(this, this, this.connections[i]));
    }
    // runtime
    // generated task chain
    laTasks.push(coGenerateFolders.checkFolder.bind(this, this.taskChainFolder, this.notOverwrite));
    laTasks.push(coGenerateTaskChain.generateTaskChainDefault.bind(this, this));
    for (let i in laTasks) {
      laTasks[i] = coAsync.asyncify(laTasks[i]);
    }
    coAsync.parallel(laTasks, arguments[arguments.length - 1]);
  }

  function lfnModifyProfile() {
    // modify dlm profile
    this.dlmProfile.set('runtime', this.runtime);
    this.dlmProfile.set('runtime.last_generation.version', this.moduleVersion);
    this.dlmProfile.set('runtime.last_generation.create_at', new Date());
    // force Unix-style for path
    this.dlmProfile.set('runtime.last_generation.target.path', this.relativeToRoot.replace(/\\/g, '/'));
  }

  coAsync.waterfall([
    lfnCheckLastGeneration.bind(this),
    lfnCheckFolders.bind(this),
    lfnGenerate.bind(this)
  ], function (ioErr) {
    if (ioErr) {
      this.log.error(ioErr);
    } else {
      lfnModifyProfile.bind(this)();
      this.log.info(`Generation for ${this.profilePath} has been completed successfully`);
    }
  }.bind(this));

};



/**
 *
 * @param {object} ioData
 * @param {string} ioData.fileName
 * @param {string} ioData.fileContent
 */
Generator.prototype.createFile = function (ioData) {
  try {
    coFs.writeFileSync(ioData.fileName, ioData.fileContent);
    this.log.info(`"${ioData.fileName}" was created successfully.`);
  } catch (ioErr) {
    this.log.error(ioErr);
    process.exit(1);
  }
};

Generator.prototype.updateNamespace = function () {
  this.namespace = this.dlmProfile.get('generate.hdinamespace.name');
  if (this.dlmProfile.get('generate.hdinamespace.subfolder') == 'append') {
    let laParentFolders = this.relativeToRoot.replace('/', '\\').split('\\');
    this.namespace += `${this.namespace ? '.' : ''}dlm.${laParentFolders.join('.')}`;
  }
};

Generator.prototype._preparePathOfAcceccTable = function (ioNode) {
  let lsDataStoreType = ioNode.dataStore.type;
  let lsTableName = this.getAccessTableName(ioNode);
  switch (lsDataStoreType) {
    case 'iq':
    case 'vora':
    case 'spark':
      return {
        'virtualTable': `${this.srcNamePrefix}${lsTableName}.hdbvirtualtable`,
        'virtualTableConfig': `${this.cfgNamePrefix}${lsTableName}.hdbvirtualtableconfig`,
        'grants': `${this.cfgNamePrefix}${lsTableName}.hdbgrants`
      };
    case 'hana':
      {
        // local table or synonym
        let lsObjectType = ioNode.dataStore.object.type;
        if (!ioNode.dataStore.object[lsObjectType].schema)
          return {};
        return {
          'grant': `${this.cfgNamePrefix}${lsTableName}.hdbgrants`,
          'synonym': `${this.srcNamePrefix}${lsTableName}.hdbsynonym`,
          'synonymconfig': `${this.cfgNamePrefix}${lsTableName}.hdbsynonymconfig`
        };
      }
    default:
      break;
  }
};

Generator.prototype.setPaths = function () {
  // basic
  this.pathOfFiles.common = {
    srcNamespace: coPath.join(this.srcTargetFolder, '.hdinamespace'),
    cfgNamespace: coPath.join(this.cfgTargetFolder, '.hdinamespace')
  };
  // task chain
  this.pathOfFiles.taskChain = {
    default: coPath.join(this.taskChainFolder, 'data_movement.dwftaskchain')
  };
  // connections
  this.pathOfFiles.connections = [];
  for (let i in this.connections) {
    // virtual table or synonym
    let loSourcePaths = this._preparePathOfAcceccTable(this.connections[i].source);
    let loTargetPaths = this._preparePathOfAcceccTable(this.connections[i].target);
    let loConnection = {
      source: loSourcePaths, // source table
      target: loTargetPaths, // target table
      view: {
        union: `${this.srcNamePrefix}${this
          .getUnionViewName(this.connections[i])}.hdbview`
      },
      procedure: {
        relocation: {},
        edit_rule: {}
      }
    };
    // procedures
    if (this.connections[i].source.dataStore.type == 'iq' ||
      this.connections[i].target.dataStore.type == 'iq') {
      loConnection.procedure.relocation.executeTable
        = `${this.srcNamePrefix}${this.getExecuteTableContext(this.connections[i])}.hdbcds`;
    }
    let lsRuleType = this.connections[i].rule.type;
    if (typeof this.connections[i].rule[lsRuleType].source_to_target != 'undefined') {
      loConnection.procedure.relocation.source_to_target = `${this.srcNamePrefix}${this
        .getRelocationNameSourceToTarget(this.connections[i])}.hdbprocedure`;
    }
    if (typeof this.connections[i].rule[lsRuleType].target_to_source != 'undefined') {
      loConnection.procedure.relocation.target_to_source = `${this.srcNamePrefix}${this
        .getRelocationNameTargetToSource(this.connections[i])}.hdbprocedure`;
    }
    // pruning table
    if (this.connections[i].rule.type == 'pruning') {
      loConnection.view.pruning = {};
      loConnection.view.pruning.cds = `${this.srcNamePrefix}${this
        .getPruningViewName(this.connections[i])}_TABLE.hdbcds`;
      loConnection.view.pruning.csv = `${this.srcNamePrefix}${this
        .getPruningViewCsv(this.connections[i])}`;
      loConnection.view.pruning.tableData = `${this.srcNamePrefix}${this
        .getPruningViewName(this.connections[i])}_DATA.hdbtabledata`;
      loConnection.view.pruning.calcView = `${this.srcNamePrefix}${this
        .getPruningViewName(this.connections[i])}.hdbcalculationview`;
    }

    // access to spark
    if ((this.connections[i].source.dataStore.type == 'spark'
      || this.connections[i].target.dataStore.type == 'spark')
      && typeof this.pathOfFiles.common.sparkAccess == 'undefined') {
      this.pathOfFiles.common.sparkAccess = {
        grant: `${this.cfgNamePrefix}SPARK_ACCESS.hdbgrants`,
        synonym: `${this.srcNamePrefix}SPARK_ACCESS.hdbsynonym`,
        synonymconfig: `${this.cfgNamePrefix}SPARK_ACCESS.hdbsynonymconfig`
      };
    }
    this.pathOfFiles.connections.push(loConnection);
    this.connections[i].pathOfFiles = loConnection;
  }
};

Generator.prototype.get = function (isKey) {
  return this.dlmProfile.get(isKey);
};



Generator.prototype.isHotToCol = function (ioConnection) {
  let loTemperature = {
    'hana': 2,
    'dt': 1,
    'spark': 0,
    'iq': 0
  };
  let lsSourceDataStoreType = ioConnection.source.dataStore.type;
  let lsTargetDataStoreType = ioConnection.target.dataStore.type;
  if (loTemperature[lsSourceDataStoreType] > loTemperature[lsTargetDataStoreType]) {
    return true;
  }
  return false;
};

Generator.prototype.getAccessTableName = function (ioNode) {
  let lsDataStoreType = ioNode.dataStore.type;
  switch (lsDataStoreType) {
    case 'hana':
      {
        let lsObjectType = ioNode.dataStore.object.type;
        if (!ioNode.dataStore.object[lsObjectType].schema)
          return ioNode.dataStore.object[lsObjectType].object;
        return this.getSynonymName(ioNode);
      }
    case 'iq':
    case 'vora':
    case 'spark':
      return this.getVirtualTableName(ioNode);
  }
};

Generator.prototype.getVirtualTableName = function (ioNode) {
  let loDataStore = ioNode[ioNode.type];
  let lsObjectType = loDataStore.object.type;
  return `${loDataStore.type.toUpperCase()}_${loDataStore.object[lsObjectType]
    .object.split('.').pop().split('::').pop().toUpperCase()}_TRG`;
};

Generator.prototype.getSynonymName = function (ioNode) {
  let loDataStore = ioNode[ioNode.type];
  let lsObjectType = loDataStore.object.type;
  return `${loDataStore.object[lsObjectType]
    .object.split('.').pop().split('::').pop().toUpperCase()}_SYNONYM`;
};

Generator.prototype.getUnionViewName = function (ioConnection) {
  let loSourceNode = ioConnection.source;
  let loTargetNode = ioConnection.target;
  let lsSourceId = loSourceNode.id;
  let lsTargetId = loTargetNode.id;
  let lsSourceType = loSourceNode.dataStore.type.toUpperCase();
  let lsTargetType = loTargetNode.dataStore.type.toUpperCase();
  return `${lsSourceType}_${lsTargetType}_${lsSourceId}_${lsTargetId}_UNION`;
};

Generator.prototype.getPruningViewName = function (ioConnection) {
  let loSourceNode = ioConnection.source;
  let loTargetNode = ioConnection.target;
  let lsSourceId = loSourceNode.id;
  let lsTargetId = loTargetNode.id;
  let lsSourceType = loSourceNode.dataStore.type.toUpperCase();
  let lsTargetType = loTargetNode.dataStore.type.toUpperCase();
  return `${lsSourceType}_${lsTargetType}_${lsSourceId}_${lsTargetId}_PRUNING`;
};

Generator.prototype.getPruningViewTable = function (ioConnection) {
  return `${this.getPruningViewName(ioConnection)}_TABLE.TABLE`;
};

Generator.prototype.getPruningViewTableContext = function (ioConnection) {
  return `${this.getPruningViewName(ioConnection)}_TABLE`;
};

Generator.prototype.getPruningViewCsv = function (ioConnection) {
  return `${this.getPruningViewName(ioConnection)}_CSV.csv`;
};

Generator.prototype.getRelocationNameSourceToTarget = function (ioConnection) {
  let loSourceNode = ioConnection.source;
  let loTargetNode = ioConnection.target;
  let lsSourceId = loSourceNode.id;
  let lsTargetId = loTargetNode.id;
  let lsSourceType = loSourceNode.dataStore.type.toUpperCase();
  let lsTargetType = loTargetNode.dataStore.type.toUpperCase();
  return `${lsSourceType}_to_${lsTargetType}_${lsSourceId}_to_${lsTargetId}`;
};

Generator.prototype.getRelocationNameTargetToSource = function (ioConnection) {
  let loSourceNode = ioConnection.source;
  let loTargetNode = ioConnection.target;
  let lsSourceId = loSourceNode.id;
  let lsTargetId = loTargetNode.id;
  let lsSourceType = loSourceNode.dataStore.type.toUpperCase();
  let lsTargetType = loTargetNode.dataStore.type.toUpperCase();
  return `${lsTargetType}_to_${lsSourceType}_${lsTargetId}_to_${lsSourceId}`;
};

Generator.prototype.getExecuteTableContext = function (ioConnection) {
  let loSourceNode = ioConnection.source;
  let loTargetNode = ioConnection.target;
  let lsSourceId = loSourceNode.id;
  let lsTargetId = loTargetNode.id;
  let lsSourceType = loSourceNode.dataStore.type.toUpperCase();
  let lsTargetType = loTargetNode.dataStore.type.toUpperCase();
  return `${lsSourceType}_${lsTargetType}_${lsSourceId}_${lsTargetId}_Execute`;
};

Generator.prototype.getColumns = function (ioNode) {
  return ioNode.dataStore.object[ioNode.dataStore.object.type].column;
};

Generator.prototype.getKeys = function (ioNode) {
  return ioNode.dataStore.object[ioNode.dataStore.object.type].keys;
};

Generator.prototype.getObjectFullName = function (isObjectName) {
  if (!this.namespace || isObjectName.match('::'))
    return isObjectName;
  return `${this.namespace}::${isObjectName}`;
};

module.exports = Generator;