periodicjs.core.controller/archive/controller.js

Customizable CMS platform

/*
 * periodic
 * http://github.com/typesettin/periodic
 *
 * Copyright (c) 2014 Yaw Joseph Etse. All rights reserved.
 */

'use strict';

const xss = require('xss');
var fs = require('fs-extra'),
  path = require('path'),
  os = require('os'),
  async = require('async'),
  bcrypt = require('bcrypt'),
  str2json = require('string-to-json'),
  Utilities = require('periodicjs.core.utilities'),
  CoreMailer = require('periodicjs.core.mailer'),
  objectComparison = require('objectcompare'),
  str2json = require('string-to-json'),
  pluralize = require('pluralize'),
  moment = require('moment'),
  merge = require('utils-merge'),
  Bluebird = require('bluebird'),
  mongoId = require('valid-objectid'),
  appenvironment,
  CoreUtilities,
  Item,
  Category,
  Tag,
  User,
  Userrole,
  Contenttypes,
  useCache = function() {
    // console.log('useCache global.CoreCache.status',global.CoreCache.status);
    return (global.CoreCache && global.CoreCache.status === 'active');
  },
  useViewCache = function() {
    // console.log('useViewCache global.CoreCache.options.view_cache_status',global.CoreCache.options.view_cache_status);
    return (global.CoreCache && global.CoreCache.status === 'active' && global.CoreCache.options.view_cache_status === 'active');
  },
  useDataCache = function() {
    // console.log('useDataCache global.CoreCache.options.data_cache_status',global.CoreCache.options.data_cache_status);
    return (global.CoreCache && global.CoreCache.status === 'active' && global.CoreCache.options.data_cache_status === 'active');
  },
  html_xss_default_whitelist = Object.assign(xss.whiteList, {
    onIgnoreTagAttr: function(tag, name, value, isWhiteAttr) {
      if ((name.substr(0, 5) === 'data-')) {
        // escape its value using built-in escapeAttrValue function 
        return name + '="' + xss.escapeAttrValue(value) + '"';
      }
    }
  }, {
    onTagAttr: function(tag, name, value, isWhiteAttr) {
      if (['img', 'audio', 'video'].indexOf(tag) !== -1 && name.toLowerCase() === 'src') {
        return name + '="' + (value) + '"';
      }
    }
  }),
  logger, appSettings, theme, applocals, controllerOptions, mongoose, periodicResources;

/**
 * A core constructor that provides numerous controller helper functions.
 * @{@link https://github.com/typesettin/periodicjs.core.controller}
 * @author Yaw Joseph Etse
 * @constructor
 * @copyright Copyright (c) 2014 Typesettin. All rights reserved.
 * @license MIT
 * @requires module:fs-extra
 * @requires module:path
 * @requires module:periodicjs.core.utilities
 * @param {object} resources variable injection from resources from instantiated periodic express app
 */
var Controller = function(resources, options) {
  periodicResources = resources;
  logger = resources.logger;
  appSettings = resources.settings;
  theme = resources.settings.theme;
  controllerOptions = options;
  mongoose = resources.mongoose;
  appenvironment = appSettings.application.environment;
  Item = mongoose.model('Item');
  Contenttypes = mongoose.model('Contenttype');
  Category = mongoose.model('Category');
  Tag = mongoose.model('Tag');
  User = mongoose.model('User');
  Userrole = mongoose.model('Userrole');
  pluralize.addIrregularRule('data', 'datas');

  if (resources.app && resources.app.locals) {
    applocals = resources.app.locals;
  }
  CoreUtilities = new Utilities(resources);
};

String.prototype.capitalizeFirstLetter = function() {
  return this.charAt(0).toUpperCase() + this.slice(1);
};

var jsonCheck = function(req) {
  return (req.query.format === 'json' || req.params.ext === 'json' || path.extname(req.originalUrl) === '.json' || req.is('json') || req.params.callback || req.is('application/json'));
};

var formatJsonResponse = function(data) {
  return {
    result: 'success',
    data: {
      doc: data
    }
  };
};

/**
 * Gets the path to the view file specified, first look at custom theme views, then extension views, then default views
 * @param  {object}   options  extname, themename, themefileext - support custom theme files
 * @param  {Function} callback async callback
 * @return {Function}            async callback(err,viewname)
 */
Controller.prototype.getPluginViewDefaultTemplate = function(options, callback) {
  var extname = options.extname || '',
    themename = theme,
    viewname = options.viewname,
    themefileext = options.themefileext || options.viewfileext,
    themetemplatefile,
    exttemplatefile;
  themetemplatefile = (themename && themefileext) ? path.join(path.resolve(process.cwd(), './content/themes'), themename, 'views', viewname + '.' + themefileext) : false;
  exttemplatefile = (extname && themefileext) ? path.join(path.resolve(process.cwd(), './node_modules', extname), 'views', viewname + '.' + themefileext) : false;

  var getExtensionView = function(viewname, callback) {
    if (extname) {
      fs.open(exttemplatefile, 'r', function(err, fd) {
        if (fd) {
          fs.close(fd);
        }
        if (err) {
          callback(err, viewname, null);
        } else {
          callback(null, viewname, exttemplatefile);
        }
      });
    } else {
      callback('no extname', viewname, viewname);
    }
  };

  var getThemeView = function(viewname, callback) {
    if (theme) {
      fs.open(themetemplatefile, 'r', function(err, fd) {
        if (fd) {
          fs.close(fd);
        }
        if (err) {
          callback(err, viewname, null);
        } else {
          callback(null, viewname, themetemplatefile);
        }
      });
    } else {
      callback('no theme', viewname, viewname);
    }
  };


  getThemeView(viewname, function(err, defaultview, themeview) {
    if (err) {
      getExtensionView(defaultview, function(err, defaultview, extname) {
        if (err) {
          callback(null, defaultview);
        } else {
          callback(null, extname);
        }
      });
    } else {
      callback(null, themeview);
    }
  });
};

Controller.prototype.respondInKind = function(options, asyncCallback) {
  var req = options.req,
    res = options.res,
    err = options.err,
    responseData = options.responseData || {},
    callback = options.callback || asyncCallback;
  responseData.periodic = responseData.periodic || {};
  responseData.periodic.version = appSettings.version;
  responseData.periodic.name = appSettings.name;
  responseData.request = {
    query: req.query,
    params: req.params,
    baseurl: req.baseUrl,
    originalurl: req.originalUrl,
    parsed: req._parsedUrl,
    'x-forwarded-for': req.headers['x-forwarded-for'],
    remoteAddress: req.connection.remoteAddress,
    referer: req.headers.referer,
    originalUrl: req.originalUrl,
    headerHost: req.headers.host
  };
  if (err) {
    responseData.status = 'error';
    responseData.result = 'error';
    responseData.data = {
      error: err
    };
  }

  if (path.extname(req.originalUrl) === '.html' || req.is('html') || req.is('text/html') || path.extname(req.originalUrl) === '.htm') {
    // console.log('responding because format is html');
    callback(req, res, responseData);
  } else if (jsonCheck(req)) {
    if (periodicResources.settings.sessions.enabled && req.session) {
      responseData.flash_messages = req.flash();
    }
    if (req.query.callback) {
      res.jsonp(responseData);
    } else {
      res.send(responseData);
    }
  } else if (options.redirecturl) {
    res.redirect(options.redirecturl);
    // console.log('req.flash()',req.flash());
  } else {
    callback(req, res, responseData);
  }
};

Controller.prototype.logError = function(options) {
  var err = options.err,
    req = options.req,
    errormeta = {
      err: err
    },
    userobject = {};
  if (req) {
    errormeta.ipinfo = {
      'x-forwarded-for': req.headers['x-forwarded-for'],
      remoteAddress: req.connection.remoteAddress,
      referer: req.headers.referer,
      originalUrl: req.originalUrl,
      headerHost: req.headers.host,
      osHostname: os.hostname()
    };
    userobject = req.user || req.body;
  }
  if (userobject.email || userobject.username) {
    errormeta.ipinfo.user = userobject.email || userobject.username;
  }
  logger.error(err.message, err.stack, errormeta);
};

Controller.prototype.logWarning = function(options) {
  var err = options.err,
    req = options.req,
    errormeta = {
      err: err
    },
    userobject = {};
  if (req) {
    errormeta.ipinfo = {
      'x-forwarded-for': req.headers['x-forwarded-for'],
      remoteAddress: req.connection.remoteAddress,
      referer: req.headers.referer,
      originalUrl: req.originalUrl,
      headerHost: req.headers.host,
      osHostname: os.hostname()
    };
    userobject = req.user || req.body;
  }
  if (userobject.email || userobject.username) {
    errormeta.ipinfo.user = userobject.email || userobject.username;
  }
  logger.warn(err.message, err.stack, errormeta);
};

/**
 * default response handler for express views, or will redirect to another request
 * @param  {object} options res,req,redirecturl,err,callback,responseData - this is what's sent to the rendered template view, also appends http request information like base url, query string parameters, etc
 * @return {object} response object render or callback
 */
Controller.prototype.handleDocumentQueryRender = function(options, asyncCallback) {

  var res = options.res,
    req = options.req,
    redirecturl = options.redirecturl,
    err = options.err,
    callback = options.callback || asyncCallback,
    renderview = options.renderView,
    responseData = options.responseData,
    cachetype,
    cachekey,
    cachedView,
    useCacheTest = (useViewCache() && req.headers.periodicCache !== 'no-periodic-cache'),
    renderResponseData = function(rendererror) {
      // console.log('this is rendererror',rendererror);
      if (rendererror) {
        err = rendererror;
      }
      if (err) {
        this.handleDocumentQueryErrorResponse({
          res: res,
          req: req,
          err: err,
          callback: callback,
          redirecturl: redirecturl
        });
      } else {
        Controller.prototype.respondInKind({
          req: req,
          res: res,
          responseData: responseData,
          callback: function(req, res, responseData) {
            if (!responseData.flash_messages && periodicResources.settings.sessions.enabled && req.session) {
              responseData.flash_messages = req.flash();
            }
            res.render(renderview, responseData, function(err, renderedview) {
              if (err) {
                if (callback) {
                  callback(err);
                } else {
                  Controller.prototype.logError({
                    err: err,
                    req: req
                  });
                  res.status(500);
                  res.render(appSettings.customThemeView, {
                    message: err.message,
                    error: err
                  });
                  res.end();
                }
              } else if (useCacheTest && global.CoreCache) {
                global.CoreCache.ViewCache.set({ key: cachekey, val: renderedview }, function(err) {
                  if (err) {
                    Controller.prototype.logError({
                      err: err,
                      req: req
                    });
                    res.status(500);
                    res.render(appSettings.customThemeView, {
                      message: err.message,
                      error: err
                    });
                    res.end();
                  } else {
                    logger.silly('saved cached ', cachekey);
                    if (callback) {
                      callback(null, renderedview);
                    } else {
                      res.send(renderedview);
                    }
                  }
                });
              } else if (callback) {
                callback(null, renderedview);
              } else {
                res.send(renderedview);
              }
            });
          }
        });
      }
    }.bind(this),
    getPageView = function(err) {
      if (err) {
        renderResponseData(err);
      } else if (applocals) {
        async.applyEach(
          applocals.additionalHTMLFunctions, { req: req, res: res },
          function(asyncerr) {
            if (asyncerr) {
              this.handleExceptionError(asyncerr, req, res);
            } else {
              renderResponseData();
            }
          }.bind(this));
      } else {
        renderResponseData();
      }
    }.bind(this);

  if (req.session && req.session.themename && renderview && theme) {
    renderview = renderview.replace(theme, req.session.themename);
  }

  if (useCacheTest && global.CoreCache) {
    cachetype = global.CoreCache.ViewCache.type;
    cachekey = global.CoreCache.generateKeyFromRequestUrl(cachetype, req.originalUrl);
    cachedView = global.CoreCache.ViewCache.get({ key: cachekey }, function(err, cachedViewData) {
      if (cachedViewData) {
        logger.silly('cache hit', cachekey);
        res.set('X-Periodic-Cache', 'hit');
        res.send(cachedViewData);
      } else {
        logger.silly('cache miss', cachekey);
        res.set('X-Periodic-Cache', 'miss');
        getPageView();
      }
    });
  } else {
    getPageView(err);
  }
};

Controller.prototype.handleExceptionError = function(err, req, res) {
  if (req.xhr) {
    res.send(500, {
      error: 'Something blew up!'
    });
  } else {
    res.status(500);
    // if(appconfig.settings().theme)
    res.render('home/error500', {
      message: err.message,
      error: err
    });
  }
};

/**
 * default response handler for error, or will redirect with flash error set
 * @param  {object} options err,req,res,callback
 * @return {object} response object render or callback
 */
Controller.prototype.handleDocumentQueryErrorResponse = function(options) {
  var err = options.err,
    errormessage,
    redirecturl = options.redirecturl,
    req = options.req,
    res = options.res,
    loggerFunction = logger.error,
    callback = options.callback; //,
  // errorFlashMessage = (options.errorflash) ? options.errorflash : errormessage;

  if (typeof options.use_warning === 'boolean' && options.use_warning === true) {
    loggerFunction = logger.warn;
  } else if (typeof options.use_warning === 'string') {
    loggerFunction = logger[options.use_warning];
  }

  if (typeof options.err === 'string') {
    errormessage = options.err;
  } else if (typeof options.err.toString() === 'string') {
    errormessage = options.err.toString();
  } else {
    errormessage = options.err.message;
  }
  // else if (typeof options.err === 'string' || typeof options.err.toString() === 'string') ? options.err.toString() : options.err.message


  if (res.statusCode !== 200 && res.statusCode !== 400) {
    res.status(res.statusCode);
  } else {
    res.status(400);
  }

  // console.log('errormessage',errormessage,typeof errormessage);
  if (res.statusCode === 404) {
    logger.debug(errormessage, req.url);
  } else {
    if (err.stack) {
      var userobject = {},
        userfieldtouse = '';
      if (req) {
        userobject = req.user || req.body;
      }
      if (userobject.email || userobject.username) {
        userfieldtouse = userobject.email || userobject.username;
      }

      loggerFunction(errormessage, err, {
        ipinfo: {
          'x-forwarded-for': req.headers['x-forwarded-for'],
          originalUrl: req.originalUrl,
          remoteAddress: req.connection.remoteAddress,
          referer: req.headers.referer,
          headerHost: req.headers.host,
          osHostname: os.hostname(),
          user: userfieldtouse
        }
      });
    }
  }

  var responseErrorData = {
    'result': 'error',
    'data': {
      error: errormessage
    }
  };
  if (jsonCheck(req)) {
    if (periodicResources.settings.sessions.enabled && req.session) {
      responseErrorData.flash_messages = req.flash();
    }
    if (req.query.callback) {
      res.jsonp(responseErrorData);
    } else {
      res.send(responseErrorData);
    }
  } else {
    if (options.errorflash !== false && periodicResources.settings.sessions.enabled && req.session) {

      req.flash('error', errormessage);
    }
    if (callback) {
      callback();
    } else if (redirecturl) {
      res.redirect(redirecturl);
    } else {
      var self = this;
      self.getPluginViewDefaultTemplate({
          viewname: 'home/error404',
          themefileext: appSettings.templatefileextension
        },
        function(err, templatepath) {
          self.handleDocumentQueryRender({
            res: res,
            req: req,
            renderView: templatepath,
            responseData: {
              pagedata: {
                title: 'Not Found'
              },
              user: req.user,
              url: req.url
            }
          });
        }
      );
    }
  }
};

/**
 * short hand method for rendering views
 * @param  {object} req          [description]
 * @param  {object} res          [description]
 * @param  {object} viewtemplate [description]
 * @param  {object} viewdata     [description]
 * @return {object}              [description]
 */
Controller.prototype.renderView = function(req, res, viewtemplate, viewdata) {
  this.getPluginViewDefaultTemplate(viewtemplate,
    function(err, templatepath) {
      this.handleDocumentQueryRender({
        res: res,
        req: req,
        err: err,
        renderView: templatepath,
        responseData: viewdata
      });
    }.bind(this)
  );
};

var clearModelDocCache = function(options, asynccallback) {
  var cachetype,
    req = options.req,
    doc = options.doc,
    model = options.model,
    datacachetype = global.CoreCache.DataCache.type,
    datacachekey_byid,
    datacachekey_byname,
    asyncDeleteFunctions = [],
    deleteKeysArray = [],
    deleteViewKeyFunction = function(delkey) {
      return function(cb) {
        global.CoreCache.ViewCache.del({ key: delkey }, cb);
      };
    },
    getKeyURLFromCacheModelOptions = function(modelroute, id) {
      var delkey = '/' + path.join(modelroute.replace(':id', id));
      return global.CoreCache.generateKeyFromRequestUrl(cachetype, delkey);
    },
    deleteDataCache = function() {
      if (doc.id) {
        datacachekey_byid = global.CoreCache.generateKeyFromRequestUrl(datacachetype, model.modelName + '+' + doc.id);
        global.CoreCache.DataCache.del({ key: datacachekey_byid }, function(err, status) {
          logger.silly('deleted memory cache', datacachekey_byid, status);
        });
      }
      if (doc.name) {
        datacachekey_byname = global.CoreCache.generateKeyFromRequestUrl(datacachetype, model.modelName + '+' + doc.name);

        global.CoreCache.DataCache.del({ key: datacachekey_byname }, function(err, status) {
          logger.silly('deleted memory cache', datacachekey_byname, status);
        });
      }
      if (model.modelName && (doc.id || doc.name)) {
        global.CoreCache.DataCache.del({
          model_name: model.modelName.toLowerCase(),
          model_name_plural: pluralize(model.modelName.toLowerCase()),
          docid: doc.id,
          docname: doc.name,
        }, function(err, status) { logger.silly('viewCache Del', status) });
      }
    },
    deleteViewCache = function(options) {
      // console.log('deleteViewCache options',options);
      var doccache = [`${model.modelName.toLowerCase()}/:id`], //options.doccache,
        doclistcache = [`${ pluralize(model.modelName.toLowerCase())}`], //options.doclistcache,
        docid = options.docid,
        docname = options.docname; //,
      // reqdoc = options.reqdoc;

      if (doccache && doccache.length > 0 /* && reqdoc */ ) {
        for (var gcid in doccache) {
          if (docname) {
            deleteKeysArray.push(getKeyURLFromCacheModelOptions(doccache[gcid], docname));
          }
          deleteKeysArray.push(getKeyURLFromCacheModelOptions(doccache[gcid], docid));
        }
      }
      if (doclistcache && doclistcache.length > 0 && docname /* && reqdoc */ ) {
        for (var gcil in doclistcache) {
          deleteKeysArray.push(getKeyURLFromCacheModelOptions(doclistcache[gcil], docname));
        }
      }
      if (req) {
        deleteKeysArray.push(global.CoreCache.generateKeyFromRequestUrl(cachetype, req.originalUrl));
        deleteKeysArray.push(global.CoreCache.generateKeyFromRequestUrl(cachetype, req.originalUrl.split('?')[0])); //strip out query string parameters from URL
        let baseURL = req.originalUrl.split(options.docid)[0];
        if (baseURL) {
          let cleanBaseURL = baseURL.replace(model.modelName.toLowerCase(), pluralize(model.modelName.toLowerCase()));
          //get listing page
          deleteKeysArray.push(global.CoreCache.generateKeyFromRequestUrl(cachetype, baseURL));
          deleteKeysArray.push(global.CoreCache.generateKeyFromRequestUrl(cachetype, cleanBaseURL));
        }
      }
      // console.log('model.modelName',model.modelName, 'docid',docid , 'docname',docname);
      if (model.modelName && (docid || docname)) {
        global.CoreCache.ViewCache.del({
          model_name: model.modelName.toLowerCase(),
          model_name_plural: pluralize(model.modelName.toLowerCase()),
          docid: docid,
          docname: docname,
        }, function(err, status) { logger.silly('viewCache Del', status) });
      }
    };
  cachetype = global.CoreCache.ViewCache.type;

  if (useDataCache()) {
    deleteDataCache();
  }
  if (useViewCache()) {
    deleteViewCache({
      doccache: global.CoreCache.options[model.modelName.toLowerCase() + '_doc_cache'],
      doclistcache: global.CoreCache.options[model.modelName.toLowerCase() + '_list_cache'],
      docid: doc.id,
      docname: doc.name //,
        // reqdoc : req.controllerData[model.modelName.toLowerCase()]
    });
  }
  if (deleteKeysArray.length > 0) {
    for (var y in deleteKeysArray) {
      asyncDeleteFunctions.push(deleteViewKeyFunction(deleteKeysArray[y]));
    }
    async.series(asyncDeleteFunctions, function(err) {
      if (err) {
        logger.error(err);
      }
      logger.silly('cleared cache for ', deleteKeysArray);
      if (asynccallback && typeof asynccallback === 'function') {
        asynccallback();
      }
    });
  }
};

/**
 * short hand mongoose load document query
 * @param  {object} options model,docid - id or name,callback,population -mongoose population, selection - mongoose selection
 * @return {Function} callback(err,document)
 */
Controller.prototype.loadModel = function(options, asyncCallback) {
  var model = options.model,
    docid = options.docid,
    docnamelookup = options.docnamelookup || 'name',
    sort = options.sort,
    callback = options.callback || asyncCallback,
    population = options.population,
    selection = options.selection,
    fields = options.fields,
    query,
    namequery = {},
    cached = (typeof options.cached === 'boolean' && options.cached === true) && true,
    useCacheTest = (useDataCache() && cached),
    cachetype = (global.CoreCache) ? global.CoreCache.DataCache.type : null,
    cachekey = (global.CoreCache) ? global.CoreCache.generateKeyFromRequestUrl(cachetype, model.modelName + '+' + docid) : null,
    cachedData,
    cachekeyexpirename = (options && options.model_type === 'sequelize') ? options.model_name.toLowerCase() + '_doc_cache_expires' : model.modelName.toLowerCase() + '_doc_cache_expires',
    // cachedDataDoc,
    queryCallback = function(err, doc) {
      if (useCacheTest && global.CoreCache && doc) {
        logger.debug('cachekeyexpirename', cachekeyexpirename, global.CoreCache.options[cachekeyexpirename]);
        global.CoreCache.DataCache.set({
          key: cachekey,
          val: doc,
          expires: global.CoreCache.options[cachekeyexpirename]
        }, function(err, expires) {
          if (err) {
            logger.error(err);
          }
          logger.silly('cached', cachekey, 'expires', expires);
        });

        callback(err, doc);
      } else {
        callback(err, doc);
      }
    },
    getDataQuery = function() {
      // console.log('query',query);
      if (options && options.model_type === 'sequelize') {
        let load_sql_function;
        let docnamequery = {};
        if (options.docnamelookup) {
          // console.log('population',population)
          docnamequery[options.docnamelookup] = docid;
          load_sql_function = options.sequelizeDB.models[options.model_name].findOne({ where: docnamequery, include: population.include });
        } else {
          load_sql_function = options.sequelizeDB.models[options.model_name].findById(docid, { include: population.include });
        }

        load_sql_function.then((doc) => {
            // console.log('sql doc',doc);
            queryCallback(null, doc);
          })
          .catch((e) => {
            queryCallback(e);
          });
      } else if (population) {
        model.findOne(query, fields).sort(sort).select(selection).populate(population).exec(queryCallback);
      } else {
        model.findOne(query, fields).sort(sort).select(selection).exec(queryCallback);
      }
    };
  namequery[docnamelookup] = docid;
  // console.log('loadModel cached',cached);

  if (mongoId.isValid(docid)) {
    query = {
      $or: [namequery, {
        _id: docid
      }]
    };
  } else if (options.searchusername) {
    query = {
      $or: [namequery, {
        username: docid
      }]
    };
  } else {
    query = namequery;
  }
  // console.log('options.cached',options.cached);
  // console.log('useCache',useCache);
  // console.log('useCacheTest',useCacheTest);
  if (useCacheTest && global.CoreCache) {
    cachedData = global.CoreCache.DataCache.get({
        key: cachekey
      },
      function(err, cachedDataDoc) {
        if (cachedDataDoc) {
          logger.silly('X-Periodic-Data-Cache', 'hit', cachekey);
          callback(null, cachedDataDoc);
        } else {
          logger.silly('X-Periodic-Data-Cache', 'miss', cachekey);
          getDataQuery();
        }
      });
  } else {
    getDataQuery();
  }
};

/**
 * short hand mongoose find by id query promisified
 * @param  {object} options model, docid, sort, population, selection, searchusername
 * options callback is automatically set to handle fulfillment of the async mongoose query
 * @return {object} chainable instance of bluebird promise
 */
Controller.prototype.loadModelPromisified = function(options) {
  return new Bluebird(function(resolve, reject) {
    var fulfiller = function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    };
    options.callback = fulfiller;
    Controller.prototype.loadModel(options);
  });
};

/**
 * short hand mongoose search documents query
 * @param  {object} options model,query - mongoose query,callback,population -mongoose population, selection - mongoose selection , limit, offset
 * @return {Function} callback(err,documents)
 */
Controller.prototype.searchModel = function(options, asyncCallback) {
  var model = options.model,
    query = options.query,
    sort = options.sort,
    offset = options.offset,
    fields = options.fields,
    selection = options.selection,
    limit = options.limit,
    callback = options.callback || asyncCallback,
    population = options.population,
    cached = (typeof options.cached === 'boolean' && options.cached === true) && true,
    useCacheTest = (useDataCache() && cached),
    cachetype = (global.CoreCache) ? global.CoreCache.DataCache.type : null,
    cachekey,
    cachedData,
    cachekeyexpirename = (options && options.model_type === 'sequelize') ? options.model_name.toLowerCase() + '_doc_cache_expires' : model.modelName.toLowerCase() + '_doc_cache_expires',
    queryDocsCallback = function(err, documents) {
      // console.log('queryDocsCallback err',err);
      // // console.log('queryDocsCallback err.stack',err.stack);
      // console.log('queryDocsCallback documents',documents);
      // // console.log('cachekeyexpirename',cachekeyexpirename,global.CoreCache.options[cachekeyexpirename]);
      if (useCacheTest && global.CoreCache && documents) {
        global.CoreCache.DataCache.set({
          key: cachekey,
          val: documents,
          expires: global.CoreCache.options[cachekeyexpirename]
        }, function(err, expires) {
          if (err) {
            logger.error(err);
          }
          logger.silly('cached', cachekey, 'expires', expires);
        });

        callback(err, documents);
      } else {
        callback(err, documents);
      }
    },
    getDocsDataQuery = function() {
      if (options && options.model_type && options.model_name && options.model_type === 'sequelize') {
        // console.log('search model query',query);
        model.findAll({
            where: query,
            offset: offset,
            limit: limit,
            raw: options.return_raw,
            include: (population.include) ? population.include : {},
          })
          .then((documents) => {
            // console.log('sql documents',documents);
            queryDocsCallback(null, documents);
          })
          .catch((e) => {
            queryDocsCallback(e);
          });
        // docnamelookup: options.docnamelookup,
        // model_type: options.model_type,
        // model_name: options.model_name,
        // sequelizeDB: options.sequelizeDB,
      } else if (population) {
        model.find(query, fields).sort(sort).select(selection).limit(limit).skip(offset).populate(population).exec(queryDocsCallback);
      } else {
        model.find(query, fields).sort(sort).select(selection).limit(limit).skip(offset).exec(queryDocsCallback);
      }
    };

  sort = (sort) ? sort : '-createdat';
  offset = (offset) ? offset : 0;
  limit = (limit) ? limit : 500;
  if (global.CoreCache) {
    let lowercase_model_name = (options && options.model_type === 'sequelize') ? options.model_name.toLowerCase() : model.modelName.toLowerCase();
    cachekey = global.CoreCache.generateKeyFromRequestUrl(cachetype, pluralize(lowercase_model_name) + '+' + JSON.stringify(query) + '+' + sort + '+' + offset + '+' + selection + '+' + limit + '+' + population);
  }

  if (useCacheTest && global.CoreCache) {
    cachedData = global.CoreCache.DataCache.get({
        key: cachekey
      },
      function(err, cachedDataDoc) {
        if (cachedDataDoc) {
          logger.silly('X-Periodic-Data-Cache', 'hit', cachekey);
          callback(null, cachedDataDoc);
        } else {
          logger.silly('X-Periodic-Data-Cache', 'miss', cachekey);
          getDocsDataQuery();
        }
      });
  } else {
    getDocsDataQuery();
  }
};

/**
 * short hand mongoose find query promisified
 * @param  {object} options model, query, sort, offset, selection, limit, population
 * options callback is automatically set to handle fulfillment of the async mongoose query
 * @return {object} chainable instance of bluebird promise
 */
Controller.prototype.searchModelPromisified = function(options) {
  return new Bluebird(function(resolve, reject) {
    var fulfiller = function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    };
    options.callback = fulfiller;
    Controller.prototype.searchModel(options);
  });
};

/**
 * short hand mongoose create document query
 * @param  {object} options model,newdoc - document to insert, req, res,callback, successredirect, appendid - append the id of newly created document on redirect
 * @return {object} responseData or redirected page
 */
Controller.prototype.createModel = function(options, asyncCallback) {
  var model = options.model,
    newdoc = options.newdoc,
    req = options.req,
    res = options.res,
    callback = options.callback || asyncCallback,
    successredirect = options.successredirect,
    appendid = options.appendid,
    responseData = {},
    cached = (typeof options.cached === 'boolean' && options.cached === true) && true,
    useCacheTest = ((useDataCache() || useViewCache()) && cached);
  var xss_white_list = options.xss_white_list || html_xss_default_whitelist;
  // console.log('useCacheTest',useCacheTest);
  // console.log('useDataCache()',useDataCache());
  // console.log('useViewCache()',useViewCache());

  if (req && req.controllerData && req.controllerData.html_xss) {
    newdoc = JSON.parse(xss(JSON.stringify(newdoc), xss_white_list));
  } else if (req && req.controllerData && !req.controllerData.skip_xss) {
    var regex = /(<([^>]+)>)/ig;
    newdoc = JSON.parse(JSON.stringify(newdoc).replace(regex, ''));
  }

  model.create(newdoc, function(err, saveddoc) {
    if (err) {
      if (callback) {
        callback(err, saveddoc);
      } else {

        this.handleDocumentQueryErrorResponse({
          err: err,
          errorflash: err.message,
          res: res,
          req: req
        });
      }
    } else {
      if (callback) {
        if (arguments.length >= 2) {
          delete arguments['0'];
          callback(null, arguments['1']);
        } else {
          callback(null, saveddoc);
        }
      } else if (jsonCheck(req)) {
        if (periodicResources.settings.sessions.enabled && req.session) {
          req.flash('success', 'Saved');
        }
        Controller.prototype.respondInKind({
          req: req,
          res: res,
          responseData: formatJsonResponse(saveddoc)
        });
      } else if (appendid) {
        if (periodicResources.settings.sessions.enabled && req.session) {
          req.flash('success', 'Saved');
        }
        res.redirect(successredirect + saveddoc._id);
      } else {
        if (periodicResources.settings.sessions.enabled && req.session) {
          req.flash('success', 'Saved');
        }
        res.redirect(successredirect);
      }

      if (useCacheTest) {
        // console.log('createModel clearModelDocCache');
        clearModelDocCache({ model: model, doc: saveddoc });
      }
    }
  }.bind(this));
};

/**
 * short hand mongoose create query promisified
 * @param  {object} options model, newdoc, req, res
 * options callback is automatically set to handle fulfillment of the async mongoose query
 * @return {object} chainable instance of bluebird promise
 */
Controller.prototype.createModelPromisified = function(options) {
  return new Bluebird(function(resolve, reject) {
    options.callback = function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    };
    Controller.prototype.createModel(options);
  });
};

var getIdFromPopulatedObject = function(object) {
  if (object && typeof object === 'object' && object._id) {
    return object._id.toString();
  } else {
    return object;
  }
};

var depopulateArray = function(array) {
  var depopulatedArray = [];
  for (var i = 0; i < array.length; i++) {
    if (typeof array[i] === 'object' && typeof array[i]._id === 'undefined') {
      depopulatedArray[i] = Controller.prototype.depopulate(array[i]);
    } else {
      depopulatedArray[i] = getIdFromPopulatedObject(array[i]);
    }
  }
  return depopulatedArray;
};

var depopulate = function(object) {
  var depopulatedObject = {};
  for (var prop in object) {
    if (Array.isArray(object[prop])) {
      depopulatedObject[prop] = depopulateArray(object[prop]);
    } else if (object[prop] !== null && typeof object[prop] === 'object' && typeof object[prop]._id === 'undefined' && new Date(object[prop]).toString() === 'Invalid Date') {
      depopulatedObject[prop] = depopulate(object[prop]);
    } else {
      depopulatedObject[prop] = getIdFromPopulatedObject(object[prop]);
    }
  }
  return depopulatedObject;
};

Controller.prototype.depopulate = depopulate;

var compareHasDifferentValues = function(originalValue, revisionValue) {
  var checkrevdate = new Date(revisionValue).toString();
  var checkoridate = new Date(originalValue).toString();

  if (originalValue !== null && revisionValue !== null && typeof originalValue !== 'undefined' && typeof revisionValue !== 'undefined' && originalValue.toString() === revisionValue.toString()) {
    // console.log('string vals are the same',originalValue,revisionValue);
    return false;
  } else if (checkrevdate !== 'Invalid Date' && checkoridate !== 'Invalid Date' && checkrevdate === checkoridate &&
    moment(revisionValue).isValid() &&
    moment(originalValue).isValid() &&
    moment(revisionValue).isSame(originalValue)) {
    // console.log('dates are the same',checkoridate,checkrevdate);
    return false;
  } else if (originalValue === null && (revisionValue === '' || revisionValue === null || typeof revisionValue === 'undefined')) {
    // console.log('originalValue empty vals are the same',originalValue,revisionValue);
    return false;
  } else if (revisionValue === null && (originalValue === '' || originalValue === null || typeof originalValue === 'undefined')) {
    // console.log('revisionValue empty vals are the same',originalValue,revisionValue);
    return false;
  } else {
    return true;
  }
};

var returnObjectDifference = function(original, revision, skipDepopulate) {
  var changesetdata = {};
  if (original.toJSON && typeof original.toJSON === 'function') {
    original = original.toJSON();
  } else if (original.toObject && typeof original.toObject === 'function') {
    original = original.toObject();
  }
  if (revision.toJSON && typeof revision.toJSON === 'function') {
    revision = revision.toJSON();
  } else if (revision.toObject && typeof revision.toObject === 'function') {
    revision = revision.toObject();
  }

  if (original.changes) {
    delete original.changes;
  }
  if (revision.changes) {
    delete revision.changes;
  }
  if (typeof original.__v !== 'undefined') {
    delete original.__v;
  }
  if (typeof revision.__v !== 'undefined') {
    delete revision.__v;
  }
  if (original.createdat) {
    delete original.createdat;
  }
  if (revision.createdat) {
    delete revision.createdat;
  }
  if (original._id) {
    delete original._id;
  }
  if (revision._id) {
    delete revision._id;
  }
  if (original.random) {
    delete original.random;
  }
  if (revision.random) {
    delete revision.random;
  }
  if (!skipDepopulate) {
    revision = depopulate(revision);
    original = depopulate(original);
  }

  var compareresult = objectComparison(original, revision),
    differences = {};
  // console.log('compareresult',compareresult);
  if (compareresult.equal === false) {
    for (var props in compareresult.differences) {
      // console.log('compareresult.differences props',props);
      if (compareHasDifferentValues(compareresult.differences[props].firstValue, compareresult.differences[props].secondValue)) {
        differences[props] = compareresult.differences[props].firstValue;
      }
    }
  }
  // console.log('differences',differences);
  changesetdata = str2json.convert(differences);
  // console.log('revision',revision);
  // console.log('original',original);
  console.log('changesetdata', changesetdata);
  return changesetdata;
};

Controller.prototype.updateMultipleModel = function(options) {
  logger.warn('cached documents not clearing and revisions on multple documents not saved');
  var model = options.model,
    // id = options.id,
    updatequery = options.updatequery,
    updateattributes = options.updateattributes,
    req = options.req,
    res = options.res,
    callback = options.callback,
    successredirect = options.successredirect,
    // originalrevision = options.originalrevision,
    // appendid = options.appendid,
    responseData = {}; //,
  // updateOperation,
  // cached = (typeof options.cached === 'boolean' && options.cached===true) && true,
  // useCacheTest = (useCache() && global.CoreCache && cached && req.headers.periodicCache!=='no-periodic-cache');

  model.update(updatequery, updateattributes, { multi: true }, function(err, numAffected) {
    if (err) {
      if (callback) {
        callback(err, null);
      } else {
        this.handleDocumentQueryErrorResponse({
          err: err,
          errorflash: err.message,
          res: res,
          req: req
        });
      }
    } else {
      if (callback) {
        callback(null, numAffected);
      } else if (jsonCheck(req)) {
        if (periodicResources.settings.sessions.enabled && req.session) {
          req.flash('success', 'Saved');
        }
        Controller.prototype.respondInKind({
          req: req,
          res: res,
          responseData: formatJsonResponse(numAffected)
        });
      } else {
        if (periodicResources.settings.sessions.enabled && req.session) {
          req.flash('success', 'Saved');
        }
        res.redirect(successredirect);
      }

      // if(useCacheTest){
      //  updatedoc.id = id;
      //  clearModelDocCache({model:model,doc:updatedoc,req:req});
      // }
    }
  });
};

/**
 * short hand mongoose update document query
 * @param  {object} options model, id - objectid of mongoose document,updatedoc - document to update, req, res,callback, successredirect, appendid - append the id of newly created document on redirect, removefromarray - sets the update operation to manipulate an array of documents with mongo $pull, appendArray - sets the update operation to manipulate an array of documents with mongo $push, saverevision - save revisions
 * @return {object} responseData or redirected page
 */
Controller.prototype.updateModel = function(options, asyncCallback) {
  if (!options.id) {
    //Added to enforce mongo id
    throw new Error('Update is missing mongo id');
  }
  var model = options.model,
    id = options.id,
    updatedoc = options.updatedoc,
    req = options.req,
    res = options.res,
    callback = options.callback || asyncCallback,
    successredirect = options.successredirect,
    originalrevision = options.originalrevision,
    skipDepopulate = options.skipDepopulate,
    appendid = options.appendid,
    responseData = {},
    updateOperation,
    cached = (typeof options.cached === 'boolean' && options.cached === true) && true,
    useCacheTest = ((useDataCache() || useViewCache()) && global.CoreCache && cached && req.headers.periodicCache !== 'no-periodic-cache');
  var xss_white_list = options.xss_white_list || html_xss_default_whitelist;


  if (req && req.controllerData && req.controllerData.html_xss) {
    updatedoc = JSON.parse(xss(JSON.stringify(updatedoc), xss_white_list));
  } else if (req && req.controllerData && !req.controllerData.skip_xss) {
    var regex = /(<([^>]+)>)/ig;
    updatedoc = JSON.parse(JSON.stringify(updatedoc).replace(regex, ''));
  }

  if (options.removeFromArray) {
    logger.silly('removing array in doc');
    updateOperation = {
      $pull: updatedoc
    };
  } else if (options.appendArray) {
    logger.silly('appending array in doc');
    updateOperation = {
      $push: updatedoc
    };
  } else {
    updatedoc.updatedat = new Date();
    logger.silly('updating entire doc');
    updateOperation = {
      $set: updatedoc
    };
  }

  if (options.forceupdate) {
    model.update({
      _id: id
    }, updateOperation, function(err, numAffected) {
      if (err) {
        if (callback) {
          callback(err, null);
        } else {
          this.handleDocumentQueryErrorResponse({
            err: err,
            errorflash: err.message,
            res: res,
            req: req
          });
        }
      } else {
        if (callback) {
          callback(null, numAffected);
        } else if (jsonCheck(req)) {
          if (periodicResources.settings.sessions.enabled && req.session) {
            req.flash('success', 'Saved');
          }
          Controller.prototype.respondInKind({
            req: req,
            res: res,
            responseData: numAffected
          });
        } else {
          if (periodicResources.settings.sessions.enabled && req.session) {
            req.flash('success', 'Saved');
          }
          res.redirect(successredirect);
        }

        if (useCacheTest) {
          // console.log('updateModel clearModelDocCache');
          updatedoc.id = id;
          clearModelDocCache({ model: model, doc: updatedoc, req: req });
        }
      }
    });
  } else {
    // console.log('before tried to save', updateOperation);
    // console.log('options.saverevision ',options.saverevision );
    model.findByIdAndUpdate(id, updateOperation, function(err, saveddoc) {
      // console.log('tried to save', updateOperation,err, saveddoc);
      // console.log('saveddoc', saveddoc);
      if (err) {
        if (callback) {
          callback(err, null);
        } else {
          this.handleDocumentQueryErrorResponse({
            err: err,
            errorflash: err.message,
            res: res,
            req: req
          });
        }
      } else {
        if (callback) {
          callback(null, saveddoc);
        } else if (jsonCheck(req)) {
          if (periodicResources.settings.sessions.enabled && req.session) {
            req.flash('success', 'Saved');
          }
          if (options.population) {
            model.findOne({
              _id: saveddoc._id
            }).populate(options.population).exec(function(err, popdoc) {
              if (err) {
                responseData.data.docpopulationerror = err;
                responseData.data.status = 'couldnt populate';
                responseData.data.doc = popdoc;
                Controller.prototype.respondInKind({
                  req: req,
                  res: res,
                  responseData: responseData
                });
              } else {
                Controller.prototype.respondInKind({
                  req: req,
                  res: res,
                  responseData: formatJsonResponse(saveddoc)
                });
              }
            });
          } else {
            Controller.prototype.respondInKind({
              req: req,
              res: res,
              responseData: formatJsonResponse(saveddoc)
            });
          }
        } else if (appendid) {
          if (periodicResources.settings.sessions.enabled && req.session) {
            req.flash('success', 'Saved');
          }
          res.redirect(successredirect + saveddoc._id);
        } else {
          if (periodicResources.settings.sessions.enabled && req.session) {
            req.flash('success', 'Saved');
          }
          res.redirect(successredirect);
        }
        //refresh cache
        if (useCacheTest) {
          updatedoc.id = id;
          clearModelDocCache({ model: model, doc: updatedoc, req: req });
        }
        //save revision
        if (options.saverevision && originalrevision) {
          var changesetdata = updatedoc,
            changesetdiff;
          delete changesetdata.docid;
          delete changesetdata._csrf;
          delete changesetdata.save_button;
          delete changesetdata.changes;
          delete originalrevision.changes;
          delete changesetdata._wysihtml5_mode;
          delete changesetdata.doctypename;
          delete changesetdata.doctypenamelink;
          delete changesetdata.time;
          delete changesetdata.searchdocumentsbutton;
          delete changesetdata.date;
          delete changesetdata.mediafiles;

          changesetdiff = returnObjectDifference(originalrevision, changesetdata, skipDepopulate);
          if (Object.keys(changesetdiff).length > 0) {
            model.findByIdAndUpdate(
              id, {
                $push: {
                  'changes': {
                    changeset: changesetdiff,
                    editor: (req && req.user && req.user._id) ? req.user._id : 'invalid_user_id',
                    editor_username: (req && req.user && req.user.username) ? req.user.username : 'invalid_user_username'
                  }
                }
              }, // // {safe: true, upsert: true},
              function(err) {
                if (err) {
                  logger.error(err);
                }
              }
            );
          } else {
            logger.silly('no changes to save');
          }
        }
      }
    }.bind(this));
  }
};

/**
 * short hand mongoose update promisified
 * @param  {object} options model, id, updatedoc
 * options callback is automatically set to handle fulfillment of the async mongoose query
 * @return {object} chainable instance of bluebird promise
 */
Controller.prototype.updateModelPromisified = function(options) {
  return new Bluebird(function(resolve, reject) {
    options.callback = function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    };
    Controller.prototype.updateModel(options);
  });
};

/**
 * short hand mongoose delete document query
 * @param  {object} options model,deleteid - id to delete,callback
 * @return {Function} callback(err)
 */
Controller.prototype.deleteModel = function(options, asyncCallback) {
  var model = options.model,
    deleteid = options.deleteid,
    req = options.req,
    // res = options.res,
    callback = options.callback || asyncCallback,
    cached = (typeof options.cached === 'boolean' && options.cached === true) && true,
    useCacheTest = ((useDataCache() || useViewCache()) && cached && global.CoreCache);
  model.remove({
    _id: deleteid
  }, function(err, status) {
    if (useCacheTest) {
      // console.log('deleteModel clearModelDocCache');
      // updatedoc.id = id;
      clearModelDocCache({
        model: model,
        req: req,
        doc: {
          id: deleteid
        }
      }, callback);
    } else {
      if (callback && typeof callback === 'function') {
        callback();
      }
    }
  });
};

/**
 * short hand mongoose remove promisified
 * @param  {object} options model, deleteid, cached
 * options callback is automatically set to handle fulfillment of the async mongoose query
 * @return {object} c