@itentialopensource/adapter-db_mongo/adapter.js

Itential adapter to connect to mongo

/* @copyright Itential, LLC 2019 (pre-modifications) */

// Set globals
/* global log */
/* eslint no-underscore-dangle: warn  */
/* eslint no-loop-func: warn */
/* eslint no-cond-assign: warn */
/* eslint no-unused-vars: warn */
/* eslint consistent-return: warn */
/* eslint no-param-reassign: warn */

/* Required libraries.  */
const fs = require('fs-extra');
const path = require('path');
const uuid = require('uuid');

/* The schema validator */
const AjvCl = require('ajv');

/* Fetch in the other needed components for the this Adaptor */
const EventEmitterCl = require('events').EventEmitter;

const { MongoClient } = require('mongodb');
const { ObjectId } = require('mongodb');

let myid = null;
let errors = [];

/**
 * @summary Build a standard error object from the data provided
 *
 * @function formatErrorObject
 * @param {String} origin - the originator of the error (optional).
 * @param {String} type - the internal error type (optional).
 * @param {String} variables - the variables to put into the error message (optional).
 * @param {Integer} sysCode - the error code from the other system (optional).
 * @param {Object} sysRes - the raw response from the other system (optional).
 * @param {Exception} stack - any available stack trace from the issue (optional).
 *
 * @return {Object} - the error object, null if missing pertinent information
 */
function formatErrorObject(origin, type, variables, sysCode, sysRes, stack) {
  log.trace(`${myid}-adapter-formatErrorObject`);

  // add the required fields
  const errorObject = {
    icode: 'AD.999',
    IAPerror: {
      origin: `${myid}-unidentified`,
      displayString: 'error not provided',
      recommendation: 'report this issue to the adapter team!'
    }
  };

  if (origin) {
    errorObject.IAPerror.origin = origin;
  }
  if (type) {
    errorObject.IAPerror.displayString = type;
  }

  // add the messages from the error.json
  for (let e = 0; e < errors.length; e += 1) {
    if (errors[e].key === type) {
      errorObject.icode = errors[e].icode;
      errorObject.IAPerror.displayString = errors[e].displayString;
      errorObject.IAPerror.recommendation = errors[e].recommendation;
    } else if (errors[e].icode === type) {
      errorObject.icode = errors[e].icode;
      errorObject.IAPerror.displayString = errors[e].displayString;
      errorObject.IAPerror.recommendation = errors[e].recommendation;
    }
  }

  // replace the variables
  let varCnt = 0;
  while (errorObject.IAPerror.displayString.indexOf('$VARIABLE$') >= 0) {
    let curVar = '';

    // get the current variable
    if (variables && Array.isArray(variables) && variables.length >= varCnt + 1) {
      curVar = variables[varCnt];
    }
    varCnt += 1;
    errorObject.IAPerror.displayString = errorObject.IAPerror.displayString.replace('$VARIABLE$', curVar);
  }

  // add all of the optional fields
  if (sysCode) {
    errorObject.IAPerror.code = sysCode;
  }
  if (sysRes) {
    errorObject.IAPerror.raw_response = sysRes;
  }
  if (stack) {
    errorObject.IAPerror.stack = stack;
  }

  // return the object
  return errorObject;
}

/**
 * @summary Parses the inputted data for the trigger "<op>" and replaces it with "$"
 *
 * @function convertOperatorTriggers
 * @param {AnyData} data - the object to check for triggers within (required)
 *
 * @return {Object} - the object with all triggers replaced with "$"
 */
function convertOperatorTriggers(data) {
  try {
    if (typeof data !== 'string') {
      if (Object.keys(data).length === 0) { // data is an empty object
        return {};
      }
      data = JSON.stringify(data);
    }
    const regex = /{op}/gi;
    return JSON.parse(data.replace(regex, '$$'));
  } catch (err) {
    log.error(err);
    return data;
  }
}

/**
 * @summary Converts ObjectIds
 *
 * @function convertObjectId
 * @param {AnyData} data - the data to check for ObjectId (required)
 *
 * @return {Object} - the resulting ObjectId
 */
function convertObjectId(data) {
  try {
    if (typeof data === 'string' && data.indexOf('ObjectId') === 0) {
      const oid = data.substring(10, data.length - 2);
      log.debug(`OID ${oid}`);
      if (ObjectId.isValid(oid)) {
        return new ObjectId(oid);
      }
    }
    return data;
  } catch (err) {
    log.debug(err);
    return data;
  }
}

/**
 * @summary Converts ObjectId to String representation for return Data
 *
 * @function convertToObjectIdString
 * @param {AnyData} data - value to check for ObjectId (required)
 *
 * @return {String} - the resulting String
 */
function convertToObjectIdString(data) {
  if (data === undefined || data === null) {
    return data;
  }
  if (data instanceof ObjectId) {
    const idString = data.toString();
    return `ObjectId("${idString}")`;
  }

  if (typeof data !== 'object' || data instanceof Date) {
    return data;
  }

  if (typeof data === 'object' && !Array.isArray(data)) {
    // object - need to recurse
    const keys = Object.keys(data);
    const converted = {};
    for (let k = 0; k < keys.length; k += 1) {
      converted[[keys[k]]] = convertToObjectIdString(data[keys[k]]);
    }

    return converted;
  }
  // Array
  const converted = [];
  for (let a = 0; a < data.length; a += 1) {
    converted.push(convertToObjectIdString(data[a]));
  }

  return converted;
}

/**
 * @summary Checks for dates or ObjectId that have been stringified and converts them back to their respective object
 *
 * @function convertToDateOrObjectId
 * @param {AnyData} data - the object to check for dates within (required).
 *
 * @return {Object} - the object containing dates
 */
function convertToDateOrObjectId(data) {
  log.trace(`${myid}-adapter-convertToDateOrObjectId`);

  // if no data - return it
  if (data === undefined || data === null) {
    return data;
  }

  // if it is a number or boolean - just return it
  if (typeof data === 'number' || typeof data === 'boolean') {
    return data;
  }

  // if it is a string need to check if a date string or objectId string
  if (typeof data === 'string') {
    // if string check to see if in data format
    const dateRegex = /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T(0[0-9]|1[0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]).([0-9][0-9][0-9])Z$/;

    if (dateRegex.test(data)) {
      // return the date instead of the string
      return new Date(data);
    }

    return convertObjectId(data); // Handle if ObjectId string
  }

  // if the data is a regular expression - just return it
  if (typeof data === 'object' && data.constructor === RegExp) {
    return data;
  }

  // if data is an object but not an array
  if (typeof data === 'object' && !Array.isArray(data)) {
    // get all of the keys from the object
    const keys = Object.keys(data);
    const converted = {};

    // go through the keys to handle the individual fields
    for (let k = 0; k < keys.length; k += 1) {
      if (data[keys[k]] instanceof ObjectId) { // value has already been converted to ObjectId
        converted[[keys[k]]] = data[keys[k]];
      } else {
        converted[[keys[k]]] = convertToDateOrObjectId(data[keys[k]]);
      }
    }

    return converted;
  }

  // must be an array
  const converted = [];
  for (let a = 0; a < data.length; a += 1) {
    converted.push(convertToDateOrObjectId(data[a]));
  }

  return converted;
}

/**
 * This is the adapter/interface into Mongo
 */
class DBMongo extends EventEmitterCl {
  /**
   * DBMongo Adapter
   * @constructor
   */
  constructor(prongid, properties) {
    super();
    this.alive = false;
    this.id = prongid;
    myid = prongid;

    // set up the properties I care about
    this.refreshProperties(properties);

    // get the path for the specific error file
    const errorFile = path.join(__dirname, '/error.json');

    // if the file does not exist - error
    if (!fs.existsSync(errorFile)) {
      const origin = `${this.id}-adapter-constructor`;
      log.warn(`${origin}: Could not locate ${errorFile} - errors will be missing details`);
    }

    // Read the action from the file system
    const errorData = JSON.parse(fs.readFileSync(errorFile, 'utf-8'));
    ({ errors } = errorData);
  }

  /**
   * refreshProperties is used to set up all of the properties for the adapter.
   * It allows properties to be changed later by simply calling refreshProperties rather
   * than having to restart the adapter.
   *
   * @function refreshProperties
   * @param {Object} properties - an object containing all of the properties
   */
  refreshProperties(properties) {
    const meth = 'adapter-refreshProperties';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    try {
      // Read the properties schema from the file system
      const propertiesSchema = JSON.parse(fs.readFileSync(path.join(__dirname, 'propertiesSchema.json'), 'utf-8'));

      // validate the entity against the schema
      const ajvInst = new AjvCl();
      const validate = ajvInst.compile(propertiesSchema);
      const result = validate(properties);

      // if invalid properties and did not already have properties, stop
      if (!result && !this.props) {
        log.error(`Attempt to configure adapter with invalid properties - ${JSON.stringify(validate.errors)}`);
        this.alive = false;
        this.emit('OFFLINE', { id: this.id });
        return;
      }

      // if invalid properties but had valid ones, keep the valid ones
      if (!result) {
        log.warn('Attempt to configure adapter with invalid properties!');
        return;
      }

      // for now just set this.props - may do individual properties later
      this.props = properties;
    } catch (e) {
      log.error(`${origin}: Properties may not have been set properly. ${e}`);
    }
  }

  /**
   * Call to connect and authenticate to the database
   *
   * @function connect
   */
  connect() {
    const meth = 'adapter-connect';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);
    // if there is no url and no host then we need to error
    if (!this.props.host && !this.props.url) {
      log.error('ERROR: Missing required host or url - can not connect to database');
      this.alive = false;
      this.emit('OFFLINE', { id: this.id });
      return;
    }
    // set up the default variables that will be used
    let url = '';
    let { db } = this.props;

    // if a url has been provided, pull it apart. URL overrides the individual properties!
    if (this.props.url && this.props.url.length > 0) {
      ({ url } = this.props);
      // checking if credentials are present in the URL
      const creds = url.split(/(?<=\/\/)(.*?)(?=@)/);
      // if credentials are not present in the url, find them and place them into URL
      if (creds.length !== 3) {
        if (this.props.credentials) {
          // check if dbAuth is enabled
          if (this.props.credentials.dbAuth) {
            if (this.props.credentials.user && this.props.credentials.passwd) {
              const dbAuthString = `${encodeURIComponent(this.props.credentials.user)}:${encodeURIComponent(this.props.credentials.passwd)}`;
              const decompStr = url.split('//');
              decompStr.splice(1, 0, `//${dbAuthString}@`);
              url = decompStr.join('');
            }
          }
        }
      }
      this.database = 'inurl';
    } else {
      const dbProtocol = 'mongodb';
      let dbAuthEnabled = false;
      let dbUsername = null;
      let dbPassword = null;
      let dbAuthString = null;
      let hostString = `${this.props.host}:27017`;
      let replSetEnabled = false;
      let replSet = null;

      // if no db throw a warning but then use test as the database
      if (!db) {
        log.warn('WARNING: No database found in property - connecting to test.');
        db = 'test';
      }
      this.database = db;

      // if there is a port defined change the hostString to include it
      if (this.props.port) {
        hostString = `${this.props.host}:${this.props.port}`;
      }

      // get the authentication properties if provided
      if (this.props.credentials) {
        if (this.props.credentials.dbAuth === true) {
          dbAuthEnabled = true;
        }
        if (this.props.credentials.user) {
          dbUsername = this.props.credentials.user;
        }
        if (this.props.credentials.passwd) {
          dbPassword = this.props.credentials.passwd;
        }
      }

      // get the replica set properties if provided
      if (this.props.replSet) {
        if (this.props.replSet.enabled === true) {
          replSetEnabled = true;
        }
        if (this.props.replSet.replicaSet) {
          if (!this.props.replSet.replicaSet.includes('replicaSet=')) {
            replSet = `replicaSet=${this.props.replSet.replicaSet}`;
          } else {
            replSet = this.props.replSet.replicaSet;
          }
        }
      }

      // format the url based on authentication or no authentication
      if (!dbAuthEnabled) {
        // format the url
        url = `${dbProtocol}://${hostString}`;
      } else {
        if (!dbAuthString) {
          // if authenticating but no credentials error out
          if (!dbUsername || !dbPassword) {
            log.error('ERROR: Database authentication is configured but username or password is not provided');
            this.alive = false;
            this.emit('OFFLINE', { id: this.id });
            return;
          }

          // format the dbAuthString - urlencoding the username and password
          dbAuthString = `${encodeURIComponent(dbUsername)}:${encodeURIComponent(dbPassword)}`;
        }

        // format the url
        url = `${dbProtocol}://${dbAuthString}@${hostString}/${db}`;
      }

      // are we using a replication set need to add it to the url
      if (replSetEnabled && replSet) {
        url += `?${replSet}`;
      }
    }

    // define some local variables to help in validating the properties.json file
    let sslEnabled = false;
    let sslValidate = false;
    let sslCheckServerIdentity = false;
    let sslCA = null;

    /*
      * this first section is configuration mapping
      * it can be replaced with the config object when available
      */
    if (this.props.ssl) {
      // enable ssl encryption?
      if (this.props.ssl.enabled === true) {
        log.info('Connecting to MongoDB with SSL.');
        sslEnabled = true;
        // validate the server's certificate
        // against a known certificate authority?
        if (this.props.ssl.acceptInvalidCerts === false) {
          sslValidate = true;
          log.info('Certificate based SSL MongoDB connections will be used.');
          // if validation is enabled, we need to read the CA file
          if (this.props.ssl.sslCA) {
            try {
              sslCA = this.props.ssl.sslCA;
            } catch (err) {
              log.error(`Error: Unable to load Mongo CA file path: ${err}`);
              this.alive = false;
              this.emit('OFFLINE', {
                id: this.id
              });
              return;
            }
          } else {
            log.error('Error: Certificate validation'
              + 'is enabled but a CA is not specified.');
            this.alive = false;
            this.emit('OFFLINE', {
              id: this.id
            });
            return;
          }
        } else {
          log.info('SSL MongoDB connection without CA certificate validation.');
        }
        // validate the server certificate against the configured url?
        if (this.props.ssl.checkServerIdentity === true) {
          sslCheckServerIdentity = true;
        } else {
          log.info('INFO: Skipping server identity validation as it is not supported by mongodb driver >=v4');
        }
      } else {
        log.warn('WARNING: Connecting to MongoDB without SSL.');
      }
    } else {
      log.warn('WARNING: Connecting to MongoDB without SSL.');
    }

    // This second section is to construct the mongo options object
    const options = {
      ssl: sslEnabled,
      tls: sslEnabled,
      tlsAllowInvalidCertificates: sslValidate,
      tlsAllowInvalidHostnames: sslCheckServerIdentity
    };

    if (sslValidate === true) {
      options.tlsCAFile = sslCA;
    }

    if (this.props.databaseConnection === 'connect on request') {
      this.db = db;
      this.url = url;
      this.options = options;
      return this.emit('ONLINE', {
        id: this.id
      });
    }

    log.debug('Connecting to MongoDB with provided url and options');
    // Now we will start the process of connecting to mongo db
    MongoClient.connect(url, options, (err, mongoClient) => {
      if (!mongoClient) {
        log.error(`Error! Exiting... Must start MongoDB first ${err}`);
        this.alive = false;
        this.emit('OFFLINE', {
          id: this.id
        });
      } else {
        log.info('mongo running');
        this.clientDB = mongoClient.db(db);

        mongoClient.on('close', () => {
          this.alive = false;
          this.emit('OFFLINE', {
            id: this.id
          });
          log.error('MONGO CONNECTION LOST...');
        });

        mongoClient.on('reconnect', () => {
          // we still need to check if we are properly authenticated
          // so we just list collections to test it.
          this.clientDB.collections((error) => {
            if (error) {
              log.error(error);
              this.alive = false;
              this.emit('OFFLINE', {
                id: this.id
              });
            } else {
              log.info('MONGO CONNECTION BACK...');
              this.alive = true;
              this.emit('ONLINE', {
                id: this.id
              });
            }
          });
        });

        // we still need to check if we are properly authenticated
        // so we just list collections to test it.
        this.clientDB.collections((error) => {
          if (error) {
            log.error(error);
            this.alive = false;
            this.emit('OFFLINE', {
              id: this.id
            });
          } else {
            /*
            * once we are connected to mongo, we need to perform a health check
            * to ensure we can read from the database
            */
            log.info('MONGO CONNECTION UP...');
            this.alive = true;
            this.emit('ONLINE', {
              id: this.id
            });
            log.info('Successfully authenticated with the Mongo DB server');

            let healthtype = 'startup';
            if (this.props.healthcheck && this.props.healthcheck.type) {
              healthtype = this.props.healthcheck.type;
            }
            if (healthtype === 'startup') {
              this.healthCheck((hc) => {
                if (hc.status === 'fail') {
                  log.error('Error connecting to Mongo DB:'
                    + 'health check has failed.');
                  this.alive = false;
                  this.emit('OFFLINE', {
                    id: this.id
                  });
                } else {
                  // successful health check; now we
                  // can declare the database as online
                  log.info('MongoDB connection has been established');
                  this.alive = true;
                  this.emit('ONLINE', {
                    id: this.id
                  });
                }
              });
            }
          }
        });
      }
    });
  }

  /**
   * Call to run a healthcheck on the database
   *
   * @function healthCheck
   * @param {healthCallback} callback - a callback function to return a result
   *                                    healthcheck success or failure
   */
  healthCheck(callback) {
    const meth = 'adapter-healthCheck';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    try {
      // verify that we are connected to Mongo
      if (!this.alive || !this.clientDB) {
        log.error('Error during healthcheck: Not connected to Database');
        return callback({
          id: this.id,
          status: 'fail'
        });
      }

      return this.clientDB.stats((err) => {
        if (err) {
          log.error(`Error during healthcheck: ${err}`);
          return callback({
            id: this.id,
            status: 'fail'
          });
        }

        return callback({
          id: this.id,
          status: 'success'
        });
      });
    } catch (ex) {
      log.error(`Exception during healthcheck: ${ex}`);
      return callback({
        id: this.id,
        status: 'fail'
      });
    }
  }

  /**
   * getAllFunctions is used to get all of the exposed function in the adapter
   *
   * @function getAllFunctions
   */
  getAllFunctions() {
    let myfunctions = [];
    let obj = this;

    // find the functions in this class
    do {
      const l = Object.getOwnPropertyNames(obj)
        .concat(Object.getOwnPropertySymbols(obj).map((s) => s.toString()))
        .sort()
        .filter((p, i, arr) => typeof obj[p] === 'function' && p !== 'constructor' && (i === 0 || p !== arr[i - 1]) && myfunctions.indexOf(p) === -1);
      myfunctions = myfunctions.concat(l);
    }
    while (
      (obj = Object.getPrototypeOf(obj)) && Object.getPrototypeOf(obj)
    );

    return myfunctions;
  }

  /**
   * getWorkflowFunctions is used to get all of the workflow function in the adapter
   *
   * @function getWorkflowFunctions
   */
  getWorkflowFunctions() {
    const myfunctions = this.getAllFunctions();
    const wffunctions = [];

    // remove the functions that should not be in a Workflow
    for (let m = 0; m < myfunctions.length; m += 1) {
      if (myfunctions[m] === 'addListener') {
        // got to the second tier (adapterBase)
        break;
      }
      if (myfunctions[m] !== 'connect' && myfunctions[m] !== 'healthCheck'
        && myfunctions[m] !== 'getAllFunctions' && myfunctions[m] !== 'getWorkflowFunctions'
        && myfunctions[m] !== 'refreshProperties') {
        wffunctions.push(myfunctions[m]);
      }
    }

    return wffunctions;
  }

  /**
   * Call to create an item in the database
   *
   * @function create
   * @param {String} collectionName - the collection to save the item in. (required)
   * @param {String} data - the data to add. (required)
   * @param {createCallback} callback - a callback function to return a result
   *                                    (created item) or the error
   */
  async create(collectionName, data, callback) {
    const meth = 'adapter-create';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    let mongoClient;
    try {
      if (this.props.databaseConnection === 'connect on request') {
        mongoClient = await MongoClient.connect(this.url, this.options);
        if (!mongoClient) {
          const errorObj = formatErrorObject(origin, 'Mongo connection failed', null, null, null, null);
          return callback(null, errorObj);
        }
        this.clientDB = mongoClient.db(this.db);
        mongoClient.on('close', () => { log.debug(`${origin} db close`); });
        mongoClient.on('error', () => { log.error(`${origin} db error`); });
        mongoClient.on('fullsetup', () => { log.debug(`${origin} db fullsetup`); });
        mongoClient.on('parseError', () => { log.error(`${origin} db parseError`); });
        mongoClient.on('timeout', () => { log.error(`${origin} db timeout`); });

        this.alive = true;
      }
      // verify the required data has been provided
      if (collectionName === undefined || collectionName === null || collectionName === '') {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['collectionName'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }
      if (data === undefined || data === null || data === '') {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['data'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // verify that we are connected to Mongo
      if (!this.alive || !this.clientDB) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`${this.database} not connected`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // get the collection so we can run the remove on the collection
      const collection = this.clientDB.collection(collectionName);

      if (!collection) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`Collection ${collectionName} not found`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      const dataInfo = data;
      let createId = true;
      if (this.props.createUuid !== undefined && this.props.createUuid !== null && this.props.createUuid === false) {
        createId = false;
      }
      if (!{}.hasOwnProperty.call(dataInfo, '_id') && createId) {
        dataInfo._id = uuid.v4();
      }
      if (dataInfo._id) {
        dataInfo._id = convertObjectId(dataInfo._id);
      }

      // insert the item into the collection
      return collection.insertOne(convertToDateOrObjectId(dataInfo), {}, async (err, result) => {
        if (mongoClient) {
          this.alive = false;
          mongoClient.close().catch((closeErr) => {
            log.error(`${origin}: Error closing MongoDB client - ${closeErr.message}`);
          });
        }

        if (err) {
          const errorObj = formatErrorObject(origin, 'Database Error', [err], null, null, null);
          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
          return callback(null, errorObj);
        }

        try {
          // Wait for the document to be available
          const insertedDocument = await collection.findOne({ _id: result.insertedId });

          if (insertedDocument) {
            return callback({
              status: 'success',
              code: 200,
              response: insertedDocument
            });
          }
        } catch (findErr) {
          log.error(`${origin}: Error retrieving inserted document - ${findErr.message}`);
          const errorObj = formatErrorObject(origin, 'Document Retrieval Error', [findErr], null, null, null);
          return callback(null, errorObj);
        }
      });
    } catch (ex) {
      const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
      log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
      return callback(null, errorObj);
    }
  }

  /**
   * Call to create many items in the database
   *
   * @function createMany
   * @param {String} collectionName - the collection to save the items in. (required)
   * @param {Array} data - the modification to make. (required)
   * @param {Boolean} ordered - whether the data needs to be ordered. (optional)
   * @param {String} writeConcern - whether the data should be overwritten. (optional)
   * @param {createCallback} callback - a callback function to return a result
   *                                    (created items) or the error
   */
  async createMany(collectionName, data, ordered, writeConcern, callback) {
    const meth = 'adapter-createMany';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    let mongoClient;
    try {
      if (this.props.databaseConnection === 'connect on request') {
        mongoClient = await MongoClient.connect(this.url, this.options);
        if (!mongoClient) {
          const errorObj = formatErrorObject(origin, 'Mongo connection failed', null, null, null, null);
          return callback(null, errorObj);
        }
        this.clientDB = mongoClient.db(this.db);
        mongoClient.on('close', () => { log.debug(`${origin} db close`); });
        mongoClient.on('error', () => { log.error(`${origin} db error`); });
        mongoClient.on('fullsetup', () => { log.debug(`${origin} db fullsetup`); });
        mongoClient.on('parseError', () => { log.error(`${origin} db parseError`); });
        mongoClient.on('timeout', () => { log.error(`${origin} db timeout`); });

        this.alive = true;
      }
      // verify the required data has been provided
      if (collectionName === undefined || collectionName === null || collectionName === '') {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['collectionName'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }
      if (data === undefined || data === null || data === '') {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['data'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }
      if (!Array.isArray(data)) {
        const errorObj = formatErrorObject(origin, 'Invalid data format - data must be an array', null, null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // IAP is passing empty strings instead of null so would prefer null if no ordered.
      if (ordered === '') {
        ordered = null;
      }
      if (writeConcern === '') {
        writeConcern = null;
      }

      // verify that we are connected to Mongo
      if (!this.alive || !this.clientDB) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`${this.database} not connected`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // get the collection so we can run the remove on the collection
      const collection = this.clientDB.collection(collectionName);

      if (!collection) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`Collection ${collectionName} not found`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      const dataInfo = data;
      let createId = true;
      if (this.props.createUuid !== undefined && this.props.createUuid !== null && this.props.createUuid === false) {
        createId = false;
      }
      const options = {
        ordered,
        writeConcern
      };

      for (let i = 0; i < dataInfo.length; i += 1) {
        if (!{}.hasOwnProperty.call(dataInfo, '_id') && createId) {
          dataInfo._id = uuid.v4();
        }
      }

      // insert the items into the collection
      return collection.insertMany(convertToDateOrObjectId(dataInfo), options, (err, result) => {
        if (mongoClient) {
          this.alive = false;
          mongoClient.close().catch((closeErr) => {
            log.error(`${origin}: Error closing MongoDB client - ${closeErr.message}`);
          });
        }

        if (err) {
          const errorObj = formatErrorObject(origin, 'Database Error', [err], null, null, null);
          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
          return callback(null, errorObj);
        }

        // only doing this to hopefully make this a non-breaking change although I think the library version changes the response format
        if (result && result.ops && Array.isArray(result.ops) && result.ops.length > 0) {
          return callback({
            status: 'success',
            code: 200,
            response: result.ops[0]
          });
        }

        return callback({
          status: 'success',
          code: 200,
          response: result
        });
      });
    } catch (ex) {
      const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
      log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
      return callback(null, errorObj);
    }
  }

  /**
 * Call to get the size of a collection in the database
 *
 * @function collectionSize
 * @param {String} collectionName - the collection to get. (required)
 * @param {getCallback} callback - a callback function to return a result
 */
  async collectionSize(collectionName, callback) {
    const meth = 'adapter-collectionSize';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    let mongoClient;
    try {
      if (this.props.databaseConnection === 'connect on request') {
        mongoClient = await MongoClient.connect(this.url, this.options);
        if (!mongoClient) {
          const errorObj = formatErrorObject(origin, 'Mongo connection failed', null, null, null, null);
          return callback(null, errorObj);
        }
        this.clientDB = mongoClient.db(this.db);
        mongoClient.on('close', () => { log.debug(`${origin} db close`); });
        mongoClient.on('error', () => { log.error(`${origin} db error`); });
        mongoClient.on('fullsetup', () => { log.debug(`${origin} db fullsetup`); });
        mongoClient.on('parseError', () => { log.error(`${origin} db parseError`); });
        mongoClient.on('timeout', () => { log.error(`${origin} db timeout`); });

        this.alive = true;
      }

      // verify the required data has been provided
      if (!collectionName) {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['collectionName'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // verify that we are connected to Mongo
      if (!this.alive || !this.clientDB) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`${this.database} not connected`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // get the collection so we can run the remove on the collection
      const collection = this.clientDB.collection(collectionName);

      if (!collection) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`Collection ${collectionName} not found`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // get the size of the collection
      const stats = await collection.stats();
      const { size } = stats;

      return callback({
        status: 'success',
        code: 200,
        response: size
      });
    } catch (ex) {
      const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
      log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
      return callback(null, errorObj);
    } finally {
      if (mongoClient) {
        this.alive = false;
        await mongoClient.close();
      }
    }
  }

  /**
   * Call to query items in the database
   *
   * @function query
   * @param {Object} queryDoc - the query to use to find data. (required)
   * queryDoc = {
   *  collection : <collection_name>,
   *  filter : <filter Obj>,
   *  projection : <projection Obj>,
   *  sort : <sort Obj>,
   *  start : <start position>,
   *  limit : <limit of results>
   * }
   * @param {getCallback} callback - a callback function to return a result
   *                                 (items) or the error
   */
  async query(queryDoc, callback) {
    const meth = 'adapter-query';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    await this.queryWithOptions(queryDoc, null, callback);
  }

  /**
   * Call to query items in the database
   *
   * @function query
   * @param {Object} queryDoc - the query to use to find data. (required)
   * queryDoc = {
   *  collection : <collection_name>,
   *  filter : <filter Obj>,
   *  projection : <projection Obj>,
   *  sort : <sort Obj>,
   *  start : <start position>,
   *  limit : <limit of results>
   * }
   * @param {Object} findOptions - Find options (optional)
   * queryDoc = {
   *  allowDiskUse : <boolean>,
   *  ...
   * }
   * @param {getCallback} callback - a callback function to return a result
   *                                 (items) or the error
   */
  async queryWithOptions(queryDoc, findOptions, callback) {
    const meth = 'adapter-queryWithOptions';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    let mongoClient;
    try {
      if (this.props.databaseConnection === 'connect on request') {
        mongoClient = await MongoClient.connect(this.url, this.options);
        if (!mongoClient) {
          const errorObj = formatErrorObject(origin, 'Mongo connection failed', null, null, null, null);
          return callback(null, errorObj);
        }
        this.clientDB = mongoClient.db(this.db);
        mongoClient.on('close', () => { log.debug(`${origin} db close`); });
        mongoClient.on('error', () => { log.error(`${origin} db error`); });
        mongoClient.on('fullsetup', () => { log.debug(`${origin} db fullsetup`); });
        mongoClient.on('parseError', () => { log.error(`${origin} db parseError`); });
        mongoClient.on('timeout', () => { log.error(`${origin} db timeout`); });

        this.alive = true;
      }
      // verify the required data has been provided
      if (queryDoc === undefined || queryDoc === null || queryDoc === '') {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['queryDoc'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }
      if (typeof queryDoc === 'string') {
        queryDoc = JSON.parse(queryDoc);
      }
      if (!queryDoc.collection) {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['queryDoc.collection'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      let filter = queryDoc.filter || {};
      filter = convertOperatorTriggers(filter);
      let projection = queryDoc.projection || {};
      projection = convertOperatorTriggers(projection);
      const sort = queryDoc.sort || {};
      const start = queryDoc.start || 0;
      const limit = queryDoc.limit || 0;
      const options = findOptions || {};

      // verify that we are connected to Mongo
      if (!this.alive || !this.clientDB) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`${this.database} not connected`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // get the collection so we can run the remove on the collection
      const collection = this.clientDB.collection(queryDoc.collection);

      if (!collection) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`Collection ${queryDoc.collection} not found`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // find the items in the collection
      return collection.find((convertToDateOrObjectId(filter) || {}), options).sort(sort).skip(start).limit(limit)
        .project((projection || {}))
        .toArray((err, docs) => {
          if (err) {
            const errorObj = formatErrorObject(origin, 'Database Error', [err], null, null, null);
            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
            return callback(null, errorObj);
          }
          let convertedDocs = docs;
          if (this.props.convertObjectId) {
            convertedDocs = convertToObjectIdString(docs);
          }

          return callback({
            status: 'success',
            code: 200,
            response: convertedDocs
          });
        });
    } catch (ex) {
      const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
      log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
      return callback(null, errorObj);
    }
  }

  /**
   * Call to find items in the database
   *
   * @function find
   * @param {Object} options - the options to use to find data. (required)
   * options = {
   *  entity : <collection_name>,
   *  filter : <filter Obj>,
   *  sort : <sort Obj>,
   *  start : <start position>,
   *  limit : <limit of results>,
   * }
   * @param {deleteCallback} callback - a callback function to return a result
   *                                    (status of the request) or the error
   */
  async find(options, callback) {
    const meth = 'adapter-find';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    let mongoClient;
    try {
      if (this.props.databaseConnection === 'connect on request') {
        mongoClient = await MongoClient.connect(this.url, this.options);
        if (!mongoClient) {
          const errorObj = formatErrorObject(origin, 'Mongo connection failed', null, null, null, null);
          return callback(null, errorObj);
        }
        this.clientDB = mongoClient.db(this.db);
        mongoClient.on('close', () => { log.debug(`${origin} db close`); });
        mongoClient.on('error', () => { log.error(`${origin} db error`); });
        mongoClient.on('fullsetup', () => { log.debug(`${origin} db fullsetup`); });
        mongoClient.on('parseError', () => { log.error(`${origin} db parseError`); });
        mongoClient.on('timeout', () => { log.error(`${origin} db timeout`); });

        this.alive = true;
      }
      // verify the required data has been provided
      if (options === undefined || options === null || options === '') {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['options'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }
      if (typeof options === 'string') {
        try {
          options = JSON.parse(options);
        } catch (pex) {
          const errorObj = formatErrorObject(origin, 'Missing Data', ['options could not be parsed'], null, null, null);
          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
          return callback(null, errorObj);
        }
      }
      if (!options.entity) {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['options.entity'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      let filter = options.filter || {};
      filter = convertOperatorTriggers(filter);

      // verify that we are connected to Mongo
      if (!this.alive || !this.clientDB) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`${this.database} not connected`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      const { entity } = options;
      const collections = await this.clientDB.listCollections({ name: entity }).toArray();
      if (!collections.length) {
        // The collection does not exist
        const errorObj = formatErrorObject(origin, 'Database Error', [`Collection ${entity} not found`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // get the collection so we can run the remove on the collection
      log.debug(`Using Mongo Collection ${entity}`);
      const collection = this.clientDB.collection(entity);

      if (!collection) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`Collection ${entity} not found`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      const sort = options.sort || {};
      const start = options.start || 0;
      let limit = 10;

      // If limit is not specified, default to 10.
      // Note: limit may be 0, which is equivalent to setting no limit.
      if (Object.hasOwnProperty.call(options, 'limit')) {
        ({ limit } = options);
      }

      // Replace filter with regex to allow for substring lookup
      // TODO: Need to create a new filter object instead of mutating the exsisting one
      const filterKeys = Object.keys(filter).filter((key) => (key[0] !== '$' && typeof filter[key] === 'string'));
      filterKeys.map((key) => {
        try {
          if (key !== '_id') {
            log.debug(`Updating Value for ${key}`);
            const escapedFilter = filter[key].replace(/([.?*+^$[\]\\(){}|-])/g, '\\$1');
            const regexedFilter = new RegExp(`.*${escapedFilter}.*`, 'i');
            filter[key] = {
              $regex: regexedFilter
            };
            log.debug(`Updating Value to ${JSON.stringify(filter[key])}`);
          } else {
            filter[key] = convertObjectId(filter[key]);
          }
        } catch (e) {
          delete filter[key];
        }
        return key;
      });

      // find the items in the collection
      filter = convertToDateOrObjectId(filter);
      log.debug(`Using Filter: ${JSON.stringify(filter)}`);
      log.debug(`Using Start: ${start}, Limit: ${limit}, and Sort: ${JSON.stringify(sort)}`);
      return collection.find(filter).sort(sort).skip(start).limit(limit)
        .toArray((err, docs) => {
          if (err) {
            if (mongoClient) {
              this.alive = false;
              mongoClient.close().catch((closeErr) => {
                log.error(`${origin}: Error closing MongoDB client - ${closeErr.message}`);
              });
            }
            const errorObj = formatErrorObject(origin, 'Database Error', [err], null, null, null);
            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
            return callback(null, errorObj);
          }
          return collection.count(filter, (error, docCount) => {
            if (mongoClient) {
              this.alive = false;
              mongoClient.close().catch((closeErr) => {
                log.error(`${origin}: Error closing MongoDB client - ${closeErr.message}`);
              });
            }

            if (error) {
              log.warn('returning results without total count');
            }

            let convertedDocs = docs;
            if (this.props.convertObjectId) {
              convertedDocs = convertToObjectIdString(docs);
            }

            return callback({
              status: 'success',
              code: 200,
              response: convertedDocs,
              total: docCount
            });
          });
        });
    } catch (ex) {
      const errorObj = formatErrorObject(origin, 'Caught Exception', null, null, null, ex);
      log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
      return callback(null, errorObj);
    }
  }

  /**
   * Call to search for count of items in the database
   *
   * @function count
   * @param {String} collectionName - the collection to find things from. (required)
   * @param {Object} filter - the filter for items to search for. (optional)
   * @param {deleteCallback} callback - a callback function to return a result
   *                                    (status of the request) or the error
   */
  async count(collectionName, filter, callback) {
    const meth = 'adapter-count';
    const origin = `${this.id}-${meth}`;
    log.trace(origin);

    let mongoClient;
    try {
      if (this.props.databaseConnection === 'connect on request') {
        mongoClient = await MongoClient.connect(this.url, this.options);
        if (!mongoClient) {
          const errorObj = formatErrorObject(origin, 'Mongo connection failed', null, null, null, null);
          return callback(null, errorObj);
        }
        this.clientDB = mongoClient.db(this.db);
        mongoClient.on('close', () => { log.debug(`${origin} db close`); });
        mongoClient.on('error', () => { log.error(`${origin} db error`); });
        mongoClient.on('fullsetup', () => { log.debug(`${origin} db fullsetup`); });
        mongoClient.on('parseError', () => { log.error(`${origin} db parseError`); });
        mongoClient.on('timeout', () => { log.error(`${origin} db timeout`); });

        this.alive = true;
      }
      // verify the required data has been provided
      if (collectionName === undefined || collectionName === null || collectionName === '') {
        const errorObj = formatErrorObject(origin, 'Missing Data', ['collectionName'], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }
      // IAP is passing empty strings instead of null so would prefer null if no filter.
      if (filter === '') {
        filter = {};
      }

      filter = convertOperatorTriggers(filter);

      // verify that we are connected to Mongo
      if (!this.alive || !this.clientDB) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`${this.database} not connected`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      const collections = await this.clientDB.listCollections({ name: collectionName }).toArray();
      if (!collections.length) {
        // The collection does not exist
        const errorObj = formatErrorObject(origin, 'Database Error', [`Collection ${collectionName} not found`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      // get the collection
      const collection = this.clientDB.collection(collectionName);

      if (!collection) {
        const errorObj = formatErrorObject(origin, 'Database Error', [`Collection ${collectionName} not found`], null, null, null);
        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
        return callback(null, errorObj);
      }

      return collection.count(filter, (error, docCount) => {
        if (mongoClient