loopback-connector-mongodb/lib/mongodb.js

The official MongoDB connector for the LoopBack framework.

// Copyright IBM Corp. 2012,2020. All Rights Reserved.
// Node module: loopback-connector-mongodb
// This file is licensed under the MIT License.
// License text available at https://opensource.org/licenses/MIT

'use strict';

/*!
 * Module dependencies
 */
const g = require('strong-globalize')();
const mongodb = require('mongodb');
const urlParser = require('mongodb/lib/connection_string').parseOptions;
const util = require('util');
const async = require('async');
const Connector = require('loopback-connector').Connector;
const debug = require('debug')('loopback:connector:mongodb');
const Decimal128 = mongodb.Decimal128;
const Decimal128TypeRegex = /decimal128/i;

const ObjectIdValueRegex = /^[0-9a-fA-F]{24}$/;
const ObjectIdTypeRegex = /objectid/i;

exports.ObjectID = ObjectID;
/*!
 * Convert the id to be a BSON ObjectID if it is compatible
 * @param {*} id The id value
 * @returns {ObjectID}
 */
function ObjectID(id) {
  if (id instanceof mongodb.ObjectId) {
    return id;
  }
  if (typeof id !== 'string') {
    return id;
  }
  try {
    // MongoDB's ObjectID constructor accepts number, 12-byte string or 24-byte
    // hex string. For LoopBack, we only allow 24-byte hex string, but 12-byte
    // string such as 'line-by-line' should be kept as string
    if (ObjectIdValueRegex.test(id)) {
      return new mongodb.ObjectId(id);
    } else {
      return id;
    }
  } catch (e) {
    return id;
  }
}

exports.generateMongoDBURL = generateMongoDBURL;
/*!
 * Generate the mongodb URL from the options
 */
function generateMongoDBURL(options) {
  // See https://docs.mongodb.com/manual/reference/connection-string/#dns-seedlist-connection-format
  // It can be `mongodb+srv` now.
  options.protocol = options.protocol || 'mongodb';
  options.hostname = options.hostname || options.host || '127.0.0.1';
  options.port = options.port || 27017;
  options.database = options.database || options.db || 'test';
  const username = options.username || options.user;
  let portUrl = '';
  // only include port if not using mongodb+srv
  if (options.protocol !== 'mongodb+srv') {
    portUrl = ':' + options.port;
  }
  if (username && options.password) {
    return (
      options.protocol +
      '://' +
      username +
      ':' +
      options.password +
      '@' +
      options.hostname +
      portUrl +
      '/' +
      options.database
    );
  } else {
    return (
      options.protocol +
      '://' +
      options.hostname +
      portUrl +
      '/' +
      options.database
    );
  }
}

/**
 * Initialize the MongoDB connector for the given data source
 * @param {DataSource} dataSource The data source instance
 * @param {Function} [callback] The callback function
 */
exports.initialize = function initializeDataSource(dataSource, callback) {
  if (!mongodb) {
    return;
  }

  const s = dataSource.settings;

  s.safe = s.safe !== false;
  s.w = s.w || 1;
  s.writeConcern = s.writeConcern || {
    w: s.w,
    wtimeout: s.wtimeout || null,
    j: s.j || null,
    journal: s.journal || null,
    fsync: s.fsync || null,
  };
  s.url = s.url || generateMongoDBURL(s);
  // useNewUrlParser and useUnifiedTopology are default now
  dataSource.connector = new MongoDB(s, dataSource);
  dataSource.ObjectID = mongodb.ObjectId;

  if (callback) {
    if (s.lazyConnect) {
      process.nextTick(function() {
        callback();
      });
    } else {
      dataSource.connector.connect(callback);
    }
  }
};

// MongoDB has deprecated some commands. To preserve
// compatibility of model connector hooks, this maps the new
// commands to previous names for the observors of this command.
const COMMAND_MAPPINGS = {
  insertOne: 'insert',
  updateOne: 'save',
  findOneAndUpdate: 'findAndModify',
  deleteOne: 'delete',
  deleteMany: 'delete',
  replaceOne: 'update',
  updateMany: 'update',
  countDocuments: 'count',
  estimatedDocumentCount: 'count',
};

/**
 * Helper function to be used in {@ fieldsArrayToObj} in order for V8 to avoid re-creating a new
 * function every time {@ fieldsArrayToObj} is called
 *
 * @see fieldsArrayToObj
 * @param {object} result
 * @param {string} field
 * @returns {object}
 */
function arrayToObjectReducer(result, field) {
  result[field] = 1;
  return result;
}

exports.fieldsArrayToObj = fieldsArrayToObj;

/**
 * Helper function to accept an array representation of fields projection and return the mongo
 * required object notation
 *
 * @param {string[]} fieldsArray
 * @returns {Object}
 */
function fieldsArrayToObj(fieldsArray) {
  if (!Array.isArray(fieldsArray)) return fieldsArray; // fail safe check in case non array object created
  return fieldsArray.length ?
    fieldsArray.reduce(arrayToObjectReducer, {}) :
    {_id: 1};
}

exports.MongoDB = MongoDB;

/**
 * The constructor for MongoDB connector
 * @param {Object} settings The settings object
 * @param {DataSource} dataSource The data source instance
 * @constructor
 */
function MongoDB(settings, dataSource) {
  Connector.call(this, 'mongodb', settings);

  this.debug = settings.debug || debug.enabled;

  if (this.debug) {
    debug('Settings: %j', settings);
  }

  this.dataSource = dataSource;

  if (
    this.settings.enableOptimisedfindOrCreate === true ||
    this.settings.enableOptimisedFindOrCreate === true ||
    this.settings.enableOptimizedfindOrCreate === true ||
    this.settings.enableOptimizedFindOrCreate === true
  ) {
    debug('Optimized findOrCreate is enabled by default, ' +
      'and the enableOptimizedFindOrCreate setting is ignored since v7.0.0.');
  }

  if (this.settings.enableGeoIndexing === true) {
    MongoDB.prototype.buildNearFilter = buildNearFilter;
  } else {
    MongoDB.prototype.buildNearFilter = undefined;
  }
}

util.inherits(MongoDB, Connector);

/**
 * Connect to MongoDB
 * @param {Function} [callback] The callback function
 *
 * @callback callback
 * @param {Error} err The error object
 * @param {Db} db The mongo DB object
 */
MongoDB.prototype.connect = function(callback) {
  const self = this;
  if (self.db) {
    process.nextTick(function() {
      if (callback) callback(null, self.db);
    });
  } else if (self.dataSource.connecting) {
    self.dataSource.once('connected', function() {
      process.nextTick(function() {
        if (callback) callback(null, self.db);
      });
    });
  } else {
    // See https://www.mongodb.com/docs/manual/reference/connection-string
    const validOptionNames = [
      'replicaSet',
      /** Enables or disables TLS/SSL for the connection. */
      'tls',
      /** A boolean to enable or disables TLS/SSL for the connection. (The ssl option is equivalent to the tls option.) */
      'ssl',
      /**
     * Specifies the location of a local TLS Certificate
     * @deprecated Will be removed in the next major version. Please use tlsCertificateKeyFile instead.
     */
      'tlsCertificateFile',
      /** Specifies the location of a local .pem file that contains either the client's TLS/SSL certificate and key or only the client's TLS/SSL key when tlsCertificateFile is used to provide the certificate. */
      'tlsCertificateKeyFile',
      /** Specifies the password to de-crypt the tlsCertificateKeyFile. */
      'tlsCertificateKeyFilePassword',
      /** Specifies the location of a local .pem file that contains the root certificate chain from the Certificate Authority. This file is used to validate the certificate presented by the mongod/mongos instance. */
      'tlsCAFile',
      /** Bypasses validation of the certificates presented by the mongod/mongos instance */
      'tlsAllowInvalidCertificates',
      /** Disables hostname validation of the certificate presented by the mongod/mongos instance. */
      'tlsAllowInvalidHostnames',
      /** Disables various certificate validations. */
      'tlsInsecure',
      /** The time in milliseconds to attempt a connection before timing out. */
      'connectTimeoutMS',
      /** The time in milliseconds to attempt a send or receive on a socket before the attempt times out. */
      'socketTimeoutMS',
      /** An array or comma-delimited string of compressors to enable network compression for communication between this client and a mongod/mongos instance. */
      'compressors',
      /** An integer that specifies the compression level if using zlib for network compression. */
      'zlibCompressionLevel',
      /** The maximum number of hosts to connect to when using an srv connection string, a setting of `0` means unlimited hosts */
      'srvMaxHosts',
      /**
       * Modifies the srv URI to look like:
       *
       * `_{srvServiceName}._tcp.{hostname}.{domainname}`
       *
       * Querying this DNS URI is expected to respond with SRV records
       */
      'srvServiceName',
      /** The maximum number of connections in the connection pool. */
      'maxPoolSize',
      /** The minimum number of connections in the connection pool. */
      'minPoolSize',
      /** The maximum number of connections that may be in the process of being established concurrently by the connection pool. */
      'maxConnecting',
      /** The maximum number of milliseconds that a connection can remain idle in the pool before being removed and closed. */
      'maxIdleTimeMS',
      /** The maximum time in milliseconds that a thread can wait for a connection to become available. */
      'waitQueueTimeoutMS',
      /** Specify a read concern for the collection (only MongoDB 3.2 or higher supported) */
      'readConcern',
      /** The level of isolation */
      'readConcernLevel',
      /** Specifies the read preferences for this connection */
      'readPreference',
      /** Specifies, in seconds, how stale a secondary can be before the client stops using it for read operations. */
      'maxStalenessSeconds',
      /** Specifies the tags document as a comma-separated list of colon-separated key-value pairs.  */
      'readPreferenceTags',
      /** The auth settings for when connection to server. */
      'auth',
      /** Specify the database name associated with the user’s credentials. */
      'authSource',
      /** Specify the authentication mechanism that MongoDB will use to authenticate the connection. */
      'authMechanism',
      /** Specify properties for the specified authMechanism as a comma-separated list of colon-separated key-value pairs. */
      'authMechanismProperties',
      /** The size (in milliseconds) of the latency window for selecting among multiple suitable MongoDB instances. */
      'localThresholdMS',
      /** Specifies how long (in milliseconds) to block for server selection before throwing an exception.  */
      'serverSelectionTimeoutMS',
      /** heartbeatFrequencyMS controls when the driver checks the state of the MongoDB deployment. Specify the interval (in milliseconds) between checks, counted from the end of the previous check until the beginning of the next one. */
      'heartbeatFrequencyMS',
      /** Sets the minimum heartbeat frequency. In the event that the driver has to frequently re-check a server's availability, it will wait at least this long since the previous check to avoid wasted effort. */
      'minHeartbeatFrequencyMS',
      /** The name of the application that created this MongoClient instance. MongoDB 3.4 and newer will print this value in the server log upon establishing each connection. It is also recorded in the slow query log and profile collections */
      'appName',
      /** Enables retryable reads. */
      'retryReads',
      /** Enable retryable writes. */
      'retryWrites',
      /** Allow a driver to force a Single topology type with a connection string containing one host */
      'directConnection',
      /** Instruct the driver it is connecting to a load balancer fronting a mongos like service */
      'loadBalanced',
      /**
       * The write concern w value
       * @deprecated Please use the `writeConcern` option instead
       */
      'w',
      /**
       * The write concern timeout
       * @deprecated Please use the `writeConcern` option instead
       */
      'wtimeoutMS',
      /**
       * The journal write concern
       * @deprecated Please use the `writeConcern` option instead
       */
      'journal',
      /**
       * A MongoDB WriteConcern, which describes the level of acknowledgement
       * requested from MongoDB for write operations.
       *
       * @see https://www.mongodb.com/docs/manual/reference/write-concern/
       */
      'writeConcern',
      /**
       * Validate mongod server certificate against Certificate Authority
       * @deprecated Will be removed in the next major version. Please use tlsAllowInvalidCertificates instead.
       */
      'sslValidate',
      /**
       * SSL Certificate file path.
       * @deprecated Will be removed in the next major version. Please use tlsCAFile instead.
       */
      'sslCA',
      /**
       * SSL Certificate file path.
       * @deprecated Will be removed in the next major version. Please use tlsCertificateKeyFile instead.
       */
      'sslCert',
      /**
       * SSL Key file file path.
       * @deprecated Will be removed in the next major version. Please use tlsCertificateKeyFile instead.
       */
      'sslKey',
      /**
       * SSL Certificate pass phrase.
       * @deprecated Will be removed in the next major version. Please use tlsCertificateKeyFilePassword instead.
       */
      'sslPass',
      /**
       * SSL Certificate revocation list file path.
       * @deprecated Will be removed in the next major version. Please use tlsCertificateKeyFile instead.
       */
      'sslCRL',
      /** TCP Connection no delay */
      'noDelay',
      /** @deprecated TCP Connection keep alive enabled. Will not be able to turn off in the future. */
      'keepAlive',
      /**
       * @deprecated The number of milliseconds to wait before initiating keepAlive on the TCP socket.
       *             Will not be configurable in the future.
       */
      'keepAliveInitialDelay',
      /** Force server to assign `_id` values instead of driver */
      'forceServerObjectId',
      /** A primary key factory function for generation of custom `_id` keys */
      'pkFactory',
      /** Enable command monitoring for this client */
      'monitorCommands',
      /** Server API version */
      'serverApi',
      /**
       * Optionally enable in-use auto encryption
       *
       * @remarks
       *  Automatic encryption is an enterprise only feature that only applies to operations on a collection. Automatic encryption is not supported for operations on a database or view, and operations that are not bypassed will result in error
       *  (see [libmongocrypt: Auto Encryption Allow-List](https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/client-side-encryption.rst#libmongocrypt-auto-encryption-allow-list)). To bypass automatic encryption for all operations, set bypassAutoEncryption=true in AutoEncryptionOpts.
       *
       *  Automatic encryption requires the authenticated user to have the [listCollections privilege action](https://www.mongodb.com/docs/manual/reference/command/listCollections/#dbcmd.listCollections).
       *
       *  If a MongoClient with a limited connection pool size (i.e a non-zero maxPoolSize) is configured with AutoEncryptionOptions, a separate internal MongoClient is created if any of the following are true:
       *  - AutoEncryptionOptions.keyVaultClient is not passed.
       *  - AutoEncryptionOptions.bypassAutomaticEncryption is false.
       *
       * If an internal MongoClient is created, it is configured with the same options as the parent MongoClient except minPoolSize is set to 0 and AutoEncryptionOptions is omitted.
       */
      'autoEncryption',
      /** Allows a wrapping driver to amend the client metadata generated by the driver to include information about the wrapping driver */
      'driverInfo',
      /** Configures a Socks5 proxy host used for creating TCP connections. */
      'proxyHost',
      /** Configures a Socks5 proxy port used for creating TCP connections. */
      'proxyPort',
      /** Configures a Socks5 proxy username when the proxy in proxyHost requires username/password authentication. */
      'proxyUsername',
      /** Configures a Socks5 proxy password when the proxy in proxyHost requires username/password authentication. */
      'proxyPassword',
    ];

    const lbOptions = Object.keys(self.settings);
    const validOptions = {};
    lbOptions.forEach(function(option) {
      if (validOptionNames.indexOf(option) > -1) {
        validOptions[option] = self.settings[option];
      }
    });
    debug('Valid options: %j', validOptions);

    function onError(err) {
      /* istanbul ignore if */
      if (self.debug) {
        g.error(
          '{{MongoDB}} connection is failed: %s %s',
          self.settings.url,
          err,
        );
      }
      if (callback) callback(err);
    }

    const urlObj = new URL(self.settings.url);

    if ((urlObj.pathname === '' ||
        urlObj.pathname.split('/')[1] === '') &&
        typeof self.settings.database === 'string') {
      urlObj.pathname = self.settings.database;
      self.settings.url = urlObj.toString();
    }

    const mongoClient = new mongodb.MongoClient(self.settings.url, validOptions);
    const callbackConnect = util.callbackify(() => mongoClient.connect());

    callbackConnect(function(
      err,
      client,
    ) {
      if (err) {
        onError(err);
        return;
      }
      if (self.debug) {
        debug('MongoDB connection is established: ' + self.settings.url);
      }
      self.client = client;
      // The database name might be in the url
      try {
        const url = urlParser(self.settings.url, validOptions); // only supports the validURL options now
        // See https://github.com/mongodb/mongodb/blob/6.0.1/lib/mongodb.d.ts#L3854
        const validDbOptionNames = [
          'authSource',
          'forceServerObjectId',
          'readPreference',
          'pkFactory',
          'readConcern',
          'retryWrites',
          'checkKeys',
          'serializeFunctions',
          'ignoreUndefined',
          'promoteLongs',
          'promoteBuffers',
          'promoteValues',
          'fieldsAsRaw',
          'bsonRegExp',
          'raw',
          'writeConcern',
          'logger',
          'loggerLevel',
        ];
        const dbOptions = url.db_options || self.settings;
        const dbOptionKeys = Object.keys(dbOptions);
        const validDbOptions = {};
        dbOptionKeys.forEach(function(option) {
          if (validDbOptionNames.indexOf(option) > -1) {
            validDbOptions[option] = dbOptions[option];
          }
        });
        self.db = client.db(
          url.dbName || self.settings.database,
          validDbOptions,
        );
        if (callback) callback(err, self.db);
      } catch (e) {
        onError(e);
      }
    });
  }
};

MongoDB.prototype.getTypes = function() {
  return ['db', 'nosql', 'mongodb'];
};

MongoDB.prototype.getDefaultIdType = function() {
  return ObjectID;
};

/**
 * Get collection name for a given model
 * @param {String} modelName The model name
 * @returns {String} collection name
 */
MongoDB.prototype.collectionName = function(modelName) {
  const modelClass = this._models[modelName];
  if (modelClass && modelClass.settings && modelClass.settings.mongodb) {
    modelName = modelClass.settings.mongodb.collection || modelName;
  }
  return modelName;
};

/**
 * Access a MongoDB collection by model name
 * @param {String} modelName The model name
 * @returns {*}
 */
MongoDB.prototype.collection = function(modelName) {
  if (!this.db) {
    throw new Error(g.f('{{MongoDB}} connection is not established'));
  }
  const collectionName = this.collectionName(modelName);
  return this.db.collection(collectionName);
};

/*!
 * Convert the data from database to JSON
 *
 * @param {String} modelName The model name
 * @param {Object} data The data from DB
 */
MongoDB.prototype.fromDatabase = function(modelName, data) {
  if (!data) {
    return null;
  }
  const modelInfo = this._models[modelName] || this.dataSource.modelBuilder.definitions[modelName];
  const props = modelInfo.properties;
  for (const p in props) {
    const prop = props[p];
    if (prop && prop.type === Buffer) {
      if (data[p] instanceof mongodb.Binary) {
        // Convert the Binary into Buffer
        data[p] = data[p].read(0, data[p].length());
      }
    } else if (prop && prop.type === String) {
      if (data[p] instanceof mongodb.Binary) {
        // Convert the Binary into String
        data[p] = data[p].toString();
      }
    } else if (
      data[p] &&
      prop &&
      prop.type &&
      prop.type.name === 'GeoPoint' &&
      this.settings.enableGeoIndexing === true
    ) {
      data[p] = {
        lat: data[p].coordinates[1],
        lng: data[p].coordinates[0],
      };
    } else if (data[p] && prop && prop.type.definition) {
      data[p] = this.fromDatabase(prop.type.definition.name, data[p]);
    }
  }

  data = this.fromDatabaseToPropertyNames(modelName, data);

  return data;
};

/*!
 * Convert JSON to database-appropriate format
 *
 * @param {String} modelName The model name
 * @param {Object} data The JSON data to convert
 */
MongoDB.prototype.toDatabase = function(modelName, data) {
  const modelCtor = this._models[modelName];
  const props = modelCtor.properties;

  if (this.settings.enableGeoIndexing !== true) {
    visitAllProperties(data, modelCtor, coercePropertyValue);
    // Override custom column names
    data = this.fromPropertyToDatabaseNames(modelName, data);
    return data;
  }

  for (const p in props) {
    const prop = props[p];
    const isGeoPoint = data[p] && prop && prop.type && prop.type.name === 'GeoPoint';
    if (isGeoPoint) {
      data[p] = {
        coordinates: [data[p].lng, data[p].lat],
        type: 'Point',
      };
    }
  }

  visitAllProperties(data, modelCtor, coercePropertyValue);
  // Override custom column names
  data = this.fromPropertyToDatabaseNames(modelName, data);
  if (debug.enabled) debug('toDatabase data: ', util.inspect(data));
  return data;
};

/**
 * Execute a mongodb command
 * @param {String} modelName The model name
 * @param {String} command The command name
 * @param [...] params Parameters for the given command
 */
MongoDB.prototype.execute = function(modelName, command) {
  const self = this;
  // Get the parameters for the given command
  const args = [].slice.call(arguments, 2);
  // The last argument must be a callback function
  const callback = args[args.length - 1];

  // Topology is destroyed when the server is disconnected
  // Execute if DB is connected and functional otherwise connect/reconnect first
  if (
    self.db && (
      !self.db.topology || (self.db.topology && !self.db.topology.isDestroyed())
    )
  ) {
    doExecute();
  } else if (self.db && !self.db.topology) {
    doExecute();
  } else {
    if (self.db) {
      self.disconnect();
      self.db = null;
    }
    self.connect(function(err, db) {
      if (err) {
        debug(
          'Connection not established - MongoDB: model=%s command=%s -- error=%s',
          modelName,
          command,
          err,
        );
      }
      doExecute();
    });
  }

  function doExecute() {
    let collection;
    const context = Object.assign({}, {
      model: modelName,
      collection: collection,
      req: {
        command: command,
        params: args,
      },
    });

    try {
      collection = self.collection(modelName);
    } catch (err) {
      debug('Error: ', err);
      callback(err);
      return;
    }

    if (command in COMMAND_MAPPINGS) {
      context.req.command = COMMAND_MAPPINGS[command];
    }

    self.notifyObserversAround(
      'execute',
      context,
      function(context, done) {
        const observerCallback = function(err, result) {
          if (err) {
            debug('Error: ', err);
          } else {
            context.res = result;
            debug('Result: ', result);
          }
          done(err, result);
        };

        // from mongoddb v5 command does not support callback
        args.pop();
        debug('MongoDB: model=%s command=%s', modelName, command, args);

        // args had callback removed
        try {
          const execute = collection[command].apply(collection, args);

          if (command === 'find') {
            return observerCallback(null, execute);
          } else {
            const callbackCommand = util.callbackify(() => execute);

            return callbackCommand(observerCallback);
          }
        } catch (err) {
          return observerCallback(err, null);
        }
      },
      callback,
    );
  }
};

MongoDB.prototype.coerceId = function(modelName, id, options) {
  // See https://github.com/strongloop/loopback-connector-mongodb/issues/206
  if (id == null) return id;
  const self = this;
  let idValue = id;
  const idName = self.idName(modelName);

  // Type conversion for id
  const idProp = self.getPropertyDefinition(modelName, idName);

  if (idProp && typeof idProp.type === 'function') {
    if (!(idValue instanceof idProp.type)) {
      idValue = idProp.type(id);
      if (idProp.type === Number && isNaN(id)) {
        // Reset to id
        idValue = id;
      }
    }

    const modelCtor = this._models[modelName];
    idValue = coerceToObjectId(modelCtor, idProp, idValue);
  }
  return idValue;
};

/**
 * Create a new model instance for the given data
 * @param {String} modelName The model name
 * @param {Object} data The model data
 * @param {Function} [callback] The callback function
 */
MongoDB.prototype.create = function(modelName, data, options, callback) {
  const self = this;
  if (self.debug) {
    debug('create', modelName, data);
  }
  let idValue = self.getIdValue(modelName, data);
  const idName = self.idName(modelName);

  if (idValue === null) {
    delete data[idName]; // Allow MongoDB to generate the id
  } else {
    const oid = self.coerceId(modelName, idValue, options); // Is it an Object ID?c
    data._id = oid; // Set it to _id
    if (idName !== '_id') {
      delete data[idName];
    }
  }
  data = self.toDatabase(modelName, data);

  this.execute(modelName, 'insertOne', data, buildOptions({safe: true},
    options), function(err, result) {
    if (self.debug) {
      debug('create.callback', modelName, err, result);
    }
    if (err) {
      return callback(err);
    }
    idValue = result.insertedId;

    try {
      idValue = self.coerceId(modelName, idValue, options);
    } catch (err) {
      return callback(err);
    }

    // Wrap it to process.nextTick as delete data._id seems to be interferring
    // with mongo insert
    process.nextTick(function() {
      // Restore the data object
      delete data._id;
      data[idName] = idValue;
      callback(err, err ? null : idValue);
    });
  });
};

/**
 * Save the model instance for the given data
 * @param {String} modelName The model name
 * @param {Object} data The model data
 * @param {Function} [callback] The callback function
 */
MongoDB.prototype.save = function(modelName, data, options, callback) {
  const self = this;
  if (self.debug) {
    debug('save', modelName, data);
  }
  const idValue = self.getIdValue(modelName, data);
  const idName = self.idName(modelName);

  const oid = self.coerceId(modelName, idValue, options);
  data._id = oid;
  if (idName !== '_id') {
    delete data[idName];
  }

  data = self.toDatabase(modelName, data);

  this.execute(modelName, 'updateOne', {_id: oid}, {$set: data}, buildOptions({upsert: true},
    options), function(err, result) {
    if (!err) {
      self.setIdValue(modelName, data, idValue);
      if (idName !== '_id') {
        delete data._id;
      }
    }
    if (self.debug) {
      debug('save.callback', modelName, err, result);
    }

    const info = {};
    if (result) {
      // new 4.0 result formats:
      //   { acknowledged: true, modifiedCount: 1, upsertedCount: 1, : modifiedCount: 1}
      if (result.acknowledged === true && (result.matchedCount === 1 || result.upsertedCount === 1)) {
        info.isNewInstance = result.upsertedCount === 1;
      } else {
        debug('save result format not recognized: %j', result);
      }
    }

    if (callback) {
      callback(err, result && result.ops, info);
    }
  });
};

/**
 * Check if a model instance exists by id
 * @param {String} modelName The model name
 * @param {*} id The id value
 * @param {Function} [callback] The callback function
 *
 */
MongoDB.prototype.exists = function(modelName, id, options, callback) {
  const self = this;
  if (self.debug) {
    debug('exists', modelName, id);
  }
  id = self.coerceId(modelName, id, options);
  this.execute(modelName, 'findOne', {_id: id}, buildOptions({}, options), function(err, data) {
    if (self.debug) {
      debug('exists.callback', modelName, id, err, data);
    }
    callback(err, !!(!err && data));
  });
};

/**
 * Find a model instance by id
 * @param {String} modelName The model name
 * @param {*} id The id value
 * @param {Function} [callback] The callback function
 */
MongoDB.prototype.find = function find(modelName, id, options, callback) {
  const self = this;
  if (self.debug) {
    debug('find', modelName, id);
  }
  const idName = self.idName(modelName);
  const oid = self.coerceId(modelName, id, options);
  this.execute(modelName, 'findOne', {_id: oid}, buildOptions({}, options), function(err, data) {
    if (self.debug) {
      debug('find.callback', modelName, id, err, data);
    }

    data = self.fromDatabase(modelName, data);
    if (data && idName !== '_id') {
      delete data._id;
    }
    if (callback) {
      callback(err, data);
    }
  });
};

Connector.defineAliases(MongoDB.prototype, 'find', 'findById');

/**
 * Parses the data input for update operations and returns the
 * sanitised version of the object.
 *
 * @param data
 * @returns {*}
 */
MongoDB.prototype.parseUpdateData = function(modelName, data, options) {
  options = options || {};
  const parsedData = {};

  const modelClass = this._models[modelName];

  let allowExtendedOperators = this.settings.allowExtendedOperators;
  if (options.hasOwnProperty('allowExtendedOperators')) {
    allowExtendedOperators = options.allowExtendedOperators === true;
  } else if (
    allowExtendedOperators !== false &&
    modelClass.settings.mongodb &&
    modelClass.settings.mongodb.hasOwnProperty('allowExtendedOperators')
  ) {
    allowExtendedOperators =
      modelClass.settings.mongodb.allowExtendedOperators === true;
  } else if (allowExtendedOperators === true) {
    allowExtendedOperators = true;
  }

  if (allowExtendedOperators) {
    // Check for other operators and sanitize the data obj
    const acceptedOperators = [
      // Field operators
      '$currentDate',
      '$inc',
      '$max',
      '$min',
      '$mul',
      '$rename',
      '$setOnInsert',
      '$set',
      '$unset',
      // Array operators
      '$addToSet',
      '$pop',
      '$pullAll',
      '$pull',
      '$push',
      // Bitwise operator
      '$bit',
    ];

    let usedOperators = 0;

    // each accepted operator will take its place on parsedData if defined
    for (let i = 0; i < acceptedOperators.length; i++) {
      if (data[acceptedOperators[i]]) {
        parsedData[acceptedOperators[i]] = data[acceptedOperators[i]];
        usedOperators++;
      }
    }

    // if parsedData is still empty, then we fallback to $set operator
    if (usedOperators === 0 && Object.keys(data).length > 0) {
      parsedData.$set = data;
    }
  } else if (Object.keys(data).length > 0) {
    parsedData.$set = data;
  }

  return parsedData;
};

/**
 * Update if the model instance exists with the same id or create a new instance
 *
 * @param {String} modelName The model name
 * @param {Object} data The model instance data
 * @param {Function} [callback] The callback function
 */
MongoDB.prototype.updateOrCreate = function updateOrCreate(
  modelName,
  data,
  options,
  callback,
) {
  const self = this;
  if (self.debug) {
    debug('updateOrCreate', modelName, data);
  }

  const id = self.getIdValue(modelName, data);
  const idName = self.idName(modelName);
  const oid = self.coerceId(modelName, id, options);
  delete data[idName];

  data = self.toDatabase(modelName, data);

  // Check for other operators and sanitize the data obj
  data = self.parseUpdateData(modelName, data, options);

  this.execute(
    modelName,
    'findOneAndUpdate',
    {
      _id: oid,
    },
    data,
    buildOptions({
      upsert: true,
      returnNewDocument: true,
      returnDocument: 'after', // ensures new document gets returned
      sort: [['_id', 'asc']],
    }, options),
    function(err, result) {
      if (self.debug) {
        debug('updateOrCreate.callback', modelName, id, err, result);
      }
      const object = result && result.value;
      if (!err && !object) {
        // No result
        err = 'No ' + modelName + ' found for id ' + id;
      }
      if (!err) {
        self.setIdValue(modelName, object, oid);
        if (object && idName !== '_id') {
          delete object._id;
        }
      }

      let info;
      if (result && result.lastErrorObject) {
        info = {isNewInstance: !result.lastErrorObject.updatedExisting};
      } else {
        debug('updateOrCreate result format not recognized: %j', result);
      }

      if (callback) {
        callback(err, self.fromDatabase(modelName, object), info);
      }
    },
  );
};

/**
 * Replace model instance if it exists or create a new one if it doesn't
 *
 * @param {String} modelName The model name
 * @param {Object} data The model instance data
 * @param {Object} options The options object
 * @param {Function} [cb] The callback function
 */
MongoDB.prototype.replaceOrCreate = function(modelName, data, options, cb) {
  if (this.debug) debug('replaceOrCreate', modelName, data);

  const id = this.getIdValue(modelName, data);
  const oid = this.coerceId(modelName, id, options);
  const idName = this.idName(modelName);
  data._id = data[idName];
  delete data[idName];
  this.replaceWithOptions(modelName, oid, data, {upsert: true}, cb);
};

/**
 * Delete a model instance by id
 * @param {String} modelName The model name
 * @param {*} id The id value
 * @param [callback] The callback function
 */
MongoDB.prototype.destroy = function destroy(modelName, id, options, callback) {
  const self = this;
  if (self.debug) {
    debug('delete', modelName, id);
  }
  id = self.coerceId(modelName, id, options);
  this.execute(modelName, 'deleteOne', {_id: id}, buildOptions({}, options), function(err, result) {
    if (self.debug) {
      debug('delete.callback', modelName, id, err, result);
    }
    let res = result;
    if (res) {
      res = {count: res.deletedCount};
    }
    if (callback) {
      callback(err, res);
    }
  });
};

/*!
 * Decide if id should be included
 * @param {Object} fields
 * @returns {Boolean}
 * @private
 */
function idIncluded(fields, idName) {
  if (!fields) {
    return true;
  }
  if (Array.isArray(fields)) {
    return fields.indexOf(idName) >= 0;
  }
  if (fields[idName]) {
    // Included
    return true;
  }
  if (idName in fields && !fields[idName]) {
    // Excluded
    return false;
  }
  for (const f in fields) {
    return !fields[f]; // If the fields has exclusion
  }
  return true;
}

MongoDB.prototype.buildWhere = function(modelName, where, options) {
  const self = this;
  const query = {};
  if (where === null || typeof where !== 'object') {
    return query;
  }

  where = sanitizeFilter(where, options);

  let implicitNullType = false;
  if (this.settings.hasOwnProperty('implicitNullType')) {
    implicitNullType = !!this.settings.implicitNullType;
  }

  const idName = self.idName(modelName);
  Object.keys(where).forEach(function(k) {
    let cond = where[k];
    if (k === 'and' || k === 'or' || k === 'nor') {
      if (Array.isArray(cond)) {
        cond = cond.map(function(c) {
          return self.buildWhere(modelName, c, options);
        });
      }
      query['$' + k] = cond;
      delete query[k];
      return;
    }
    if (k === idName) {
      k = '_id';
    }
    let propName = k;
    if (k === '_id') {
      propName = idName;
    }

    const propDef = self.getPropertyDefinition(modelName, propName);
    if (propDef && propDef.mongodb && typeof propDef.mongodb.dataType === 'string') {
      if (Decimal128TypeRegex.test(propDef.mongodb.dataType)) {
        cond = Decimal128.fromString(cond);
        debug('buildWhere decimal value: %s, constructor name: %s', cond, cond.constructor.name);
      } else if (isStoredAsObjectID(propDef)) {
        cond = ObjectID(cond);
      }
    }

    // Convert property to database column name
    k = self.getDatabaseColumnName(modelName, k);

    let spec = false;
    let regexOptions = null;
    if (cond && cond.constructor.name === 'Object') {
      regexOptions = cond.options;
      spec = Object.keys(cond)[0];
      cond = cond[spec];
    }

    const modelCtor = self._models[modelName];

    if (spec) {
      spec = trimLeadingDollarSigns(spec);
      if (spec === 'between') {
        query[k] = {$gte: cond[0], $lte: cond[1]};
      } else if (spec === 'inq') {
        cond = [].concat(cond || []);
        query[k] = {
          $in: cond.map(function(x) {
            if (isObjectIDProperty(modelCtor, propDef, x, options))
              return ObjectID(x);
            return x;
          }),
        };
      } else if (spec === 'nin') {
        cond = [].concat(cond || []);
        query[k] = {
          $nin: cond.map(function(x) {
            if (isObjectIDProperty(modelCtor, propDef, x, options))
              return ObjectID(x);
            return x;
          }),
        };
      } else if (spec === 'like') {
        if (cond instanceof RegExp) {
          query[k] = {$regex: cond};
        } else {
          query[k] = {$regex: new RegExp(cond, regexOptions)};
        }
      } else if (spec === 'nlike') {
        if (cond instanceof RegExp) {
          query[k] = {$not: cond};
        } else {
          query[k] = {$not: new RegExp(cond, regexOptions)};
        }
      } else if (spec === 'neq') {
        query[k] = {$ne: cond};
      } else if (spec === 'regexp') {
        if (cond.global)
          g.warn('{{MongoDB}} regex syntax does not respect the {{`g`}} flag');

        query[k] = {$regex: cond};
      } else {
        if (isObjectIDProperty(modelCtor, propDef, cond, options)) {
          if (Array.isArray(cond)) {
            cond = cond.map(function(c) {
              return ObjectID(c);
            });
          } else {
            cond = ObjectID(cond);
          }
        }

        query[k] = {};
        query[k]['$' + spec] = cond;
      }
    } else {
      if (cond === null && !implicitNullType) {
        // http://docs.mongodb.org/manual/reference/operator/query/type/
        // Null: 10
        query[k] = {$type: 10};
      } else {
        if (isObjectIDProperty(modelCtor, propDef, cond, options)) {
          if (Array.isArray(cond)) {
            cond = cond.map(function(c) {
              return ObjectID(c);
            });
          } else {
            cond = ObjectID(cond);
          }
        }

        query[k] = cond;
      }
    }
  });
  return query;
};

MongoDB.prototype.buildSort = function(modelName, order, options) {
  let sort = {};
  const idName = this.idName(modelName);

  const modelClass = this._models[modelName];

  let disableDefaultSort = false;
  if (this.settings.hasOwnProperty('disableDefaultSort')) {
    disableDefaultSort = this.settings.disableDefaultSort;
  }
  if (modelClass.settings.hasOwnProperty('disableDefaultSort')) {
    disableDefaultSort = modelClass.settings.disableDefaultSort;
  }
  if (options && options.hasOwnProperty('disableDefaultSort')) {
    disableDefaultSort = options.disableDefaultSort;
  }

  if (!order && !disableDefaultSort) {
    const idNames = this.idNames(modelName);
    if (idNames && idNames.length) {
      order = idNames;
    }
  }
  if (order) {
    order = sanitizeFilter(order, options);
    let keys = order;
    if (typeof keys === 'string') {
      keys = keys.split(',');
    }
    for (let index = 0, len = keys.length; index < len; index++) {
      const m = keys[index].match(/\s+(A|DE)SC$/);
      let key = keys[index];
      key = key.replace(/\s+(A|DE)SC$/, '').trim();
      if (key === idName) {
        key = '_id';
      } else {
        key = this.getDatabaseColumnName(modelName, key);
      }

      if (m && m[1] === 'DE') {
        sort[key] = -1;
      } else {
        sort[key] = 1;
      }
    }
  } else if (!disableDefaultSort) {
    // order by _id by default
    sort = {_id: 1};
  }
  return sort;
};

function convertToMeters(distance, unit) {
  switch (unit) {
    case 'meters':
      return distance;
    case 'kilometers':
      return distance * 1000;
    case 'miles':
      return distance * 1600;
    case 'feet':
      return distance * 0.3048;
    default:
      console.warn(
        'unsupported unit ' +
        unit +
        ", fallback to mongodb default unit 'meters'",
      );
      return distance;
  }
}

function buildNearFilter(query, params) {
  if (!Array.isArray(params)) {
    params = [params];
  }

  params.forEach(function(near) {
    let coordinates = {};

    if (typeof near.near === 'string') {
      const s = near.near.split(',');
      coordinates.lng = parseFloat(s[0]);
      coordinates.lat = parseFloat(s[1]);
    } else if (Array.isArray(near.near)) {
      coordinates.lng = near.near[0];
      coordinates.lat = near.near[1];
    } else {
      coordinates = near.near;
    }

    const props = ['maxDistance', 'minDistance'];
    // use mongodb default unit 'meters' rather than 'miles'
    const unit = near.unit || 'meters';

    const queryValue = {
      near: {
        $geometry: {
          coordinates: [coordinates.lng, coordinates.lat],
          type: 'Point',
        },
      },
    };

    props.forEach(function(p) {
      if (near[p]) {
        queryValue.near['$' + p] = convertToMeters(near[p], unit);
      }
    });

    let property;

    if (near.mongoKey) {
      // if mongoKey is an Array, set the $near query at the right depth, following the Array
      if (Array.isArray(near.mongoKey)) {
        property = query.where;

        let i;
        for (i = 0; i < near.mongoKey.length; i++) {
          const subKey = near.mongoKey[i];

          if (near.mongoKey.hasOwnProperty(i + 1)) {
            if (!property.hasOwnProperty(subKey)) {
              property[subKey] = Number.isInteger(near.mongoKey[i + 1]) ?
                [] :
                {};
            }

            property = property[subKey];
          }
        }

        property[near.mongoKey[i - 1]] = queryValue;
      } else {
        // mongoKey is a single string, just set it directly
        property = query.where[near.mongoKey] = queryValue;
      }
    }
  });
}

function hasNearFilter(where) {
  if (!where) return false;
  // TODO: Optimize to return once a `near` key is found
  // instead of searching through everything

  let isFound = false;

  searchForNear(where);

  function found(prop) {
    return prop && prop.near;
  }

  function searchForNear(node) {
    if (!node) {
      return;
    }

    if (Array.isArray(node)) {
      node.forEach(function(prop) {
        isFound = found(prop);

        if (!isFound) {
          searchForNear(prop);
        }
      });
    } else if (typeof node === 'object') {
      Object.keys(node).forEach(function(key) {
        const prop = node[key];

        isFound = found(prop);

        if (!isFound) {
          searchForNear(prop);
        }
      });
    }
  }
  return isFound;
}

MongoDB.prototype.getDatabaseColumnName = function(model, propName) {
  if (typeof model === 'string') {
    model = this._models[model];
  }

  if (typeof model !== 'object') {
    return propName; // unknown model type?
  }

  if (typeof model.properties !== 'object') {
    return propName; // missing model properties?
  }

  const prop = model.properties[propName] || {};

  // mongoDB connector doesn't support custom id property field name
  if (prop.id) {
    // throws if a custom field name for id is set
    const customFieldName = (prop.mongodb && (
      prop.mongodb.fieldName ||
      prop.mongodb.field ||
      prop.mongodb.columnName ||
      prop.mongodb.column)
    ) ||
      prop.columnName ||
      prop.column;
    if (customFieldName) {
      throw new Error(`custom id field name '${customFieldName}' is not allowed` +
      ` in model ${model.model.definition.name}`);
    }
    return propName;
  } else {
    // Try mongo overrides
    if (prop.mongodb) {
      propName =
        prop.mongodb.fieldName ||
        prop.mongodb.field ||
        prop.mongodb.columnName ||
        prop.mongodb.column ||
        prop.columnName ||
        prop.column ||
        propName;
    } else {
      // Try top level overrides
      propName = prop.columnName || prop.column || propName;
    }
  }

  return propName;
};

MongoDB.prototype.convertColumnNames = function(model, data, direction) {
  if (typeof data !== 'object') {
    return data; // skip
  }

  if (typeof model === 'string') {
    model = this._models[model];
  }

  if (typeof model !== 'object') {
    return data; // unknown model type?
  }

  if (typeof model.properties !== 'object') {
    return data; // missing model properties?
  }

  for (const propName in model.properties) {
    const columnName = this.getDatabaseColumnName(model, propName);

    // Copy keys/data if needed
    if (propName === columnName) {
      continue;
    }

    if (direction === 'database') {
      // Handle data is Array object - in case of fields filter
      if (Array.isArray(data)) {
        const idx = data.indexOf(propName);
        if (idx !== -1) {
          data.push(columnName);
          delete data[idx];
        }
      } else { // Handle data as Object - in case to create / update
        data[columnName] = data[propName];
        delete data[propName];
      }
    }

    if (direction === 'property') {
      data[propName] = data[columnName];
      delete data[columnName];
    }
  }

  return data;
};

MongoDB.prototype.fromPropertyToDatabaseNames = function(model, data) {
  return this.convertColumnNames(model, data, 'database');
};

MongoDB.prototype.fromDatabaseToPropertyNames = function(model, data) {
  return this.convertColumnNames(model, data, 'property');
};

/**
 * Find matching model instances by the filter
 *
 * @param {String} modelName The model name
 * @param {Object} filter The filter
 * @param {Function} [callback] The callback function
 */
MongoDB.prototype.all = function all(modelName, filter, options, callback) {
  const self = this;
  if (self.debug) {
    debug('all', modelName, filter);
  }
  filter = filter || {};

  let query = {};
  if (filter.where) {
    query = self.buildWhere(modelName, filter.where, options);
  }
  // Use Object.assign to avoid change filter.fields
  // which will cause error when create model from data
  let fields = undefined;
  if (typeof filter.fields !== 'undefined') {
    fields = [];
    Object.assign(fields, filter.fields);
  }

  // Convert custom column names
  fields = self.fromPropertyToDatabaseNames(modelName, fields);

  options = buildOptions({}, options);

  if (fields) {
    options.projection = fieldsArrayToObj(fields);
  }
  this.execute(modelName, 'find', query, options, processResponse);

  function processResponse(err, cursor) {
    if (err) {
      return callback(err);
    }

    const collation = options && options.collation;
    if (collation) {
      cursor.collation(collation);
    }

    // don't apply sorting if dealing with a geo query
    if (!hasNearFilter(filter.where)) {
      const order = self.buildSort(modelName, filter.order, options);
      cursor.sort(order);
    }

    if (filter.limit) {
      cursor.limit(filter.limit);
    }
    if (filter.skip) {
      cursor.skip(filter.skip);
    } else if (filter.offset) {
      cursor.skip(filter.offset);
    }

    const callbackCursor = util.callbackify(() => cursor.toArray());

    callbackCursor(function(err, data) {
      if (self.debug) {
        debug('all', modelName, filter, err, data);
      }
      if (err) {
        return callback(err);
      }
      const objs = self.toModelEntity(modelName, data, fields);

      if (filter && filter.include) {
        self._models[modelName].model.include(
          objs,
          filter.include,
          options,
          callback,
        );
      } else {
        callback(null, objs);
      }
    });
  }
};

/**
 * Find a matching model instances by the filter or create a new instance
 *
 * Only supported on mongodb 2.6+
 *
 * @param {String} modelName The model name
 * @param {Object} data The model instance data
 * @param {Object} filter The filter
 * @param {Function} [callback] The callback function
 */
MongoDB.prototype.findOrCreate = function findOrCreate(modelName, filter, data, options, callback) {
  const self = this;
  if (self.debug) {
    debug('findOrCreate', modelName, filter, data);
  }

  if (!callback) callback = options;

  const idValue = self.getIdValue(modelName, data);
  const idName = self.idName(modelName);

  if (idValue == null) {
    delete data[idName]; // Allow MongoDB to generate the id
  } else {
    const oid = self.coerceId(modelName, idValue, options); // Is it an Object ID?
    data._id = oid; // Set it to _id
    if (idName !== '_id') {
      delete data[idName];
    }
  }

  filter = filter || {};
  let query = {};
  if (filter.where) {
    if (filter.where[idName]) {
      let id = filter.where[idName];
      delete filter.where[idName];
      id = self.coerceId(modelName, id, options);
      filter.where._id = id;
    }
    quer