dynogels/lib/table.js

DynamoDB data mapper

'use strict';

const _ = require('lodash');
const Item = require('./item');
const Query = require('./query');
const Scan = require('./scan');
const EventEmitter = require('events').EventEmitter;
const async = require('async');
const utils = require('./utils');
const ParallelScan = require('./parallelScan');
const expressions = require('./expressions');

const internals = {};

const Table = module.exports = function (name, schema, serializer, docClient, logger) {
  this.config = { name: name };
  this.schema = schema;
  this.serializer = serializer;
  this.docClient = docClient;
  this.log = logger;

  this._before = new EventEmitter();
  this.before = this._before.on.bind(this._before);

  this._after = new EventEmitter();
  this.after = this._after.on.bind(this._after);
};

Table.prototype.initItem = function (attrs) {
  const self = this;

  if (self.itemFactory) {
    return new self.itemFactory(attrs);
  } else {
    return new Item(attrs, self);
  }
};

Table.prototype.tableName = function () {
  if (this.schema.tableName) {
    if (_.isFunction(this.schema.tableName)) {
      return this.schema.tableName.call(this);
    } else {
      return this.schema.tableName;
    }
  } else {
    return this.config.name;
  }
};

Table.prototype.sendRequest = function (method, params, callback) {
  const self = this;

  let driver;
  if (_.isFunction(self.docClient[method])) {
    driver = self.docClient;
  } else if (_.isFunction(self.docClient.service[method])) {
    driver = self.docClient.service;
  }

  const startTime = Date.now();

  self.log.info({ params: params }, 'dynogels %s request', method.toUpperCase());
  driver[method].call(driver, params, (err, data) => {
    const elapsed = Date.now() - startTime;

    if (err) {
      self.log.warn({ err: err }, 'dynogels %s error', method.toUpperCase());
      return callback(err);
    } else {
      self.log.info({ data: data }, 'dynogels %s response - %sms', method.toUpperCase(), elapsed);
      return callback(null, data);
    }
  });
};

Table.prototype.get = function (hashKey, rangeKey, options, callback) {
  const self = this;

  if (_.isPlainObject(rangeKey) && typeof options === 'function' && !callback) {
    callback = options;
    options = rangeKey;
    rangeKey = null;
  } else if (typeof rangeKey === 'function' && !callback) {
    callback = rangeKey;
    options = {};
    rangeKey = null;
  } else if (typeof options === 'function' && !callback) {
    callback = options;
    options = {};
  }

  let params = {
    TableName: self.tableName(),
    Key: self.serializer.buildKey(hashKey, rangeKey, self.schema)
  };

  params = _.merge({}, params, options);

  self.sendRequest('get', params, (err, data) => {
    if (err) {
      return callback(err);
    }

    let item = null;
    if (data.Item) {
      item = self.initItem(self.serializer.deserializeItem(data.Item));
    }

    return callback(null, item);
  });
};

internals.callBeforeHooks = (table, name, startFun, callback) => {
  const listeners = table._before.listeners(name);

  return async.waterfall([startFun].concat(listeners), callback);
};

Table.prototype.create = function (item, options, callback) {
  const self = this;

  if (typeof options === 'function' && !callback) {
    callback = options;
    options = {};
  }

  callback = callback || _.noop;
  options = options || {};

  if (_.isArray(item)) {
    async.map(item, (data, callback) => internals.createItem(self, data, options, callback), callback);
  } else {
    return internals.createItem(self, item, options, callback);
  }
};

internals.createItem = (table, item, options, callback) => {
  const self = table;

  const start = callback => {
    const data = self.schema.applyDefaults(item);

    const paramName = _.isString(self.schema.createdAt) ? self.schema.createdAt : 'createdAt';

    if (self.schema.timestamps && self.schema.createdAt !== false && !_.has(data, paramName)) {
      data[paramName] = new Date().toISOString();
    }

    return callback(null, data);
  };

  internals.callBeforeHooks(self, 'create', start, (err, data) => {
    if (err) {
      return callback(err);
    }

    const result = self.schema.validate(data);

    if (result.error) {
      result.error.message = `${result.error.message} on ${self.tableName()}`;
      return callback(result.error);
    }

    const attrs = utils.omitNulls(data);

    let params = {
      TableName: self.tableName(),
      Item: self.serializer.serializeItem(self.schema, attrs)
    };

    if (options.expected) {
      internals.addConditionExpression(params, options.expected);
      options = _.omit(options, 'expected');
    }

    if (options.overwrite === false) {
      const expected = _.chain([self.schema.hashKey, self.schema.rangeKey]).compact().reduce((result, key) => {
        _.set(result, `${key}.<>`, _.get(params.Item, key));
        return result;
      }, {}).value();

      internals.addConditionExpression(params, expected);
    }

    options = _.omit(options, 'overwrite'); // remove overwrite flag regardless if true or false

    params = _.merge({}, params, options);

    self.sendRequest('put', params, err => {
      if (err) {
        return callback(err);
      }

      const item = self.initItem(attrs);
      self._after.emit('create', item);

      return callback(null, item);
    });
  });
};

internals.updateExpressions = (schema, data, options) => {
  const exp = expressions.serializeUpdateExpression(schema, data);

  if (options.UpdateExpression) {
    const parsed = expressions.parse(options.UpdateExpression);

    exp.expressions = _.reduce(parsed, (result, val, key) => {
      if (!_.isEmpty(val)) {
        result[key] = result[key].concat(val);
      }

      return result;
    }, exp.expressions);
  }

  if (_.isPlainObject(options.ExpressionAttributeValues)) {
    exp.values = _.merge({}, exp.values, options.ExpressionAttributeValues);
  }

  if (_.isPlainObject(options.ExpressionAttributeNames)) {
    exp.attributeNames = _.merge({}, exp.attributeNames, options.ExpressionAttributeNames);
  }

  return _.merge({}, {
    ExpressionAttributeValues: exp.values,
    ExpressionAttributeNames: exp.attributeNames,
    UpdateExpression: expressions.stringify(exp.expressions),
  });
};

internals.validateItemFragment = (item, schema) => {
  const result = {};
  const error = {};

  // get the list of attributes to remove
  const removeAttributes = _.pickBy(item, _.isNull);

  // get the list of attributes whose value is an object
  const setOperationValues = _.pickBy(item, i => _.isPlainObject(i) && (i.$add || i.$del));

  // get the list of attributes to modify
  const updateAttributes = _.omit(
    item,
    Object.keys(removeAttributes).concat(Object.keys(setOperationValues))
  );

  // check attribute removals for .required() schema violation
  const removalValidation = schema.validate(
    {},
    { abortEarly: false }
  );

  if (removalValidation.error) {
    const errors = _.pickBy(
      removalValidation.error.details,
      e => _.isEqual(e.type, 'any.required')
      && Object.prototype.hasOwnProperty.call(removeAttributes, e.path)
    );
    if (!_.isEmpty(errors)) {
      error.remove = errors;
      result.error = error;
    }
  }

  // check attribute updates match the schema
  const updateValidation = schema.validate(
    updateAttributes,
    { abortEarly: false }
  );

  if (updateValidation.error) {
    const errors = _.omitBy(
      updateValidation.error.details,
      e => _.isEqual(e.type, 'any.required')
    );
    if (!_.isEmpty(errors)) {
      error.update = errors;
      result.error = error;
    }
  }

  return result;
};

Table.prototype.update = function (item, options, callback) {
  const self = this;

  if (typeof options === 'function' && !callback) {
    callback = options;
    options = {};
  }

  callback = callback || _.noop;
  options = options || {};

  const schemaValidation = internals.validateItemFragment(item, self.schema);
  if (schemaValidation.error) {
    return callback(_.assign(new Error(`Schema validation error while updating item in table ${self.tableName()}: ${JSON.stringify(schemaValidation.error)}`), {
      name: 'DynogelsUpdateError',
      detail: schemaValidation.error
    }));
  }

  const start = callback => {
    const paramName = _.isString(self.schema.updatedAt) ? self.schema.updatedAt : 'updatedAt';

    if (self.schema.timestamps && self.schema.updatedAt !== false && !_.has(item, paramName)) {
      item[paramName] = new Date().toISOString();
    }

    return callback(null, item);
  };

  internals.callBeforeHooks(self, 'update', start, (err, data) => {
    if (err) {
      return callback(err);
    }

    const hashKey = data[self.schema.hashKey];
    let rangeKey = data[self.schema.rangeKey];

    if (_.isUndefined(rangeKey)) {
      rangeKey = null;
    }

    let params = {
      TableName: self.tableName(),
      Key: self.serializer.buildKey(hashKey, rangeKey, self.schema),
      ReturnValues: 'ALL_NEW'
    };

    let exp = null;
    try {
      exp = internals.updateExpressions(self.schema, data, options);
    } catch (e) {
      return callback(e);
    }

    params = _.assign(params, exp);

    if (options.expected) {
      internals.addConditionExpression(params, options.expected);
    }

    const unprocessedOptions = _.omit(options, ['UpdateExpression', 'ExpressionAttributeValues', 'ExpressionAttributeNames', 'expected']);

    params = _.chain({}).merge(params, unprocessedOptions).omitBy(_.isEmpty).value();

    self.sendRequest('update', params, (err, data) => {
      if (err) {
        return callback(err);
      }

      let result = null;
      if (data.Attributes) {
        result = self.initItem(self.serializer.deserializeItem(data.Attributes));
      }

      self._after.emit('update', result);
      return callback(null, result);
    });
  });
};

internals.addConditionExpression = (params, expectedConditions) => {
  _.each(expectedConditions, (val, key) => {
    let operator;
    let expectedValue = null;

    const existingValueKeys = _.keys(params.ExpressionAttributeValues);

    if (_.isObject(val) && _.isBoolean(val.Exists) && val.Exists === true) {
      operator = 'attribute_exists';
    } else if (_.isObject(val) && _.isBoolean(val.Exists) && val.Exists === false) {
      operator = 'attribute_not_exists';
    } else if (_.isObject(val) && _.has(val, '<>')) {
      operator = '<>';
      expectedValue = _.get(val, '<>');
    } else {
      operator = '=';
      expectedValue = val;
    }

    const condition = expressions.buildFilterExpression(key, operator, existingValueKeys, expectedValue, null);
    params.ExpressionAttributeNames = _.merge({}, condition.attributeNames, params.ExpressionAttributeNames);
    params.ExpressionAttributeValues = _.merge({}, condition.attributeValues, params.ExpressionAttributeValues);

    if (_.isString(params.ConditionExpression)) {
      params.ConditionExpression = `${params.ConditionExpression} AND (${condition.statement})`;
    } else {
      params.ConditionExpression = `(${condition.statement})`;
    }
  });
};

Table.prototype.destroy = function (hashKey, rangeKey, options, callback) {
  const self = this;

  if (_.isPlainObject(rangeKey) && typeof options === 'function' && !callback) {
    callback = options;
    options = rangeKey;
    rangeKey = null;
  } else if (typeof rangeKey === 'function' && !callback) {
    callback = rangeKey;
    options = {};
    rangeKey = null;
  } else if (_.isPlainObject(rangeKey) && !callback) {
    callback = options;
    options = rangeKey;
    rangeKey = null;
  } else if (typeof options === 'function' && !callback) {
    callback = options;
    options = {};
  }

  callback = callback || _.noop;
  options = options || {};

  if (_.isPlainObject(hashKey)) {
    rangeKey = hashKey[self.schema.rangeKey];

    if (_.isUndefined(rangeKey)) {
      rangeKey = null;
    }

    hashKey = hashKey[self.schema.hashKey];
  }

  let params = {
    TableName: self.tableName(),
    Key: self.serializer.buildKey(hashKey, rangeKey, self.schema)
  };

  if (options.expected) {
    internals.addConditionExpression(params, options.expected);

    delete options.expected;
  }

  params = _.merge({}, params, options);

  self.sendRequest('delete', params, (err, data) => {
    if (err) {
      return callback(err);
    }

    let item = null;
    if (data.Attributes) {
      item = self.initItem(self.serializer.deserializeItem(data.Attributes));
    }

    self._after.emit('destroy', item);
    return callback(null, item);
  });
};

Table.prototype.query = function (hashKey) {
  const self = this;

  return new Query(hashKey, self, self.serializer);
};

Table.prototype.scan = function () {
  const self = this;

  return new Scan(self, self.serializer);
};

Table.prototype.parallelScan = function (totalSegments) {
  const self = this;

  return new ParallelScan(self, self.serializer, totalSegments);
};


internals.deserializeItems = (table, callback) => (err, data) => {
  if (err) {
    return callback(err);
  }

  const result = {};
  if (data.Items) {
    result.Items = _.map(data.Items, i => table.initItem(table.serializer.deserializeItem(i)));

    delete data.Items;
  }

  if (data.LastEvaluatedKey) {
    result.LastEvaluatedKey = data.LastEvaluatedKey;

    delete data.LastEvaluatedKey;
  }

  return callback(null, _.merge({}, data, result));
};

Table.prototype.runQuery = function (params, callback) {
  const self = this;

  self.sendRequest('query', params, internals.deserializeItems(self, callback));
};

Table.prototype.runScan = function (params, callback) {
  const self = this;

  self.sendRequest('scan', params, internals.deserializeItems(self, callback));
};

Table.prototype.runBatchGetItems = function (params, callback) {
  const self = this;
  self.sendRequest('batchGet', params, callback);
};

internals.attributeDefinition = (schema, key) => {
  let type = schema._modelDatatypes[key];

  if (type === 'DATE') {
    type = 'S';
  }

  return {
    AttributeName: key,
    AttributeType: type
  };
};

internals.keySchema = (hashKey, rangeKey) => {
  const result = [{
    AttributeName: hashKey,
    KeyType: 'HASH'
  }];

  if (rangeKey) {
    result.push({
      AttributeName: rangeKey,
      KeyType: 'RANGE'
    });
  }

  return result;
};

internals.secondaryIndex = (schema, params) => {
  const projection = params.projection || { ProjectionType: 'ALL' };

  return {
    IndexName: params.name,
    KeySchema: internals.keySchema(schema.hashKey, params.rangeKey),
    Projection: projection
  };
};

internals.globalIndex = (indexName, params) => {
  const projection = params.projection || { ProjectionType: 'ALL' };

  return {
    IndexName: indexName,
    KeySchema: internals.keySchema(params.hashKey, params.rangeKey),
    Projection: projection,
    ProvisionedThroughput: {
      ReadCapacityUnits: params.readCapacity || 1,
      WriteCapacityUnits: params.writeCapacity || 1
    }
  };
};

Table.prototype.createTable = function (options, callback) {
  const self = this;

  if (typeof options === 'function' && !callback) {
    callback = options;
    options = {};
  }
  const attributeDefinitions = [];

  attributeDefinitions.push(internals.attributeDefinition(self.schema, self.schema.hashKey));

  if (self.schema.rangeKey) {
    attributeDefinitions.push(internals.attributeDefinition(self.schema, self.schema.rangeKey));
  }

  const localSecondaryIndexes = [];

  _.forEach(self.schema.secondaryIndexes, params => {
    attributeDefinitions.push(internals.attributeDefinition(self.schema, params.rangeKey));
    localSecondaryIndexes.push(internals.secondaryIndex(self.schema, params));
  });

  const globalSecondaryIndexes = [];

  _.forEach(self.schema.globalIndexes, (params, indexName) => {
    if (!_.find(attributeDefinitions, { AttributeName: params.hashKey })) {
      attributeDefinitions.push(internals.attributeDefinition(self.schema, params.hashKey));
    }

    if (params.rangeKey && !_.find(attributeDefinitions, { AttributeName: params.rangeKey })) {
      attributeDefinitions.push(internals.attributeDefinition(self.schema, params.rangeKey));
    }

    globalSecondaryIndexes.push(internals.globalIndex(indexName, params));
  });

  const keySchema = internals.keySchema(self.schema.hashKey, self.schema.rangeKey);

  const params = {
    AttributeDefinitions: attributeDefinitions,
    TableName: self.tableName(),
    KeySchema: keySchema,
    ProvisionedThroughput: {
      ReadCapacityUnits: options.readCapacity || 1,
      WriteCapacityUnits: options.writeCapacity || 1
    }
  };

  if (localSecondaryIndexes.length >= 1) {
    params.LocalSecondaryIndexes = localSecondaryIndexes;
  }

  if (globalSecondaryIndexes.length >= 1) {
    params.GlobalSecondaryIndexes = globalSecondaryIndexes;
  }

  if (options.hasOwnProperty('streamSpecification') && typeof options.streamSpecification === 'object') {
    params.StreamSpecification = {
      StreamEnabled: options.streamSpecification.streamEnabled || false
    };
    if (params.StreamSpecification.StreamEnabled) {
      params.StreamSpecification.StreamViewType = options.streamSpecification.streamViewType || 'NEW_AND_OLD_IMAGES';
    }
  }

  self.sendRequest('createTable', params, callback);
};

Table.prototype.describeTable = function (callback) {
  const params = {
    TableName: this.tableName(),
  };

  this.sendRequest('describeTable', params, callback);
};

Table.prototype.deleteTable = function (callback) {
  callback = callback || _.noop;

  const params = {
    TableName: this.tableName(),
  };

  this.sendRequest('deleteTable', params, callback);
};

Table.prototype.updateTable = function (throughput, callback) {
  const self = this;
  if (typeof throughput === 'function' && !callback) {
    callback = throughput;
    throughput = {};
  }

  callback = callback || _.noop;
  throughput = throughput || {};

  async.parallel([
    async.apply(internals.syncIndexes, self),
    async.apply(internals.updateTableCapacity, self, throughput),
  ], callback);
};

internals.updateTableCapacity = (table, throughput, callback) => {
  const params = {};

  if (_.has(throughput, 'readCapacity') || _.has(throughput, 'writeCapacity')) {
    params.ProvisionedThroughput = {};

    if (_.has(throughput, 'readCapacity')) {
      params.ProvisionedThroughput.ReadCapacityUnits = throughput.readCapacity;
    }

    if (_.has(throughput, 'writeCapacity')) {
      params.ProvisionedThroughput.WriteCapacityUnits = throughput.writeCapacity;
    }
  }

  if (!_.isEmpty(params)) {
    params.TableName = table.tableName();
    table.sendRequest('updateTable', params, callback);
  } else {
    return callback();
  }
};

internals.syncIndexes = (table, callback) => {
  callback = callback || _.noop;

  table.describeTable((err, data) => {
    if (err) {
      return callback(err);
    }

    const missing = _.values(internals.findMissingGlobalIndexes(table, data));
    if (_.isEmpty(missing)) {
      return callback();
    }

    // UpdateTable only allows one new index per UpdateTable call
    // http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.OnlineOps.html#GSI.OnlineOps.Creating
    const maxIndexCreationsAtaTime = 5;
    async.mapLimit(missing, maxIndexCreationsAtaTime, (params, callback) => {
      const attributeDefinitions = [];

      if (!_.find(attributeDefinitions, { AttributeName: params.hashKey })) {
        attributeDefinitions.push(internals.attributeDefinition(table.schema, params.hashKey));
      }

      if (params.rangeKey && !_.find(attributeDefinitions, { AttributeName: params.rangeKey })) {
        attributeDefinitions.push(internals.attributeDefinition(table.schema, params.rangeKey));
      }

      const currentWriteThroughput = data.Table.ProvisionedThroughput.WriteCapacityUnits;
      const newIndexWriteThroughput = _.ceil(currentWriteThroughput * 1.5);
      params.writeCapacity = params.writeCapacity || newIndexWriteThroughput;

      table.log.info('adding index %s to table %s', params.name, table.tableName());

      const updateParams = {
        TableName: table.tableName(),
        AttributeDefinitions: attributeDefinitions,
        GlobalSecondaryIndexUpdates: [{ Create: internals.globalIndex(params.name, params) }]
      };

      table.sendRequest('updateTable', updateParams, callback);
    }, callback);
  });
};

internals.findMissingGlobalIndexes = (table, data) => {
  if (_.isNull(data) || _.isUndefined(data)) {
    // table does not exist
    return table.schema.globalIndexes;
  } else {
    const indexData = _.get(data, 'Table.GlobalSecondaryIndexes');
    const existingIndexNames = _.map(indexData, 'IndexName');

    const missing = _.reduce(table.schema.globalIndexes, (result, idx, indexName) => {
      if (!_.includes(existingIndexNames, idx.name)) {
        result[indexName] = idx;
      }

      return result;
    }, {});

    return missing;
  }
};