@bowtie/sls/src/utils/s3.js

Serverless helpers & utilities

const AWS = require('aws-sdk');
const async = require('async');

const s3 = new AWS.S3();

const filterItemsS3 = (item) => !!item;

const loadObjectsS3 = (Bucket, Prefix, MaxKeys = 100) => {
  return new Promise(
    (resolve, reject) => {
      let token, truncated;
      const objects = [];
      const objectMap = {};
      const objectKeys = [];

      async.doWhilst(
        (next) => {
          const params = {
            MaxKeys,
            Bucket,
            Prefix
          };

          if (token) {
            Object.assign(params, { ContinuationToken: token });
          }

          s3.listObjectsV2(params, (err, data) => {
            if (err) {
              return next(err);
            }

            token = data.NextContinuationToken;
            truncated = data.IsTruncated;

            data.Contents.filter(filterItemsS3).forEach((item) => {
              const relKey = item.Key.replace(Prefix, '');
              objects.push(item);
              objectMap[relKey] = item.ETag;
              objectKeys.push(relKey);
            })

            next();
          });
        },
        async () => truncated,
        (err) => {
          if (err) {
            return reject(err);
          }

          return resolve({
            objects,
            objectMap,
            objectKeys
          });
        }
      );
    }
  );
};

const compareS3 = (buckets) => {
  return new Promise(
    (resolve, reject) => {
      const data = {};

      async.each(Object.keys(buckets), async (group) => {
        data[group] = await loadObjectsS3(buckets[group].Bucket, buckets[group].Prefix);
      }, (err) => {
        if (err) {
          return reject(err);
        }

        const { source, destination } = data;

        resolve({
          add: source.objectKeys.filter(key => !destination.objectMap[key] || destination.objectMap[key] !== source.objectMap[key]),
          del: destination.objectKeys.filter(key => !source.objectMap[key])
        });
      });
    }
  )
};

const getActions = ({ source, destination }) => ({
  add: (keys, cb) => {
    async.each(keys, (key, next) => {
      s3.copyObject({
        Key: `${destination.Prefix}${key}`,
        Bucket: destination.Bucket,
        CopySource: `${source.Bucket}/${source.Prefix}${key}`
      }, next);
    }, cb);
  },
  del: (keys, cb) => {
    if (!keys || keys.length === 0) {
      return cb();
    }

    s3.deleteObjects({
      Bucket: destination.Bucket,
      Delete: {
        Quiet: false,
        Objects: keys.map(key => ({ Key: `${destination.Prefix}${key}` }))
      }
    }, cb);
  }
});

const syncS3 = (buckets) => {
  return new Promise(
    (resolve, reject) => {
      return compareS3(buckets).then((changes) => {
        const actions = getActions(buckets);

        async.each(Object.keys(changes), (action, next) => {
          actions[action](changes[action], next);
        }, (err) => {
          if (err) {
            return reject(err);
          }

          resolve(changes);
        })
      });
    }
  )
};

module.exports = {
  filterItemsS3,
  loadObjectsS3,
  compareS3,
  syncS3
};