mock-aws-s3
Version:
Mock AWS S3 SDK for Node.js
643 lines (533 loc) • 15.6 kB
JavaScript
/*
* grunt-mock-s3
* https://github.com/MathieuLoutre/grunt-mock-s3
*
* Copyright (c) 2013 Mathieu Triay
* Licensed under the MIT license.
*/
var _ = require('underscore')
var fs = require('fs-extra')
var crypto = require('crypto')
var path = require('path')
var Buffer = require('buffer').Buffer
var Promise = require('bluebird')
var path = require('path')
var config = {}
// Gathered from http://stackoverflow.com/questions/5827612/node-js-fs-readdir-recursive-directory-search
function walk (dir) {
var results = []
var list = fs.readdirSync(dir)
list.forEach(function (file) {
file = dir + '/' + file
var stat = fs.statSync(file)
if (stat && stat.isDirectory()) {
results = results.concat(walk(file))
}
else {
results.push(file)
}
})
return results
}
/** Add basePath to selected keys */
function applyBasePath (search) {
if (_.isUndefined(config.basePath)) {
return search
}
var modifyKeys = ['Bucket', 'CopySource']
var ret = _.mapObject(search, function (value, key) {
if (_.indexOf(modifyKeys, key) === -1) {
return value
}
else {
if (config.basePath == null || value == null) {
return
}
return path.join(config.basePath, value)
}
})
return ret
}
/** FakeStream object for mocking S3 streams */
function FakeStream (search) {
this.src = search.Bucket + '/' + search.Key
}
FakeStream.prototype.createReadStream = function () {
return fs.createReadStream(this.src)
}
/**
* Decorate a method to enable calling `.promise()` on the returned value to get a Promise, when the method
* is initially called without a callback.
* @decorator
* @private
*/
function createPromisable (wrapped) {
return function () {
var self = this
var promisified = Promise.promisify(wrapped, { context: self })
var args = [].slice.call(arguments)
var callback = null
var lastArgIndex = Math.min(args.length - 1, wrapped.length)
if (args.length >= 1) {
var lastArg = args[lastArgIndex]
if (_.isFunction(lastArg)) {
callback = lastArg
}
}
if (!_.isFunction(callback)) {
return {
createReadStream: function () {
this.send()
if (this._returned instanceof FakeStream) {
return this._returned.createReadStream()
}
},
promise: function () {
return promisified.apply(self, args)
},
send: function (cb) {
args.push(cb)
this._returned = wrapped.apply(self, args)
}
}
}
else {
wrapped.apply(self, args)
}
}
}
/** Mocks key pieces of the amazon s3 sdk */
function S3Mock (options) {
if (!_.isUndefined(options) && !_.isUndefined(options.params)) {
this.defaultOptions = _.extend({}, applyBasePath(options.params))
}
this.config = {
update: function () {}
}
}
S3Mock.prototype = {
objectMetadataDictionary: [],
objectTaggingDictionary: [],
listObjectsV2: function (searchV2, callback) {
var searchV1 = _(searchV2).clone()
// Marker in V1 is StartAfter in V2
// ContinuationToken trumps marker on subsequent requests.
searchV1.Marker = searchV2.ContinuationToken || searchV2.StartAfter
this.listObjects(searchV1, function (err, resultV1) {
var resultV2 = _(resultV1).clone()
// Rewrite NextMarker to NextContinuationToken
resultV2.NextContinuationToken = resultV1.NextMarker
// Remember original ContinuationToken and StartAfter
resultV2.ContinuationToken = searchV2.ContinuationToken
resultV2.StartAfter = searchV2.StartAfter
callback(err, resultV2)
})
},
listObjects: function (search, callback) {
search = _.extend({}, this.defaultOptions, applyBasePath(search))
var files = walk(search.Bucket)
var filteredFiles = _.filter(files, function (file) {
return !search.Prefix || file.replace(search.Bucket + '/', '').indexOf(search.Prefix) === 0
})
var start = 0
var truncated = false
if (search.Marker) {
var isPartial
var markerFile = _(filteredFiles).find(function (file) {
var marker = search.Bucket + '/' + search.Marker
if (file.indexOf(marker) === 0) {
isPartial = file.length !== marker.length
return true
}
})
var startFile
if (isPartial) {
startFile = filteredFiles[filteredFiles.indexOf(markerFile)]
}
else {
startFile = filteredFiles[filteredFiles.indexOf(markerFile) + 1]
}
start = filteredFiles.indexOf(startFile)
}
if (start === -1) {
filteredFiles = []
}
else {
filteredFiles = _.rest(filteredFiles, start)
}
if (filteredFiles.length > Math.min(1000, search.MaxKeys || 1000)) {
truncated = true
filteredFiles = filteredFiles.slice(0, Math.min(1000, search.MaxKeys || 1000))
}
var result = {
Contents: _.map(filteredFiles, function (path) {
var stat = fs.statSync(path)
return {
Key: path.replace(search.Bucket + '/', ''),
ETag: '"' + crypto.createHash('md5').update(fs.readFileSync(path)).digest('hex') + '"',
LastModified: stat.mtime,
Size: stat.size
}
}),
CommonPrefixes: _.reduce(filteredFiles, function (prefixes, path) {
var prefix = path
.replace(search.Bucket + '/', '')
.split('/')
.slice(0, -1)
.join('/')
.concat('/')
return prefixes.indexOf(prefix) === -1 ? prefixes.concat([prefix]) : prefixes
}, []).map(function (prefix) {
return {
Prefix: prefix
}
}),
IsTruncated: truncated
}
if (search.Marker) {
result.Marker = search.Marker
}
if (truncated && search.Delimiter) {
result.NextMarker = _.last(result.Contents).Key
}
callback(null, result)
},
deleteObjects: function (search, callback) {
search = _.extend({}, this.defaultOptions, applyBasePath(search))
var deleted = []
var errors = []
_.each(search.Delete.Objects, function (file) {
if (fs.existsSync(search.Bucket + '/' + file.Key)) {
deleted.push(file)
fs.unlinkSync(search.Bucket + '/' + file.Key)
}
else {
errors.push(file)
}
})
if (errors.length > 0) {
callback('Error deleting objects', { Errors: errors, Deleted: deleted })
}
else {
callback(null, { Deleted: deleted })
}
},
deleteObject: function (search, callback) {
search = _.extend({}, this.defaultOptions, applyBasePath(search))
if (fs.existsSync(search.Bucket + '/' + search.Key)) {
fs.unlinkSync(search.Bucket + '/' + search.Key)
callback(null, true)
}
else {
callback(null, {})
}
},
headObject: function (search, callback) {
var self = this
search = _.extend({}, this.defaultOptions, applyBasePath(search))
if (!callback) {
return new FakeStream(search)
}
else {
fs.readFile(search.Bucket + '/' + search.Key, function (err, data) {
if (!err) {
var props = {
Key: search.Key,
ETag: '"' + crypto.createHash('md5').update(data).digest('hex') + '"',
ContentLength: data.length
}
if (self.objectMetadataDictionary[search.Key]) {
props.Metadata = self.objectMetadataDictionary[search.Key]
}
callback(null, props)
}
else {
if (err.code === 'ENOENT') {
err.statusCode = 404
}
callback(err, search)
}
})
}
},
getObject: function (search, callback) {
var self = this
search = _.extend({}, this.defaultOptions, applyBasePath(search))
if (!callback) {
return new FakeStream(search)
}
else {
var path = search.Bucket + '/' + search.Key
fs.readFile(path, function (err, data) {
if (!err) {
var stat = fs.statSync(path)
var props = {
Key: search.Key,
ETag: '"' + crypto.createHash('md5').update(data).digest('hex') + '"',
Body: data,
LastModified: stat.mtime,
ContentLength: data.length
}
if (self.objectMetadataDictionary[search.Key]) {
props.Metadata = self.objectMetadataDictionary[search.Key]
}
callback(null, props)
}
else {
if (err.code === 'ENOENT') {
return callback({
cfId: undefined,
code: 'NoSuchKey',
message: 'The specified key does not exist.',
name: 'NoSuchKey',
region: null,
statusCode: 404
}, search)
}
callback(err, search)
}
})
}
},
copyObject: function (search, callback) {
search = _.extend({}, this.defaultOptions, applyBasePath(search))
fs.mkdirsSync(path.dirname(search.Bucket + '/' + search.Key))
fs.copy(decodeURIComponent(search.CopySource), search.Bucket + '/' + search.Key, function (err, data) {
callback(err, search)
})
},
createBucket: function (params, callback) {
var err = null
// param prop tests - these need to be done here to avoid issues with defaulted values
if (typeof (params) === 'object' && params !== null) { // null is an object, at least in older V8's
// Bucket - required, String
if (typeof (params.Bucket) !== 'string' || params.Bucket.length <= 0) {
// NOTE: This *will not* match the error provided by the AWS SDK - but that's chasing a moving target
err = new Error("Mock-AWS-S3: Argument 'params' must contain a 'Bucket' (String) property")
}
// Should we check the remaining props of the params Object? (probably)
}
else {
err = new Error("Mock-AWS-S3: Argument 'params' must be an Object")
}
// Note: this.defaultOptions is an object which was passed in to the constructor
var opts = _.extend({}, this.defaultOptions, applyBasePath(params))
// If the params object is well-formed...
if (err === null) {
// We'll assume that if basePath is set, it's correctly set (i.e. data type etc.) and if not...
// we'll default to the local dir (which seems to be the existing behaviour - in e.g. putObject)
// It would be nicer if there were a strongly defined default
var bucketPath = opts.basePath || ''
bucketPath += opts.Bucket
fs.mkdirs(bucketPath, function (err) {
return callback(err)
})
}
else { // ...if the params object is not well-formed, fail fast
return callback(err)
}
},
/**
* Deletes a bucket. Behaviour as createBucket
* @param params {Bucket: bucketName}. Name of bucket to delete
* @param callback
* @returns void
*/
deleteBucket: function (params, callback) {
var err = null
if (typeof (params) === 'object' && params !== null) {
if (typeof (params.Bucket) !== 'string' || params.Bucket.length <= 0) {
err = new Error("Mock-AWS-S3: Argument 'params' must contain a 'Bucket' (String) property")
}
}
else {
err = new Error("Mock-AWS-S3: Argument 'params' must be an Object")
}
var opts = _.extend({}, this.defaultOptions, applyBasePath(params))
if (err !== null) {
callback(err)
}
var bucketPath = opts.basePath || ''
bucketPath += opts.Bucket
fs.remove(bucketPath, function (err) {
return callback(err)
})
},
putObject: function (search, callback) {
search = _.extend({}, this.defaultOptions, applyBasePath(search))
if (search.Metadata) {
this.objectMetadataDictionary[search.Key] = search.Metadata
}
if (typeof search.Tagging === 'string') {
// URL query parameter encoded
var tags = {}
var tagSet = []
// quick'n'dirty parsing into an object (does not support hashes or arrays)
search.Tagging.split('&').forEach(function (part) {
var item = part.split('=')
tags[decodeURIComponent(item[0])] = decodeURIComponent(item[1])
})
// expand into tagset
Object.keys(tags).forEach(function (key) {
tagSet.push({
Key: key,
Value: tags[key]
})
})
this.objectTaggingDictionary[search.Key] = tagSet
}
var dest = search.Bucket + '/' + search.Key
var sendCallback = null
var done = function () {
if (typeof sendCallback === 'function') {
sendCallback.apply(this, arguments)
}
if (typeof callback === 'function') {
callback.apply(this, arguments)
}
}
if (typeof search.Body === 'string') {
search.Body = Buffer.from(search.Body)
}
if (search.Body instanceof Buffer) {
fs.createFileSync(dest)
fs.writeFile(dest, search.Body, function (err) {
done(err, { Location: dest, Key: search.Key, Bucket: search.Bucket })
})
}
else {
fs.mkdirsSync(path.dirname(dest))
var stream = fs.createWriteStream(dest)
stream.on('finish', function () {
done(null, true)
})
search.Body.on('error', function (err) {
done(err)
})
stream.on('error', function (err) {
done(err)
})
search.Body.pipe(stream)
}
return {
send: function (cb) {
sendCallback = cb
}
}
},
getObjectTagging: function (search, callback) {
var self = this
this.headObject(search, function (err, props) {
if (err) {
return callback(err)
}
else {
return callback(null, {
VersionId: '1',
TagSet: self.objectTaggingDictionary[search.Key] || []
})
}
})
},
putObjectTagging: function (search, callback) {
var self = this
if (!search.Tagging || !search.Tagging.TagSet) {
return callback(new Error('Tagging.TagSet required'))
}
this.headObject(search, function (err, props) {
if (err) {
return callback(err)
}
else {
self.objectTaggingDictionary[search.Key] = search.Tagging.TagSet
return callback(null, {
VersionId: '1'
})
}
})
},
upload: function (search, options, callback) {
if (typeof options === 'function' && callback === undefined) {
callback = options
options = null
}
if (options && options.tags) {
if (!Array.isArray(options.tags)) {
return callback(new Error('Tags must be specified as an array; ' + typeof options.tags + ' provided'))
}
}
return this.putObject(search, function (err, data) {
if (options && options.tags) {
// https://github.com/aws/aws-sdk-js/pull/1425
return this.putObjectTagging({
Bucket: search.Bucket,
Key: search.Key,
Tagging: {
TagSet: options.tags
}
}, function (err) {
if (err) {
return callback(err)
}
return callback(null, data)
})
}
return callback(err, data)
}.bind(this))
},
getSignedUrl: function (operation, params, callback) {
var url = 'https://s3.us-east-2.amazonaws.com/' +
params.Bucket +
'/' + params.Key +
'?X-Amz-Date=20170720T182534Z&X-Amz-SignedHeaders=host' +
'&X-Amz-Credential=ASIAIYLQNVRRFNZOCFBA%2F20170720%2' +
'Fus-east-2%2Fs3%2Faws4_request&X-Amz-Algorithm=AWS4-HMAC-SHA256&X' +
'-Amz-Expires=604800&X-Amz-Security-Token=FQoDYXdzEJP%2F%2F%2F%2F%2' +
'F%2F%2F%2F%2F%2FwEaDOLWx95j90zPxGh7WSLdAVnoYoKC4gjrrR1xbokFWRRwutm' +
'uAmOxaIVcQqOy%2Fqxy%2FXQt3Iz%2FohuEEmI7%2FHPzShy%2BfgQtvfUeDaojrAx' +
'5q8fG9P1KuIfcedfkiU%2BCxpM2foyCGlXzoZuNlcF8ohm%2BaM3wh4%2BxQ%2FpSh' +
'Ll18cKiKEiw0QF1UQGj%2FsiEqzoM81vOSUVWL9SpTTkVq8EQHY1chYKBkBWt7eIQc' +
'xjTI2dQeYOohlrbnZ5Y1%2F1cxPgrbk6PkNFO3whAoliSjyRC8e4TSjIY2j3V6d9fU' +
'y4%2Fp6nLZIf9wuERL7xW9PjE6eZbKOHnw8sF&X-Amz-Signature=a14b3065ab82' +
'2105e8d7892eb5dcc455ddd603c61e47520774a7289178af9ecc'
switch (operation) {
case 'getObject':
this.headObject(params, function (err, props) {
if (err) {
err.statusCode = 404
}
if (callback) {
if (err) {
callback(err, null)
}
else {
callback(null, url)
}
}
else {
if (err) {
throw new Error(err)
}
return url
}
})
break
case 'putObject':
if (!params.Bucket || !params.Key) {
throw new Error({ statusCode: 404 })
}
return url
default:
}
}
}
_.forEach(S3Mock.prototype, function (value, key, obj) {
if (_.isFunction(value)) {
obj[key] = createPromisable(value)
}
})
exports.config = config
exports.S3 = function (options) {
return new S3Mock(options)
}