universal-s3
Version:
Universal S3 SDK for JavaScript, available for Node.js backends
616 lines (563 loc) • 22.6 kB
JavaScript
var AWS = require('./core');
var SequentialExecutor = require('./sequential_executor');
var DISCOVER_ENDPOINT = require('./discover_endpoint').discoverEndpoint;
/**
* The namespace used to register global event listeners for request building
* and sending.
*/
AWS.EventListeners = {
/**
* @!attribute VALIDATE_CREDENTIALS
* A request listener that validates whether the request is being
* sent with credentials.
* Handles the {AWS.Request~validate 'validate' Request event}
* @example Sending a request without validating credentials
* var listener = AWS.EventListeners.Core.VALIDATE_CREDENTIALS;
* request.removeListener('validate', listener);
* @readonly
* @return [Function]
* @!attribute VALIDATE_REGION
* A request listener that validates whether the region is set
* for a request.
* Handles the {AWS.Request~validate 'validate' Request event}
* @example Sending a request without validating region configuration
* var listener = AWS.EventListeners.Core.VALIDATE_REGION;
* request.removeListener('validate', listener);
* @readonly
* @return [Function]
* @!attribute VALIDATE_PARAMETERS
* A request listener that validates input parameters in a request.
* Handles the {AWS.Request~validate 'validate' Request event}
* @example Sending a request without validating parameters
* var listener = AWS.EventListeners.Core.VALIDATE_PARAMETERS;
* request.removeListener('validate', listener);
* @example Disable parameter validation globally
* AWS.EventListeners.Core.removeListener('validate',
* AWS.EventListeners.Core.VALIDATE_REGION);
* @readonly
* @return [Function]
* @!attribute SEND
* A request listener that initiates the HTTP connection for a
* request being sent. Handles the {AWS.Request~send 'send' Request event}
* @example Replacing the HTTP handler
* var listener = AWS.EventListeners.Core.SEND;
* request.removeListener('send', listener);
* request.on('send', function(response) {
* customHandler.send(response);
* });
* @return [Function]
* @readonly
* @!attribute HTTP_DATA
* A request listener that reads data from the HTTP connection in order
* to build the response data.
* Handles the {AWS.Request~httpData 'httpData' Request event}.
* Remove this handler if you are overriding the 'httpData' event and
* do not want extra data processing and buffering overhead.
* @example Disabling default data processing
* var listener = AWS.EventListeners.Core.HTTP_DATA;
* request.removeListener('httpData', listener);
* @return [Function]
* @readonly
*/
Core: {} /* doc hack */
};
/**
* @api private
*/
function getOperationAuthtype(req) {
if (!req.service.api.operations) {
return '';
}
var operation = req.service.api.operations[req.operation];
return operation ? operation.authtype : '';
}
AWS.EventListeners = {
Core: new SequentialExecutor().addNamedListeners(function(add, addAsync) {
addAsync('VALIDATE_CREDENTIALS', 'validate',
function VALIDATE_CREDENTIALS(req, done) {
if (!req.service.api.signatureVersion && !req.service.config.signatureVersion) return done(); // none
req.service.config.getCredentials(function(err) {
if (err) {
req.response.error = AWS.util.error(err,
{code: 'CredentialsError', message: 'Missing credentials in config'});
}
done();
});
});
add('VALIDATE_REGION', 'validate', function VALIDATE_REGION(req) {
if (!req.service.config.region && !req.service.isGlobalEndpoint) {
req.response.error = AWS.util.error(new Error(),
{code: 'ConfigError', message: 'Missing region in config'});
}
});
add('BUILD_IDEMPOTENCY_TOKENS', 'validate', function BUILD_IDEMPOTENCY_TOKENS(req) {
if (!req.service.api.operations) {
return;
}
var operation = req.service.api.operations[req.operation];
if (!operation) {
return;
}
var idempotentMembers = operation.idempotentMembers;
if (!idempotentMembers.length) {
return;
}
// creates a copy of params so user's param object isn't mutated
var params = AWS.util.copy(req.params);
for (var i = 0, iLen = idempotentMembers.length; i < iLen; i++) {
if (!params[idempotentMembers[i]]) {
// add the member
params[idempotentMembers[i]] = AWS.util.uuid.v4();
}
}
req.params = params;
});
add('VALIDATE_PARAMETERS', 'validate', function VALIDATE_PARAMETERS(req) {
if (!req.service.api.operations) {
return;
}
var rules = req.service.api.operations[req.operation].input;
var validation = req.service.config.paramValidation;
new AWS.ParamValidator(validation).validate(rules, req.params);
});
addAsync('COMPUTE_SHA256', 'afterBuild', function COMPUTE_SHA256(req, done) {
req.haltHandlersOnError();
if (!req.service.api.operations) {
return;
}
var operation = req.service.api.operations[req.operation];
var authtype = operation ? operation.authtype : '';
if (!req.service.api.signatureVersion && !authtype && !req.service.config.signatureVersion) return done(); // none
if (req.service.getSignerClass(req) === AWS.Signers.V4) {
var body = req.httpRequest.body || '';
if (authtype.indexOf('unsigned-body') >= 0) {
req.httpRequest.headers['X-Amz-Content-Sha256'] = 'UNSIGNED-PAYLOAD';
return done();
}
AWS.util.computeSha256(body, function(err, sha) {
if (err) {
done(err);
}
else {
req.httpRequest.headers['X-Amz-Content-Sha256'] = sha;
done();
}
});
} else {
done();
}
});
add('SET_CONTENT_LENGTH', 'afterBuild', function SET_CONTENT_LENGTH(req) {
var authtype = getOperationAuthtype(req);
var payloadMember = AWS.util.getRequestPayloadShape(req);
if (req.httpRequest.headers['Content-Length'] === undefined) {
try {
var length = AWS.util.string.byteLength(req.httpRequest.body);
req.httpRequest.headers['Content-Length'] = length;
} catch (err) {
if (payloadMember && payloadMember.isStreaming) {
if (payloadMember.requiresLength) {
//streaming payload requires length(s3, glacier)
throw err;
} else if (authtype.indexOf('unsigned-body') >= 0) {
//unbounded streaming payload(lex, mediastore)
req.httpRequest.headers['Transfer-Encoding'] = 'chunked';
return;
} else {
throw err;
}
}
throw err;
}
}
});
add('SET_HTTP_HOST', 'afterBuild', function SET_HTTP_HOST(req) {
req.httpRequest.headers['Host'] = req.httpRequest.endpoint.host;
});
add('RESTART', 'restart', function RESTART() {
var err = this.response.error;
if (!err || !err.retryable) return;
this.httpRequest = new AWS.HttpRequest(
this.service.endpoint,
this.service.region
);
if (this.response.retryCount < this.service.config.maxRetries) {
this.response.retryCount++;
} else {
this.response.error = null;
}
});
var addToHead = true;
addAsync('DISCOVER_ENDPOINT', 'sign', DISCOVER_ENDPOINT, addToHead);
addAsync('SIGN', 'sign', function SIGN(req, done) {
var service = req.service;
var operations = req.service.api.operations || {};
var operation = operations[req.operation];
var authtype = operation ? operation.authtype : '';
if (!service.api.signatureVersion && !authtype && !service.config.signatureVersion) return done(); // none
service.config.getCredentials(function (err, credentials) {
if (err) {
req.response.error = err;
return done();
}
try {
var date = service.getSkewCorrectedDate();
var SignerClass = service.getSignerClass(req);
var signer = new SignerClass(req.httpRequest,
service.api.signingName || service.api.endpointPrefix,
{
signatureCache: service.config.signatureCache,
operation: operation,
signatureVersion: service.api.signatureVersion
});
signer.setServiceClientId(service._clientId);
// clear old authorization headers
delete req.httpRequest.headers['Authorization'];
delete req.httpRequest.headers['Date'];
delete req.httpRequest.headers['X-Amz-Date'];
// add new authorization
signer.addAuthorization(credentials, date);
req.signedAt = date;
} catch (e) {
req.response.error = e;
}
done();
});
});
add('VALIDATE_RESPONSE', 'validateResponse', function VALIDATE_RESPONSE(resp) {
if (this.service.successfulResponse(resp, this)) {
resp.data = {};
resp.error = null;
} else {
resp.data = null;
resp.error = AWS.util.error(new Error(),
{code: 'UnknownError', message: 'An unknown error occurred.'});
}
});
addAsync('SEND', 'send', function SEND(resp, done) {
resp.httpResponse._abortCallback = done;
resp.error = null;
resp.data = null;
function callback(httpResp) {
resp.httpResponse.stream = httpResp;
var stream = resp.request.httpRequest.stream;
var service = resp.request.service;
var api = service.api;
var operationName = resp.request.operation;
var operation = api.operations[operationName] || {};
httpResp.on('headers', function onHeaders(statusCode, headers, statusMessage) {
resp.request.emit(
'httpHeaders',
[statusCode, headers, resp, statusMessage]
);
if (!resp.httpResponse.streaming) {
if (AWS.HttpClient.streamsApiVersion === 2) { // streams2 API check
// if we detect event streams, we're going to have to
// return the stream immediately
if (operation.hasEventOutput && service.successfulResponse(resp)) {
// skip reading the IncomingStream
resp.request.emit('httpDone');
done();
return;
}
httpResp.on('readable', function onReadable() {
var data = httpResp.read();
if (data !== null) {
resp.request.emit('httpData', [data, resp]);
}
});
} else { // legacy streams API
httpResp.on('data', function onData(data) {
resp.request.emit('httpData', [data, resp]);
});
}
}
});
httpResp.on('end', function onEnd() {
if (!stream || !stream.didCallback) {
if (AWS.HttpClient.streamsApiVersion === 2 && (operation.hasEventOutput && service.successfulResponse(resp))) {
// don't concatenate response chunks when streaming event stream data when response is successful
return;
}
resp.request.emit('httpDone');
done();
}
});
}
function progress(httpResp) {
httpResp.on('sendProgress', function onSendProgress(value) {
resp.request.emit('httpUploadProgress', [value, resp]);
});
httpResp.on('receiveProgress', function onReceiveProgress(value) {
resp.request.emit('httpDownloadProgress', [value, resp]);
});
}
function error(err) {
if (err.code !== 'RequestAbortedError') {
var errCode = err.code === 'TimeoutError' ? err.code : 'NetworkingError';
err = AWS.util.error(err, {
code: errCode,
region: resp.request.httpRequest.region,
hostname: resp.request.httpRequest.endpoint.hostname,
retryable: true
});
}
resp.error = err;
resp.request.emit('httpError', [resp.error, resp], function() {
done();
});
}
function executeSend() {
var http = AWS.HttpClient.getInstance();
var httpOptions = resp.request.service.config.httpOptions || {};
try {
var stream = http.handleRequest(resp.request.httpRequest, httpOptions,
callback, error);
progress(stream);
} catch (err) {
error(err);
}
}
var timeDiff = (resp.request.service.getSkewCorrectedDate() - this.signedAt) / 1000;
if (timeDiff >= 60 * 10) { // if we signed 10min ago, re-sign
this.emit('sign', [this], function(err) {
if (err) done(err);
else executeSend();
});
} else {
executeSend();
}
});
add('HTTP_HEADERS', 'httpHeaders',
function HTTP_HEADERS(statusCode, headers, resp, statusMessage) {
resp.httpResponse.statusCode = statusCode;
resp.httpResponse.statusMessage = statusMessage;
resp.httpResponse.headers = headers;
resp.httpResponse.body = AWS.util.buffer.toBuffer('');
resp.httpResponse.buffers = [];
resp.httpResponse.numBytes = 0;
var dateHeader = headers.date || headers.Date;
var service = resp.request.service;
if (dateHeader) {
var serverTime = Date.parse(dateHeader);
if (service.config.correctClockSkew
&& service.isClockSkewed(serverTime)) {
service.applyClockOffset(serverTime);
}
}
});
add('HTTP_DATA', 'httpData', function HTTP_DATA(chunk, resp) {
if (chunk) {
if (AWS.util.isNode()) {
resp.httpResponse.numBytes += chunk.length;
var total = resp.httpResponse.headers['content-length'];
var progress = { loaded: resp.httpResponse.numBytes, total: total };
resp.request.emit('httpDownloadProgress', [progress, resp]);
}
resp.httpResponse.buffers.push(AWS.util.buffer.toBuffer(chunk));
}
});
add('HTTP_DONE', 'httpDone', function HTTP_DONE(resp) {
// convert buffers array into single buffer
if (resp.httpResponse.buffers && resp.httpResponse.buffers.length > 0) {
var body = AWS.util.buffer.concat(resp.httpResponse.buffers);
resp.httpResponse.body = body;
}
delete resp.httpResponse.numBytes;
delete resp.httpResponse.buffers;
});
add('FINALIZE_ERROR', 'retry', function FINALIZE_ERROR(resp) {
if (resp.httpResponse.statusCode) {
resp.error.statusCode = resp.httpResponse.statusCode;
if (resp.error.retryable === undefined) {
resp.error.retryable = this.service.retryableError(resp.error, this);
}
}
});
add('INVALIDATE_CREDENTIALS', 'retry', function INVALIDATE_CREDENTIALS(resp) {
if (!resp.error) return;
switch (resp.error.code) {
case 'RequestExpired': // EC2 only
case 'ExpiredTokenException':
case 'ExpiredToken':
resp.error.retryable = true;
resp.request.service.config.credentials.expired = true;
}
});
add('EXPIRED_SIGNATURE', 'retry', function EXPIRED_SIGNATURE(resp) {
var err = resp.error;
if (!err) return;
if (typeof err.code === 'string' && typeof err.message === 'string') {
if (err.code.match(/Signature/) && err.message.match(/expired/)) {
resp.error.retryable = true;
}
}
});
add('CLOCK_SKEWED', 'retry', function CLOCK_SKEWED(resp) {
if (!resp.error) return;
if (this.service.clockSkewError(resp.error)
&& this.service.config.correctClockSkew) {
resp.error.retryable = true;
}
});
add('REDIRECT', 'retry', function REDIRECT(resp) {
if (resp.error && resp.error.statusCode >= 300 &&
resp.error.statusCode < 400 && resp.httpResponse.headers['location']) {
this.httpRequest.endpoint =
new AWS.Endpoint(resp.httpResponse.headers['location']);
this.httpRequest.headers['Host'] = this.httpRequest.endpoint.host;
resp.error.redirect = true;
resp.error.retryable = true;
}
});
add('RETRY_CHECK', 'retry', function RETRY_CHECK(resp) {
if (resp.error) {
if (resp.error.redirect && resp.redirectCount < resp.maxRedirects) {
resp.error.retryDelay = 0;
} else if (resp.retryCount < resp.maxRetries) {
resp.error.retryDelay = this.service.retryDelays(resp.retryCount) || 0;
}
}
});
addAsync('RESET_RETRY_STATE', 'afterRetry', function RESET_RETRY_STATE(resp, done) {
var delay, willRetry = false;
if (resp.error) {
delay = resp.error.retryDelay || 0;
if (resp.error.retryable && resp.retryCount < resp.maxRetries) {
resp.retryCount++;
willRetry = true;
} else if (resp.error.redirect && resp.redirectCount < resp.maxRedirects) {
resp.redirectCount++;
willRetry = true;
}
}
if (willRetry) {
resp.error = null;
setTimeout(done, delay);
} else {
done();
}
});
}),
CorePost: new SequentialExecutor().addNamedListeners(function(add) {
add('EXTRACT_REQUEST_ID', 'extractData', AWS.util.extractRequestId);
add('EXTRACT_REQUEST_ID', 'extractError', AWS.util.extractRequestId);
add('ENOTFOUND_ERROR', 'httpError', function ENOTFOUND_ERROR(err) {
if (err.code === 'NetworkingError' && err.errno === 'ENOTFOUND') {
var message = 'Inaccessible host: `' + err.hostname +
'\'. This service may not be available in the `' + err.region +
'\' region.';
this.response.error = AWS.util.error(new Error(message), {
code: 'UnknownEndpoint',
region: err.region,
hostname: err.hostname,
retryable: true,
originalError: err
});
}
});
}),
Logger: new SequentialExecutor().addNamedListeners(function(add) {
add('LOG_REQUEST', 'complete', function LOG_REQUEST(resp) {
var req = resp.request;
var logger = req.service.config.logger;
if (!logger) return;
function filterSensitiveLog(inputShape, shape) {
if (!shape) {
return shape;
}
switch (inputShape.type) {
case 'structure':
var struct = {};
AWS.util.each(shape, function(subShapeName, subShape) {
if (Object.prototype.hasOwnProperty.call(inputShape.members, subShapeName)) {
struct[subShapeName] = filterSensitiveLog(inputShape.members[subShapeName], subShape);
} else {
struct[subShapeName] = subShape;
}
});
return struct;
case 'list':
var list = [];
AWS.util.arrayEach(shape, function(subShape, index) {
list.push(filterSensitiveLog(inputShape.member, subShape));
});
return list;
case 'map':
var map = {};
AWS.util.each(shape, function(key, value) {
map[key] = filterSensitiveLog(inputShape.value, value);
});
return map;
default:
if (inputShape.isSensitive) {
return '***SensitiveInformation***';
} else {
return shape;
}
}
}
function buildMessage() {
var time = resp.request.service.getSkewCorrectedDate().getTime();
var delta = (time - req.startTime.getTime()) / 1000;
var ansi = logger.isTTY ? true : false;
var status = resp.httpResponse.statusCode;
var censoredParams = req.params;
if (
req.service.api.operations &&
req.service.api.operations[req.operation] &&
req.service.api.operations[req.operation].input
) {
var inputShape = req.service.api.operations[req.operation].input;
censoredParams = filterSensitiveLog(inputShape, req.params);
}
var params = require('util').inspect(censoredParams, true, null);
var message = '';
if (ansi) message += '\x1B[33m';
message += '[AWS ' + req.service.serviceIdentifier + ' ' + status;
message += ' ' + delta.toString() + 's ' + resp.retryCount + ' retries]';
if (ansi) message += '\x1B[0;1m';
message += ' ' + AWS.util.string.lowerFirst(req.operation);
message += '(' + params + ')';
if (ansi) message += '\x1B[0m';
return message;
}
var line = buildMessage();
if (typeof logger.log === 'function') {
logger.log(line);
} else if (typeof logger.write === 'function') {
logger.write(line + '\n');
}
});
}),
Json: new SequentialExecutor().addNamedListeners(function(add) {
var svc = require('./protocol/json');
add('BUILD', 'build', svc.buildRequest);
add('EXTRACT_DATA', 'extractData', svc.extractData);
add('EXTRACT_ERROR', 'extractError', svc.extractError);
}),
Rest: new SequentialExecutor().addNamedListeners(function(add) {
var svc = require('./protocol/rest');
add('BUILD', 'build', svc.buildRequest);
add('EXTRACT_DATA', 'extractData', svc.extractData);
add('EXTRACT_ERROR', 'extractError', svc.extractError);
}),
RestJson: new SequentialExecutor().addNamedListeners(function(add) {
var svc = require('./protocol/rest_json');
add('BUILD', 'build', svc.buildRequest);
add('EXTRACT_DATA', 'extractData', svc.extractData);
add('EXTRACT_ERROR', 'extractError', svc.extractError);
}),
RestXml: new SequentialExecutor().addNamedListeners(function(add) {
var svc = require('./protocol/rest_xml');
add('BUILD', 'build', svc.buildRequest);
add('EXTRACT_DATA', 'extractData', svc.extractData);
add('EXTRACT_ERROR', 'extractError', svc.extractError);
}),
Query: new SequentialExecutor().addNamedListeners(function(add) {
var svc = require('./protocol/query');
add('BUILD', 'build', svc.buildRequest);
add('EXTRACT_DATA', 'extractData', svc.extractData);
add('EXTRACT_ERROR', 'extractError', svc.extractError);
})
};