minio
Version:
S3 Compatible Cloud Storage client
1,397 lines (1,341 loc) • 370 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", {
value: true
});
var crypto = _interopRequireWildcard(require("crypto"), true);
var fs = _interopRequireWildcard(require("fs"), true);
var http = _interopRequireWildcard(require("http"), true);
var https = _interopRequireWildcard(require("https"), true);
var path = _interopRequireWildcard(require("path"), true);
var stream = _interopRequireWildcard(require("stream"), true);
var async = _interopRequireWildcard(require("async"), true);
var _blockStream = require("block-stream2");
var _browserOrNode = require("browser-or-node");
var _lodash = require("lodash");
var qs = _interopRequireWildcard(require("query-string"), true);
var _xml2js = require("xml2js");
var _CredentialProvider = require("../CredentialProvider.js");
var errors = _interopRequireWildcard(require("../errors.js"), true);
var _helpers = require("../helpers.js");
var _signing = require("../signing.js");
var _async2 = require("./async.js");
var _copyConditions = require("./copy-conditions.js");
var _extensions = require("./extensions.js");
var _helper = require("./helper.js");
var _joinHostPort = require("./join-host-port.js");
var _postPolicy = require("./post-policy.js");
var _request = require("./request.js");
var _response = require("./response.js");
var _s3Endpoints = require("./s3-endpoints.js");
var xmlParsers = _interopRequireWildcard(require("./xml-parser.js"), true);
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
const xml = new _xml2js.Builder({
renderOpts: {
pretty: false
},
headless: true
});
// will be replaced by bundler.
const Package = {
version: "8.0.5" || 'development'
};
const requestOptionProperties = ['agent', 'ca', 'cert', 'ciphers', 'clientCertEngine', 'crl', 'dhparam', 'ecdhCurve', 'family', 'honorCipherOrder', 'key', 'passphrase', 'pfx', 'rejectUnauthorized', 'secureOptions', 'secureProtocol', 'servername', 'sessionIdContext'];
class TypedClient {
partSize = 64 * 1024 * 1024;
maximumPartSize = 5 * 1024 * 1024 * 1024;
maxObjectSize = 5 * 1024 * 1024 * 1024 * 1024;
constructor(params) {
// @ts-expect-error deprecated property
if (params.secure !== undefined) {
throw new Error('"secure" option deprecated, "useSSL" should be used instead');
}
// Default values if not specified.
if (params.useSSL === undefined) {
params.useSSL = true;
}
if (!params.port) {
params.port = 0;
}
// Validate input params.
if (!(0, _helper.isValidEndpoint)(params.endPoint)) {
throw new errors.InvalidEndpointError(`Invalid endPoint : ${params.endPoint}`);
}
if (!(0, _helper.isValidPort)(params.port)) {
throw new errors.InvalidArgumentError(`Invalid port : ${params.port}`);
}
if (!(0, _helper.isBoolean)(params.useSSL)) {
throw new errors.InvalidArgumentError(`Invalid useSSL flag type : ${params.useSSL}, expected to be of type "boolean"`);
}
// Validate region only if its set.
if (params.region) {
if (!(0, _helper.isString)(params.region)) {
throw new errors.InvalidArgumentError(`Invalid region : ${params.region}`);
}
}
const host = params.endPoint.toLowerCase();
let port = params.port;
let protocol;
let transport;
let transportAgent;
// Validate if configuration is not using SSL
// for constructing relevant endpoints.
if (params.useSSL) {
// Defaults to secure.
transport = https;
protocol = 'https:';
port = port || 443;
transportAgent = https.globalAgent;
} else {
transport = http;
protocol = 'http:';
port = port || 80;
transportAgent = http.globalAgent;
}
// if custom transport is set, use it.
if (params.transport) {
if (!(0, _helper.isObject)(params.transport)) {
throw new errors.InvalidArgumentError(`Invalid transport type : ${params.transport}, expected to be type "object"`);
}
transport = params.transport;
}
// if custom transport agent is set, use it.
if (params.transportAgent) {
if (!(0, _helper.isObject)(params.transportAgent)) {
throw new errors.InvalidArgumentError(`Invalid transportAgent type: ${params.transportAgent}, expected to be type "object"`);
}
transportAgent = params.transportAgent;
}
// User Agent should always following the below style.
// Please open an issue to discuss any new changes here.
//
// MinIO (OS; ARCH) LIB/VER APP/VER
//
const libraryComments = `(${process.platform}; ${process.arch})`;
const libraryAgent = `MinIO ${libraryComments} minio-js/${Package.version}`;
// User agent block ends.
this.transport = transport;
this.transportAgent = transportAgent;
this.host = host;
this.port = port;
this.protocol = protocol;
this.userAgent = `${libraryAgent}`;
// Default path style is true
if (params.pathStyle === undefined) {
this.pathStyle = true;
} else {
this.pathStyle = params.pathStyle;
}
this.accessKey = params.accessKey ?? '';
this.secretKey = params.secretKey ?? '';
this.sessionToken = params.sessionToken;
this.anonymous = !this.accessKey || !this.secretKey;
if (params.credentialsProvider) {
this.anonymous = false;
this.credentialsProvider = params.credentialsProvider;
}
this.regionMap = {};
if (params.region) {
this.region = params.region;
}
if (params.partSize) {
this.partSize = params.partSize;
this.overRidePartSize = true;
}
if (this.partSize < 5 * 1024 * 1024) {
throw new errors.InvalidArgumentError(`Part size should be greater than 5MB`);
}
if (this.partSize > 5 * 1024 * 1024 * 1024) {
throw new errors.InvalidArgumentError(`Part size should be less than 5GB`);
}
// SHA256 is enabled only for authenticated http requests. If the request is authenticated
// and the connection is https we use x-amz-content-sha256=UNSIGNED-PAYLOAD
// header for signature calculation.
this.enableSHA256 = !this.anonymous && !params.useSSL;
this.s3AccelerateEndpoint = params.s3AccelerateEndpoint || undefined;
this.reqOptions = {};
this.clientExtensions = new _extensions.Extensions(this);
}
/**
* Minio extensions that aren't necessary present for Amazon S3 compatible storage servers
*/
get extensions() {
return this.clientExtensions;
}
/**
* @param endPoint - valid S3 acceleration end point
*/
setS3TransferAccelerate(endPoint) {
this.s3AccelerateEndpoint = endPoint;
}
/**
* Sets the supported request options.
*/
setRequestOptions(options) {
if (!(0, _helper.isObject)(options)) {
throw new TypeError('request options should be of type "object"');
}
this.reqOptions = _lodash.pick(options, requestOptionProperties);
}
/**
* This is s3 Specific and does not hold validity in any other Object storage.
*/
getAccelerateEndPointIfSet(bucketName, objectName) {
if (!(0, _helper.isEmpty)(this.s3AccelerateEndpoint) && !(0, _helper.isEmpty)(bucketName) && !(0, _helper.isEmpty)(objectName)) {
// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html
// Disable transfer acceleration for non-compliant bucket names.
if (bucketName.includes('.')) {
throw new Error(`Transfer Acceleration is not supported for non compliant bucket:${bucketName}`);
}
// If transfer acceleration is requested set new host.
// For more details about enabling transfer acceleration read here.
// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html
return this.s3AccelerateEndpoint;
}
return false;
}
/**
* Set application specific information.
* Generates User-Agent in the following style.
* MinIO (OS; ARCH) LIB/VER APP/VER
*/
setAppInfo(appName, appVersion) {
if (!(0, _helper.isString)(appName)) {
throw new TypeError(`Invalid appName: ${appName}`);
}
if (appName.trim() === '') {
throw new errors.InvalidArgumentError('Input appName cannot be empty.');
}
if (!(0, _helper.isString)(appVersion)) {
throw new TypeError(`Invalid appVersion: ${appVersion}`);
}
if (appVersion.trim() === '') {
throw new errors.InvalidArgumentError('Input appVersion cannot be empty.');
}
this.userAgent = `${this.userAgent} ${appName}/${appVersion}`;
}
/**
* returns options object that can be used with http.request()
* Takes care of constructing virtual-host-style or path-style hostname
*/
getRequestOptions(opts) {
const method = opts.method;
const region = opts.region;
const bucketName = opts.bucketName;
let objectName = opts.objectName;
const headers = opts.headers;
const query = opts.query;
let reqOptions = {
method,
headers: {},
protocol: this.protocol,
// If custom transportAgent was supplied earlier, we'll inject it here
agent: this.transportAgent
};
// Verify if virtual host supported.
let virtualHostStyle;
if (bucketName) {
virtualHostStyle = (0, _helper.isVirtualHostStyle)(this.host, this.protocol, bucketName, this.pathStyle);
}
let path = '/';
let host = this.host;
let port;
if (this.port) {
port = this.port;
}
if (objectName) {
objectName = (0, _helper.uriResourceEscape)(objectName);
}
// For Amazon S3 endpoint, get endpoint based on region.
if ((0, _helper.isAmazonEndpoint)(host)) {
const accelerateEndPoint = this.getAccelerateEndPointIfSet(bucketName, objectName);
if (accelerateEndPoint) {
host = `${accelerateEndPoint}`;
} else {
host = (0, _s3Endpoints.getS3Endpoint)(region);
}
}
if (virtualHostStyle && !opts.pathStyle) {
// For all hosts which support virtual host style, `bucketName`
// is part of the hostname in the following format:
//
// var host = 'bucketName.example.com'
//
if (bucketName) {
host = `${bucketName}.${host}`;
}
if (objectName) {
path = `/${objectName}`;
}
} else {
// For all S3 compatible storage services we will fallback to
// path style requests, where `bucketName` is part of the URI
// path.
if (bucketName) {
path = `/${bucketName}`;
}
if (objectName) {
path = `/${bucketName}/${objectName}`;
}
}
if (query) {
path += `?${query}`;
}
reqOptions.headers.host = host;
if (reqOptions.protocol === 'http:' && port !== 80 || reqOptions.protocol === 'https:' && port !== 443) {
reqOptions.headers.host = (0, _joinHostPort.joinHostPort)(host, port);
}
reqOptions.headers['user-agent'] = this.userAgent;
if (headers) {
// have all header keys in lower case - to make signing easy
for (const [k, v] of Object.entries(headers)) {
reqOptions.headers[k.toLowerCase()] = v;
}
}
// Use any request option specified in minioClient.setRequestOptions()
reqOptions = Object.assign({}, this.reqOptions, reqOptions);
return {
...reqOptions,
headers: _lodash.mapValues(_lodash.pickBy(reqOptions.headers, _helper.isDefined), v => v.toString()),
host,
port,
path
};
}
async setCredentialsProvider(credentialsProvider) {
if (!(credentialsProvider instanceof _CredentialProvider.CredentialProvider)) {
throw new Error('Unable to get credentials. Expected instance of CredentialProvider');
}
this.credentialsProvider = credentialsProvider;
await this.checkAndRefreshCreds();
}
async checkAndRefreshCreds() {
if (this.credentialsProvider) {
try {
const credentialsConf = await this.credentialsProvider.getCredentials();
this.accessKey = credentialsConf.getAccessKey();
this.secretKey = credentialsConf.getSecretKey();
this.sessionToken = credentialsConf.getSessionToken();
} catch (e) {
throw new Error(`Unable to get credentials: ${e}`, {
cause: e
});
}
}
}
/**
* log the request, response, error
*/
logHTTP(reqOptions, response, err) {
// if no logStream available return.
if (!this.logStream) {
return;
}
if (!(0, _helper.isObject)(reqOptions)) {
throw new TypeError('reqOptions should be of type "object"');
}
if (response && !(0, _helper.isReadableStream)(response)) {
throw new TypeError('response should be of type "Stream"');
}
if (err && !(err instanceof Error)) {
throw new TypeError('err should be of type "Error"');
}
const logStream = this.logStream;
const logHeaders = headers => {
Object.entries(headers).forEach(([k, v]) => {
if (k == 'authorization') {
if ((0, _helper.isString)(v)) {
const redactor = new RegExp('Signature=([0-9a-f]+)');
v = v.replace(redactor, 'Signature=**REDACTED**');
}
}
logStream.write(`${k}: ${v}\n`);
});
logStream.write('\n');
};
logStream.write(`REQUEST: ${reqOptions.method} ${reqOptions.path}\n`);
logHeaders(reqOptions.headers);
if (response) {
this.logStream.write(`RESPONSE: ${response.statusCode}\n`);
logHeaders(response.headers);
}
if (err) {
logStream.write('ERROR BODY:\n');
const errJSON = JSON.stringify(err, null, '\t');
logStream.write(`${errJSON}\n`);
}
}
/**
* Enable tracing
*/
traceOn(stream) {
if (!stream) {
stream = process.stdout;
}
this.logStream = stream;
}
/**
* Disable tracing
*/
traceOff() {
this.logStream = undefined;
}
/**
* makeRequest is the primitive used by the apis for making S3 requests.
* payload can be empty string in case of no payload.
* statusCode is the expected statusCode. If response.statusCode does not match
* we parse the XML error and call the callback with the error message.
*
* A valid region is passed by the calls - listBuckets, makeBucket and getBucketRegion.
*
* @internal
*/
async makeRequestAsync(options, payload = '', expectedCodes = [200], region = '') {
if (!(0, _helper.isObject)(options)) {
throw new TypeError('options should be of type "object"');
}
if (!(0, _helper.isString)(payload) && !(0, _helper.isObject)(payload)) {
// Buffer is of type 'object'
throw new TypeError('payload should be of type "string" or "Buffer"');
}
expectedCodes.forEach(statusCode => {
if (!(0, _helper.isNumber)(statusCode)) {
throw new TypeError('statusCode should be of type "number"');
}
});
if (!(0, _helper.isString)(region)) {
throw new TypeError('region should be of type "string"');
}
if (!options.headers) {
options.headers = {};
}
if (options.method === 'POST' || options.method === 'PUT' || options.method === 'DELETE') {
options.headers['content-length'] = payload.length.toString();
}
const sha256sum = this.enableSHA256 ? (0, _helper.toSha256)(payload) : '';
return this.makeRequestStreamAsync(options, payload, sha256sum, expectedCodes, region);
}
/**
* new request with promise
*
* No need to drain response, response body is not valid
*/
async makeRequestAsyncOmit(options, payload = '', statusCodes = [200], region = '') {
const res = await this.makeRequestAsync(options, payload, statusCodes, region);
await (0, _response.drainResponse)(res);
return res;
}
/**
* makeRequestStream will be used directly instead of makeRequest in case the payload
* is available as a stream. for ex. putObject
*
* @internal
*/
async makeRequestStreamAsync(options, body, sha256sum, statusCodes, region) {
if (!(0, _helper.isObject)(options)) {
throw new TypeError('options should be of type "object"');
}
if (!(Buffer.isBuffer(body) || typeof body === 'string' || (0, _helper.isReadableStream)(body))) {
throw new errors.InvalidArgumentError(`stream should be a Buffer, string or readable Stream, got ${typeof body} instead`);
}
if (!(0, _helper.isString)(sha256sum)) {
throw new TypeError('sha256sum should be of type "string"');
}
statusCodes.forEach(statusCode => {
if (!(0, _helper.isNumber)(statusCode)) {
throw new TypeError('statusCode should be of type "number"');
}
});
if (!(0, _helper.isString)(region)) {
throw new TypeError('region should be of type "string"');
}
// sha256sum will be empty for anonymous or https requests
if (!this.enableSHA256 && sha256sum.length !== 0) {
throw new errors.InvalidArgumentError(`sha256sum expected to be empty for anonymous or https requests`);
}
// sha256sum should be valid for non-anonymous http requests.
if (this.enableSHA256 && sha256sum.length !== 64) {
throw new errors.InvalidArgumentError(`Invalid sha256sum : ${sha256sum}`);
}
await this.checkAndRefreshCreds();
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
region = region || (await this.getBucketRegionAsync(options.bucketName));
const reqOptions = this.getRequestOptions({
...options,
region
});
if (!this.anonymous) {
// For non-anonymous https requests sha256sum is 'UNSIGNED-PAYLOAD' for signature calculation.
if (!this.enableSHA256) {
sha256sum = 'UNSIGNED-PAYLOAD';
}
const date = new Date();
reqOptions.headers['x-amz-date'] = (0, _helper.makeDateLong)(date);
reqOptions.headers['x-amz-content-sha256'] = sha256sum;
if (this.sessionToken) {
reqOptions.headers['x-amz-security-token'] = this.sessionToken;
}
reqOptions.headers.authorization = (0, _signing.signV4)(reqOptions, this.accessKey, this.secretKey, region, date, sha256sum);
}
const response = await (0, _request.requestWithRetry)(this.transport, reqOptions, body);
if (!response.statusCode) {
throw new Error("BUG: response doesn't have a statusCode");
}
if (!statusCodes.includes(response.statusCode)) {
// For an incorrect region, S3 server always sends back 400.
// But we will do cache invalidation for all errors so that,
// in future, if AWS S3 decides to send a different status code or
// XML error code we will still work fine.
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
delete this.regionMap[options.bucketName];
const err = await xmlParsers.parseResponseError(response);
this.logHTTP(reqOptions, response, err);
throw err;
}
this.logHTTP(reqOptions, response);
return response;
}
/**
* gets the region of the bucket
*
* @param bucketName
*
* @internal
*/
async getBucketRegionAsync(bucketName) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError(`Invalid bucket name : ${bucketName}`);
}
// Region is set with constructor, return the region right here.
if (this.region) {
return this.region;
}
const cached = this.regionMap[bucketName];
if (cached) {
return cached;
}
const extractRegionAsync = async response => {
const body = await (0, _response.readAsString)(response);
const region = xmlParsers.parseBucketRegion(body) || _helpers.DEFAULT_REGION;
this.regionMap[bucketName] = region;
return region;
};
const method = 'GET';
const query = 'location';
// `getBucketLocation` behaves differently in following ways for
// different environments.
//
// - For nodejs env we default to path style requests.
// - For browser env path style requests on buckets yields CORS
// error. To circumvent this problem we make a virtual host
// style request signed with 'us-east-1'. This request fails
// with an error 'AuthorizationHeaderMalformed', additionally
// the error XML also provides Region of the bucket. To validate
// this region is proper we retry the same request with the newly
// obtained region.
const pathStyle = this.pathStyle && !_browserOrNode.isBrowser;
let region;
try {
const res = await this.makeRequestAsync({
method,
bucketName,
query,
pathStyle
}, '', [200], _helpers.DEFAULT_REGION);
return extractRegionAsync(res);
} catch (e) {
// make alignment with mc cli
if (e instanceof errors.S3Error) {
const errCode = e.code;
const errRegion = e.region;
if (errCode === 'AccessDenied' && !errRegion) {
return _helpers.DEFAULT_REGION;
}
}
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
if (!(e.name === 'AuthorizationHeaderMalformed')) {
throw e;
}
// @ts-expect-error we set extra properties on error object
region = e.Region;
if (!region) {
throw e;
}
}
const res = await this.makeRequestAsync({
method,
bucketName,
query,
pathStyle
}, '', [200], region);
return await extractRegionAsync(res);
}
/**
* makeRequest is the primitive used by the apis for making S3 requests.
* payload can be empty string in case of no payload.
* statusCode is the expected statusCode. If response.statusCode does not match
* we parse the XML error and call the callback with the error message.
* A valid region is passed by the calls - listBuckets, makeBucket and
* getBucketRegion.
*
* @deprecated use `makeRequestAsync` instead
*/
makeRequest(options, payload = '', expectedCodes = [200], region = '', returnResponse, cb) {
let prom;
if (returnResponse) {
prom = this.makeRequestAsync(options, payload, expectedCodes, region);
} else {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error compatible for old behaviour
prom = this.makeRequestAsyncOmit(options, payload, expectedCodes, region);
}
prom.then(result => cb(null, result), err => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
cb(err);
});
}
/**
* makeRequestStream will be used directly instead of makeRequest in case the payload
* is available as a stream. for ex. putObject
*
* @deprecated use `makeRequestStreamAsync` instead
*/
makeRequestStream(options, stream, sha256sum, statusCodes, region, returnResponse, cb) {
const executor = async () => {
const res = await this.makeRequestStreamAsync(options, stream, sha256sum, statusCodes, region);
if (!returnResponse) {
await (0, _response.drainResponse)(res);
}
return res;
};
executor().then(result => cb(null, result),
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
err => cb(err));
}
/**
* @deprecated use `getBucketRegionAsync` instead
*/
getBucketRegion(bucketName, cb) {
return this.getBucketRegionAsync(bucketName).then(result => cb(null, result),
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
err => cb(err));
}
// Bucket operations
/**
* Creates the bucket `bucketName`.
*
*/
async makeBucket(bucketName, region = '', makeOpts) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
// Backward Compatibility
if ((0, _helper.isObject)(region)) {
makeOpts = region;
region = '';
}
if (!(0, _helper.isString)(region)) {
throw new TypeError('region should be of type "string"');
}
if (makeOpts && !(0, _helper.isObject)(makeOpts)) {
throw new TypeError('makeOpts should be of type "object"');
}
let payload = '';
// Region already set in constructor, validate if
// caller requested bucket location is same.
if (region && this.region) {
if (region !== this.region) {
throw new errors.InvalidArgumentError(`Configured region ${this.region}, requested ${region}`);
}
}
// sending makeBucket request with XML containing 'us-east-1' fails. For
// default region server expects the request without body
if (region && region !== _helpers.DEFAULT_REGION) {
payload = xml.buildObject({
CreateBucketConfiguration: {
$: {
xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
},
LocationConstraint: region
}
});
}
const method = 'PUT';
const headers = {};
if (makeOpts && makeOpts.ObjectLocking) {
headers['x-amz-bucket-object-lock-enabled'] = true;
}
// For custom region clients default to custom region specified in client constructor
const finalRegion = this.region || region || _helpers.DEFAULT_REGION;
const requestOpt = {
method,
bucketName,
headers
};
try {
await this.makeRequestAsyncOmit(requestOpt, payload, [200], finalRegion);
} catch (err) {
if (region === '' || region === _helpers.DEFAULT_REGION) {
if (err instanceof errors.S3Error) {
const errCode = err.code;
const errRegion = err.region;
if (errCode === 'AuthorizationHeaderMalformed' && errRegion !== '') {
// Retry with region returned as part of error
await this.makeRequestAsyncOmit(requestOpt, payload, [200], errCode);
}
}
}
throw err;
}
}
/**
* To check if a bucket already exists.
*/
async bucketExists(bucketName) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
const method = 'HEAD';
try {
await this.makeRequestAsyncOmit({
method,
bucketName
});
} catch (err) {
// @ts-ignore
if (err.code === 'NoSuchBucket' || err.code === 'NotFound') {
return false;
}
throw err;
}
return true;
}
/**
* @deprecated use promise style API
*/
async removeBucket(bucketName) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
const method = 'DELETE';
await this.makeRequestAsyncOmit({
method,
bucketName
}, '', [204]);
delete this.regionMap[bucketName];
}
/**
* Callback is called with readable stream of the object content.
*/
async getObject(bucketName, objectName, getOpts) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
return this.getPartialObject(bucketName, objectName, 0, 0, getOpts);
}
/**
* Callback is called with readable stream of the partial object content.
* @param bucketName
* @param objectName
* @param offset
* @param length - length of the object that will be read in the stream (optional, if not specified we read the rest of the file from the offset)
* @param getOpts
*/
async getPartialObject(bucketName, objectName, offset, length = 0, getOpts) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!(0, _helper.isNumber)(offset)) {
throw new TypeError('offset should be of type "number"');
}
if (!(0, _helper.isNumber)(length)) {
throw new TypeError('length should be of type "number"');
}
let range = '';
if (offset || length) {
if (offset) {
range = `bytes=${+offset}-`;
} else {
range = 'bytes=0-';
offset = 0;
}
if (length) {
range += `${+length + offset - 1}`;
}
}
let query = '';
let headers = {
...(range !== '' && {
range
})
};
if (getOpts) {
const sseHeaders = {
...(getOpts.SSECustomerAlgorithm && {
'X-Amz-Server-Side-Encryption-Customer-Algorithm': getOpts.SSECustomerAlgorithm
}),
...(getOpts.SSECustomerKey && {
'X-Amz-Server-Side-Encryption-Customer-Key': getOpts.SSECustomerKey
}),
...(getOpts.SSECustomerKeyMD5 && {
'X-Amz-Server-Side-Encryption-Customer-Key-MD5': getOpts.SSECustomerKeyMD5
})
};
query = qs.stringify(getOpts);
headers = {
...(0, _helper.prependXAMZMeta)(sseHeaders),
...headers
};
}
const expectedStatusCodes = [200];
if (range) {
expectedStatusCodes.push(206);
}
const method = 'GET';
return await this.makeRequestAsync({
method,
bucketName,
objectName,
headers,
query
}, '', expectedStatusCodes);
}
/**
* download object content to a file.
* This method will create a temp file named `${filename}.${base64(etag)}.part.minio` when downloading.
*
* @param bucketName - name of the bucket
* @param objectName - name of the object
* @param filePath - path to which the object data will be written to
* @param getOpts - Optional object get option
*/
async fGetObject(bucketName, objectName, filePath, getOpts) {
// Input validation.
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!(0, _helper.isString)(filePath)) {
throw new TypeError('filePath should be of type "string"');
}
const downloadToTmpFile = async () => {
let partFileStream;
const objStat = await this.statObject(bucketName, objectName, getOpts);
const encodedEtag = Buffer.from(objStat.etag).toString('base64');
const partFile = `${filePath}.${encodedEtag}.part.minio`;
await _async2.fsp.mkdir(path.dirname(filePath), {
recursive: true
});
let offset = 0;
try {
const stats = await _async2.fsp.stat(partFile);
if (objStat.size === stats.size) {
return partFile;
}
offset = stats.size;
partFileStream = fs.createWriteStream(partFile, {
flags: 'a'
});
} catch (e) {
if (e instanceof Error && e.code === 'ENOENT') {
// file not exist
partFileStream = fs.createWriteStream(partFile, {
flags: 'w'
});
} else {
// other error, maybe access deny
throw e;
}
}
const downloadStream = await this.getPartialObject(bucketName, objectName, offset, 0, getOpts);
await _async2.streamPromise.pipeline(downloadStream, partFileStream);
const stats = await _async2.fsp.stat(partFile);
if (stats.size === objStat.size) {
return partFile;
}
throw new Error('Size mismatch between downloaded file and the object');
};
const partFile = await downloadToTmpFile();
await _async2.fsp.rename(partFile, filePath);
}
/**
* Stat information of the object.
*/
async statObject(bucketName, objectName, statOpts) {
const statOptDef = statOpts || {};
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!(0, _helper.isObject)(statOptDef)) {
throw new errors.InvalidArgumentError('statOpts should be of type "object"');
}
const query = qs.stringify(statOptDef);
const method = 'HEAD';
const res = await this.makeRequestAsyncOmit({
method,
bucketName,
objectName,
query
});
return {
size: parseInt(res.headers['content-length']),
metaData: (0, _helper.extractMetadata)(res.headers),
lastModified: new Date(res.headers['last-modified']),
versionId: (0, _helper.getVersionId)(res.headers),
etag: (0, _helper.sanitizeETag)(res.headers.etag)
};
}
async removeObject(bucketName, objectName, removeOpts) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError(`Invalid bucket name: ${bucketName}`);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (removeOpts && !(0, _helper.isObject)(removeOpts)) {
throw new errors.InvalidArgumentError('removeOpts should be of type "object"');
}
const method = 'DELETE';
const headers = {};
if (removeOpts !== null && removeOpts !== void 0 && removeOpts.governanceBypass) {
headers['X-Amz-Bypass-Governance-Retention'] = true;
}
if (removeOpts !== null && removeOpts !== void 0 && removeOpts.forceDelete) {
headers['x-minio-force-delete'] = true;
}
const queryParams = {};
if (removeOpts !== null && removeOpts !== void 0 && removeOpts.versionId) {
queryParams.versionId = `${removeOpts.versionId}`;
}
const query = qs.stringify(queryParams);
await this.makeRequestAsyncOmit({
method,
bucketName,
objectName,
headers,
query
}, '', [200, 204]);
}
// Calls implemented below are related to multipart.
listIncompleteUploads(bucket, prefix, recursive) {
if (prefix === undefined) {
prefix = '';
}
if (recursive === undefined) {
recursive = false;
}
if (!(0, _helper.isValidBucketName)(bucket)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucket);
}
if (!(0, _helper.isValidPrefix)(prefix)) {
throw new errors.InvalidPrefixError(`Invalid prefix : ${prefix}`);
}
if (!(0, _helper.isBoolean)(recursive)) {
throw new TypeError('recursive should be of type "boolean"');
}
const delimiter = recursive ? '' : '/';
let keyMarker = '';
let uploadIdMarker = '';
const uploads = [];
let ended = false;
// TODO: refactor this with async/await and `stream.Readable.from`
const readStream = new stream.Readable({
objectMode: true
});
readStream._read = () => {
// push one upload info per _read()
if (uploads.length) {
return readStream.push(uploads.shift());
}
if (ended) {
return readStream.push(null);
}
this.listIncompleteUploadsQuery(bucket, prefix, keyMarker, uploadIdMarker, delimiter).then(result => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
result.prefixes.forEach(prefix => uploads.push(prefix));
async.eachSeries(result.uploads, (upload, cb) => {
// for each incomplete upload add the sizes of its uploaded parts
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
this.listParts(bucket, upload.key, upload.uploadId).then(parts => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
upload.size = parts.reduce((acc, item) => acc + item.size, 0);
uploads.push(upload);
cb();
}, err => cb(err));
}, err => {
if (err) {
readStream.emit('error', err);
return;
}
if (result.isTruncated) {
keyMarker = result.nextKeyMarker;
uploadIdMarker = result.nextUploadIdMarker;
} else {
ended = true;
}
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
readStream._read();
});
}, e => {
readStream.emit('error', e);
});
};
return readStream;
}
/**
* Called by listIncompleteUploads to fetch a batch of incomplete uploads.
*/
async listIncompleteUploadsQuery(bucketName, prefix, keyMarker, uploadIdMarker, delimiter) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isString)(prefix)) {
throw new TypeError('prefix should be of type "string"');
}
if (!(0, _helper.isString)(keyMarker)) {
throw new TypeError('keyMarker should be of type "string"');
}
if (!(0, _helper.isString)(uploadIdMarker)) {
throw new TypeError('uploadIdMarker should be of type "string"');
}
if (!(0, _helper.isString)(delimiter)) {
throw new TypeError('delimiter should be of type "string"');
}
const queries = [];
queries.push(`prefix=${(0, _helper.uriEscape)(prefix)}`);
queries.push(`delimiter=${(0, _helper.uriEscape)(delimiter)}`);
if (keyMarker) {
queries.push(`key-marker=${(0, _helper.uriEscape)(keyMarker)}`);
}
if (uploadIdMarker) {
queries.push(`upload-id-marker=${uploadIdMarker}`);
}
const maxUploads = 1000;
queries.push(`max-uploads=${maxUploads}`);
queries.sort();
queries.unshift('uploads');
let query = '';
if (queries.length > 0) {
query = `${queries.join('&')}`;
}
const method = 'GET';
const res = await this.makeRequestAsync({
method,
bucketName,
query
});
const body = await (0, _response.readAsString)(res);
return xmlParsers.parseListMultipart(body);
}
/**
* Initiate a new multipart upload.
* @internal
*/
async initiateNewMultipartUpload(bucketName, objectName, headers) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!(0, _helper.isObject)(headers)) {
throw new errors.InvalidObjectNameError('contentType should be of type "object"');
}
const method = 'POST';
const query = 'uploads';
const res = await this.makeRequestAsync({
method,
bucketName,
objectName,
query,
headers
});
const body = await (0, _response.readAsBuffer)(res);
return (0, xmlParsers.parseInitiateMultipart)(body.toString());
}
/**
* Internal Method to abort a multipart upload request in case of any errors.
*
* @param bucketName - Bucket Name
* @param objectName - Object Name
* @param uploadId - id of a multipart upload to cancel during compose object sequence.
*/
async abortMultipartUpload(bucketName, objectName, uploadId) {
const method = 'DELETE';
const query = `uploadId=${uploadId}`;
const requestOptions = {
method,
bucketName,
objectName: objectName,
query
};
await this.makeRequestAsyncOmit(requestOptions, '', [204]);
}
async findUploadId(bucketName, objectName) {
var _latestUpload;
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
let latestUpload;
let keyMarker = '';
let uploadIdMarker = '';
for (;;) {
const result = await this.listIncompleteUploadsQuery(bucketName, objectName, keyMarker, uploadIdMarker, '');
for (const upload of result.uploads) {
if (upload.key === objectName) {
if (!latestUpload || upload.initiated.getTime() > latestUpload.initiated.getTime()) {
latestUpload = upload;
}
}
}
if (result.isTruncated) {
keyMarker = result.nextKeyMarker;
uploadIdMarker = result.nextUploadIdMarker;
continue;
}
break;
}
return (_latestUpload = latestUpload) === null || _latestUpload === void 0 ? void 0 : _latestUpload.uploadId;
}
/**
* this call will aggregate the parts on the server into a single object.
*/
async completeMultipartUpload(bucketName, objectName, uploadId, etags) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!(0, _helper.isString)(uploadId)) {
throw new TypeError('uploadId should be of type "string"');
}
if (!(0, _helper.isObject)(etags)) {
throw new TypeError('etags should be of type "Array"');
}
if (!uploadId) {
throw new errors.InvalidArgumentError('uploadId cannot be empty');
}
const method = 'POST';
const query = `uploadId=${(0, _helper.uriEscape)(uploadId)}`;
const builder = new _xml2js.Builder();
const payload = builder.buildObject({
CompleteMultipartUpload: {
$: {
xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
},
Part: etags.map(etag => {
return {
PartNumber: etag.part,
ETag: etag.etag
};
})
}
});
const res = await this.makeRequestAsync({
method,
bucketName,
objectName,
query
}, payload);
const body = await (0, _response.readAsBuffer)(res);
const result = (0, xmlParsers.parseCompleteMultipart)(body.toString());
if (!result) {
throw new Error('BUG: failed to parse server response');
}
if (result.errCode) {
// Multipart Complete API returns an error XML after a 200 http status
throw new errors.S3Error(result.errMessage);
}
return {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
etag: result.etag,
versionId: (0, _helper.getVersionId)(res.headers)
};
}
/**
* Get part-info of all parts of an incomplete upload specified by uploadId.
*/
async listParts(bucketName, objectName, uploadId) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!(0, _helper.isString)(uploadId)) {
throw new TypeError('uploadId should be of type "string"');
}
if (!uploadId) {
throw new errors.InvalidArgumentError('uploadId cannot be empty');
}
const parts = [];
let marker = 0;
let result;
do {
result = await this.listPartsQuery(bucketName, objectName, uploadId, marker);
marker = result.marker;
parts.push(...result.parts);
} while (result.isTruncated);
return parts;
}
/**
* Called by listParts to fetch a batch of part-info
*/
async listPartsQuery(bucketName, objectName, uploadId, marker) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!(0, _helper.isString)(uploadId)) {
throw new TypeError('uploadId should be of type "string"');
}
if (!(0, _helper.isNumber)(marker)) {
throw new TypeError('marker should be of type "number"');
}
if (!uploadId) {
throw new errors.InvalidArgumentError('uploadId cannot be empty');
}
let query = `uploadId=${(0, _helper.uriEscape)(uploadId)}`;
if (marker) {
query += `&part-number-marker=${marker}`;
}
const method = 'GET';
const res = await this.makeRequestAsync({
method,
bucketName,
objectName,
query
});
return xmlParsers.parseListParts(await (0, _response.readAsString)(res));
}
async listBuckets() {
const method = 'GET';
const regionConf = this.region || _helpers.DEFAULT_REGION;
const httpRes = await this.makeRequestAsync({
method
}, '', [200], regionConf);
const xmlResult = await (0, _response.readAsString)(httpRes);
return xmlParsers.parseListBucket(xmlResult);
}
/**
* Calculate part size given the object size. Part size will be atleast this.partSize
*/
calculatePartSize(size) {
if (!(0, _helper.isNumber)(size)) {
throw new TypeError('size should be of type "number"');
}
if (size > this.maxObjectSize) {
throw new TypeError(`size should not be more than ${this.maxObjectSize}`);
}
if (this.overRidePartSize) {
return this.partSize;
}
let partSize = this.partSize;
for (;;) {
// while(true) {...} throws linting error.
// If partSize is big enough to accomodate the object size, then use it.
if (partSize * 10000 > size) {
return partSize;
}
// Try part sizes as 64MB, 80MB, 96MB etc.
partSize += 16 * 1024 * 1024;
}
}
/**
* Uploads the object using contents from a file
*/
async fPutObject(bucketName, objectName, filePath, metaData) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!(0, _helper.isString)(filePath)) {
throw new TypeError('filePath should be of type "string"');
}
if (metaData && !(0, _helper.isObject)(metaData)) {
throw new TypeError('metaData should be of type "object"');
}
// Inserts correct `content-type` attribute based on metaData and filePath
metaData = (0, _helper.insertContentType)(metaData || {}, filePath);
const stat = await _async2.fsp.lstat(filePath);
return await this.putObject(bucketName, objectName, fs.createReadStream(filePath), stat.size, metaData);
}
/**
* Uploading a stream, "Buffer" or "string".
* It's recommended to pass `size` argument with stream.
*/
async putObject(bucketName, objectName, stream, size, metaData) {
if (!(0, _helper.isValidBucketName)(bucketName)) {
throw new errors.InvalidBucketNameError(`Invalid bucket name: ${bucketName}`);
}
if (!(0, _helper.isValidObjectName)(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
// We'll need to shift arguments to the left because of metaData
// and size being optional.
if ((0, _helper.isObject)(size)) {
metaData = size;
}
// Ensures Metadata has appropriate prefix for A3 API
const headers = (0, _helper.prependXAMZMeta)(metaData);
if (typeof stream === 'string' || stream instanceof Buffer) {
// Adapts the non-stream interface into a stream.
size = stream.length;
stream = (0, _helper.readableStream)(stream);
} else if (!(0, _helper.isReadableStream)(stream)) {
throw new TypeError('third argument should be of type "stream.Readable" or "Buffer" or "string"');
}
if ((0, _helper.isNumber)(size) && size < 0) {
throw new errors.InvalidArgumentError(`size cannot be negative, given size: ${size}`);
}
// Get the part size and forward that to the BlockStream. Default to the
// largest block size possible if necessary.
if (!(0, _helper.isNumber)(size)) {
size = this.maxObjectSize;
}
// Get the part size and forward that to the BlockStream. Default to the
// largest block size possible if necessary.
if (size === undefined) {
const statSize = await (0, _helper.getContentLength)(stream);
if (statSize !== null) {
size = statSize;
}
}
if (!(0, _helper.isNumber)(size)) {
// Backward compatibility
size = this.maxObjectSize;
}
const partSize = this.calculatePartSize(size);
if (typeof stream === 'string' || stream.readableLength === 0 || Buffer.isBuffer(stream) || size <= partSize) {
const buf = (0, _helper.isReadableStream)(stream) ? await (0, _response.readAsBuffer)(stream) : Buffer.from(stream);
return this.uploadBuffer(bucketName, objectName, headers, buf);
}
retur