minio
Version:
S3 Compatible Cloud Storage client
1,465 lines (1,405 loc) • 363 kB
JavaScript
import * as crypto from "crypto";
import * as fs from "fs";
import * as http from "http";
import * as https from "https";
import * as path from "path";
import * as stream from "stream";
import * as async from 'async';
import BlockStream2 from 'block-stream2';
import { isBrowser } from 'browser-or-node';
import _ from 'lodash';
import * as qs from 'query-string';
import xml2js from 'xml2js';
import { CredentialProvider } from "../CredentialProvider.mjs";
import * as errors from "../errors.mjs";
import { CopyDestinationOptions, CopySourceOptions, DEFAULT_REGION, LEGAL_HOLD_STATUS, PRESIGN_EXPIRY_DAYS_MAX, RETENTION_MODES, RETENTION_VALIDITY_UNITS } from "../helpers.mjs";
import { postPresignSignatureV4, presignSignatureV4, signV4 } from "../signing.mjs";
import { fsp, streamPromise } from "./async.mjs";
import { CopyConditions } from "./copy-conditions.mjs";
import { Extensions } from "./extensions.mjs";
import { calculateEvenSplits, extractMetadata, getContentLength, getScope, getSourceVersionId, getVersionId, hashBinary, insertContentType, isAmazonEndpoint, isBoolean, isDefined, isEmpty, isNumber, isObject, isReadableStream, isString, isValidBucketName, isValidEndpoint, isValidObjectName, isValidPort, isValidPrefix, isVirtualHostStyle, makeDateLong, PART_CONSTRAINTS, partsRequired, prependXAMZMeta, readableStream, sanitizeETag, toMd5, toSha256, uriEscape, uriResourceEscape } from "./helper.mjs";
import { joinHostPort } from "./join-host-port.mjs";
import { PostPolicy } from "./post-policy.mjs";
import { requestWithRetry } from "./request.mjs";
import { drainResponse, readAsBuffer, readAsString } from "./response.mjs";
import { getS3Endpoint } from "./s3-endpoints.mjs";
import { parseCompleteMultipart, parseInitiateMultipart, parseListObjects, parseObjectLegalHoldConfig, parseSelectObjectContentResponse, uploadPartParser } from "./xml-parser.mjs";
import * as xmlParsers from "./xml-parser.mjs";
const xml = new xml2js.Builder({
renderOpts: {
pretty: false
},
headless: true
});
// will be replaced by bundler.
const Package = {
version: "8.0.5" || 'development'
};
const requestOptionProperties = ['agent', 'ca', 'cert', 'ciphers', 'clientCertEngine', 'crl', 'dhparam', 'ecdhCurve', 'family', 'honorCipherOrder', 'key', 'passphrase', 'pfx', 'rejectUnauthorized', 'secureOptions', 'secureProtocol', 'servername', 'sessionIdContext'];
export class TypedClient {
partSize = 64 * 1024 * 1024;
maximumPartSize = 5 * 1024 * 1024 * 1024;
maxObjectSize = 5 * 1024 * 1024 * 1024 * 1024;
constructor(params) {
// @ts-expect-error deprecated property
if (params.secure !== undefined) {
throw new Error('"secure" option deprecated, "useSSL" should be used instead');
}
// Default values if not specified.
if (params.useSSL === undefined) {
params.useSSL = true;
}
if (!params.port) {
params.port = 0;
}
// Validate input params.
if (!isValidEndpoint(params.endPoint)) {
throw new errors.InvalidEndpointError(`Invalid endPoint : ${params.endPoint}`);
}
if (!isValidPort(params.port)) {
throw new errors.InvalidArgumentError(`Invalid port : ${params.port}`);
}
if (!isBoolean(params.useSSL)) {
throw new errors.InvalidArgumentError(`Invalid useSSL flag type : ${params.useSSL}, expected to be of type "boolean"`);
}
// Validate region only if its set.
if (params.region) {
if (!isString(params.region)) {
throw new errors.InvalidArgumentError(`Invalid region : ${params.region}`);
}
}
const host = params.endPoint.toLowerCase();
let port = params.port;
let protocol;
let transport;
let transportAgent;
// Validate if configuration is not using SSL
// for constructing relevant endpoints.
if (params.useSSL) {
// Defaults to secure.
transport = https;
protocol = 'https:';
port = port || 443;
transportAgent = https.globalAgent;
} else {
transport = http;
protocol = 'http:';
port = port || 80;
transportAgent = http.globalAgent;
}
// if custom transport is set, use it.
if (params.transport) {
if (!isObject(params.transport)) {
throw new errors.InvalidArgumentError(`Invalid transport type : ${params.transport}, expected to be type "object"`);
}
transport = params.transport;
}
// if custom transport agent is set, use it.
if (params.transportAgent) {
if (!isObject(params.transportAgent)) {
throw new errors.InvalidArgumentError(`Invalid transportAgent type: ${params.transportAgent}, expected to be type "object"`);
}
transportAgent = params.transportAgent;
}
// User Agent should always following the below style.
// Please open an issue to discuss any new changes here.
//
// MinIO (OS; ARCH) LIB/VER APP/VER
//
const libraryComments = `(${process.platform}; ${process.arch})`;
const libraryAgent = `MinIO ${libraryComments} minio-js/${Package.version}`;
// User agent block ends.
this.transport = transport;
this.transportAgent = transportAgent;
this.host = host;
this.port = port;
this.protocol = protocol;
this.userAgent = `${libraryAgent}`;
// Default path style is true
if (params.pathStyle === undefined) {
this.pathStyle = true;
} else {
this.pathStyle = params.pathStyle;
}
this.accessKey = params.accessKey ?? '';
this.secretKey = params.secretKey ?? '';
this.sessionToken = params.sessionToken;
this.anonymous = !this.accessKey || !this.secretKey;
if (params.credentialsProvider) {
this.anonymous = false;
this.credentialsProvider = params.credentialsProvider;
}
this.regionMap = {};
if (params.region) {
this.region = params.region;
}
if (params.partSize) {
this.partSize = params.partSize;
this.overRidePartSize = true;
}
if (this.partSize < 5 * 1024 * 1024) {
throw new errors.InvalidArgumentError(`Part size should be greater than 5MB`);
}
if (this.partSize > 5 * 1024 * 1024 * 1024) {
throw new errors.InvalidArgumentError(`Part size should be less than 5GB`);
}
// SHA256 is enabled only for authenticated http requests. If the request is authenticated
// and the connection is https we use x-amz-content-sha256=UNSIGNED-PAYLOAD
// header for signature calculation.
this.enableSHA256 = !this.anonymous && !params.useSSL;
this.s3AccelerateEndpoint = params.s3AccelerateEndpoint || undefined;
this.reqOptions = {};
this.clientExtensions = new Extensions(this);
}
/**
* Minio extensions that aren't necessary present for Amazon S3 compatible storage servers
*/
get extensions() {
return this.clientExtensions;
}
/**
* @param endPoint - valid S3 acceleration end point
*/
setS3TransferAccelerate(endPoint) {
this.s3AccelerateEndpoint = endPoint;
}
/**
* Sets the supported request options.
*/
setRequestOptions(options) {
if (!isObject(options)) {
throw new TypeError('request options should be of type "object"');
}
this.reqOptions = _.pick(options, requestOptionProperties);
}
/**
* This is s3 Specific and does not hold validity in any other Object storage.
*/
getAccelerateEndPointIfSet(bucketName, objectName) {
if (!isEmpty(this.s3AccelerateEndpoint) && !isEmpty(bucketName) && !isEmpty(objectName)) {
// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html
// Disable transfer acceleration for non-compliant bucket names.
if (bucketName.includes('.')) {
throw new Error(`Transfer Acceleration is not supported for non compliant bucket:${bucketName}`);
}
// If transfer acceleration is requested set new host.
// For more details about enabling transfer acceleration read here.
// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html
return this.s3AccelerateEndpoint;
}
return false;
}
/**
* Set application specific information.
* Generates User-Agent in the following style.
* MinIO (OS; ARCH) LIB/VER APP/VER
*/
setAppInfo(appName, appVersion) {
if (!isString(appName)) {
throw new TypeError(`Invalid appName: ${appName}`);
}
if (appName.trim() === '') {
throw new errors.InvalidArgumentError('Input appName cannot be empty.');
}
if (!isString(appVersion)) {
throw new TypeError(`Invalid appVersion: ${appVersion}`);
}
if (appVersion.trim() === '') {
throw new errors.InvalidArgumentError('Input appVersion cannot be empty.');
}
this.userAgent = `${this.userAgent} ${appName}/${appVersion}`;
}
/**
* returns options object that can be used with http.request()
* Takes care of constructing virtual-host-style or path-style hostname
*/
getRequestOptions(opts) {
const method = opts.method;
const region = opts.region;
const bucketName = opts.bucketName;
let objectName = opts.objectName;
const headers = opts.headers;
const query = opts.query;
let reqOptions = {
method,
headers: {},
protocol: this.protocol,
// If custom transportAgent was supplied earlier, we'll inject it here
agent: this.transportAgent
};
// Verify if virtual host supported.
let virtualHostStyle;
if (bucketName) {
virtualHostStyle = isVirtualHostStyle(this.host, this.protocol, bucketName, this.pathStyle);
}
let path = '/';
let host = this.host;
let port;
if (this.port) {
port = this.port;
}
if (objectName) {
objectName = uriResourceEscape(objectName);
}
// For Amazon S3 endpoint, get endpoint based on region.
if (isAmazonEndpoint(host)) {
const accelerateEndPoint = this.getAccelerateEndPointIfSet(bucketName, objectName);
if (accelerateEndPoint) {
host = `${accelerateEndPoint}`;
} else {
host = getS3Endpoint(region);
}
}
if (virtualHostStyle && !opts.pathStyle) {
// For all hosts which support virtual host style, `bucketName`
// is part of the hostname in the following format:
//
// var host = 'bucketName.example.com'
//
if (bucketName) {
host = `${bucketName}.${host}`;
}
if (objectName) {
path = `/${objectName}`;
}
} else {
// For all S3 compatible storage services we will fallback to
// path style requests, where `bucketName` is part of the URI
// path.
if (bucketName) {
path = `/${bucketName}`;
}
if (objectName) {
path = `/${bucketName}/${objectName}`;
}
}
if (query) {
path += `?${query}`;
}
reqOptions.headers.host = host;
if (reqOptions.protocol === 'http:' && port !== 80 || reqOptions.protocol === 'https:' && port !== 443) {
reqOptions.headers.host = joinHostPort(host, port);
}
reqOptions.headers['user-agent'] = this.userAgent;
if (headers) {
// have all header keys in lower case - to make signing easy
for (const [k, v] of Object.entries(headers)) {
reqOptions.headers[k.toLowerCase()] = v;
}
}
// Use any request option specified in minioClient.setRequestOptions()
reqOptions = Object.assign({}, this.reqOptions, reqOptions);
return {
...reqOptions,
headers: _.mapValues(_.pickBy(reqOptions.headers, isDefined), v => v.toString()),
host,
port,
path
};
}
async setCredentialsProvider(credentialsProvider) {
if (!(credentialsProvider instanceof CredentialProvider)) {
throw new Error('Unable to get credentials. Expected instance of CredentialProvider');
}
this.credentialsProvider = credentialsProvider;
await this.checkAndRefreshCreds();
}
async checkAndRefreshCreds() {
if (this.credentialsProvider) {
try {
const credentialsConf = await this.credentialsProvider.getCredentials();
this.accessKey = credentialsConf.getAccessKey();
this.secretKey = credentialsConf.getSecretKey();
this.sessionToken = credentialsConf.getSessionToken();
} catch (e) {
throw new Error(`Unable to get credentials: ${e}`, {
cause: e
});
}
}
}
/**
* log the request, response, error
*/
logHTTP(reqOptions, response, err) {
// if no logStream available return.
if (!this.logStream) {
return;
}
if (!isObject(reqOptions)) {
throw new TypeError('reqOptions should be of type "object"');
}
if (response && !isReadableStream(response)) {
throw new TypeError('response should be of type "Stream"');
}
if (err && !(err instanceof Error)) {
throw new TypeError('err should be of type "Error"');
}
const logStream = this.logStream;
const logHeaders = headers => {
Object.entries(headers).forEach(([k, v]) => {
if (k == 'authorization') {
if (isString(v)) {
const redactor = new RegExp('Signature=([0-9a-f]+)');
v = v.replace(redactor, 'Signature=**REDACTED**');
}
}
logStream.write(`${k}: ${v}\n`);
});
logStream.write('\n');
};
logStream.write(`REQUEST: ${reqOptions.method} ${reqOptions.path}\n`);
logHeaders(reqOptions.headers);
if (response) {
this.logStream.write(`RESPONSE: ${response.statusCode}\n`);
logHeaders(response.headers);
}
if (err) {
logStream.write('ERROR BODY:\n');
const errJSON = JSON.stringify(err, null, '\t');
logStream.write(`${errJSON}\n`);
}
}
/**
* Enable tracing
*/
traceOn(stream) {
if (!stream) {
stream = process.stdout;
}
this.logStream = stream;
}
/**
* Disable tracing
*/
traceOff() {
this.logStream = undefined;
}
/**
* makeRequest is the primitive used by the apis for making S3 requests.
* payload can be empty string in case of no payload.
* statusCode is the expected statusCode. If response.statusCode does not match
* we parse the XML error and call the callback with the error message.
*
* A valid region is passed by the calls - listBuckets, makeBucket and getBucketRegion.
*
* @internal
*/
async makeRequestAsync(options, payload = '', expectedCodes = [200], region = '') {
if (!isObject(options)) {
throw new TypeError('options should be of type "object"');
}
if (!isString(payload) && !isObject(payload)) {
// Buffer is of type 'object'
throw new TypeError('payload should be of type "string" or "Buffer"');
}
expectedCodes.forEach(statusCode => {
if (!isNumber(statusCode)) {
throw new TypeError('statusCode should be of type "number"');
}
});
if (!isString(region)) {
throw new TypeError('region should be of type "string"');
}
if (!options.headers) {
options.headers = {};
}
if (options.method === 'POST' || options.method === 'PUT' || options.method === 'DELETE') {
options.headers['content-length'] = payload.length.toString();
}
const sha256sum = this.enableSHA256 ? toSha256(payload) : '';
return this.makeRequestStreamAsync(options, payload, sha256sum, expectedCodes, region);
}
/**
* new request with promise
*
* No need to drain response, response body is not valid
*/
async makeRequestAsyncOmit(options, payload = '', statusCodes = [200], region = '') {
const res = await this.makeRequestAsync(options, payload, statusCodes, region);
await drainResponse(res);
return res;
}
/**
* makeRequestStream will be used directly instead of makeRequest in case the payload
* is available as a stream. for ex. putObject
*
* @internal
*/
async makeRequestStreamAsync(options, body, sha256sum, statusCodes, region) {
if (!isObject(options)) {
throw new TypeError('options should be of type "object"');
}
if (!(Buffer.isBuffer(body) || typeof body === 'string' || isReadableStream(body))) {
throw new errors.InvalidArgumentError(`stream should be a Buffer, string or readable Stream, got ${typeof body} instead`);
}
if (!isString(sha256sum)) {
throw new TypeError('sha256sum should be of type "string"');
}
statusCodes.forEach(statusCode => {
if (!isNumber(statusCode)) {
throw new TypeError('statusCode should be of type "number"');
}
});
if (!isString(region)) {
throw new TypeError('region should be of type "string"');
}
// sha256sum will be empty for anonymous or https requests
if (!this.enableSHA256 && sha256sum.length !== 0) {
throw new errors.InvalidArgumentError(`sha256sum expected to be empty for anonymous or https requests`);
}
// sha256sum should be valid for non-anonymous http requests.
if (this.enableSHA256 && sha256sum.length !== 64) {
throw new errors.InvalidArgumentError(`Invalid sha256sum : ${sha256sum}`);
}
await this.checkAndRefreshCreds();
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
region = region || (await this.getBucketRegionAsync(options.bucketName));
const reqOptions = this.getRequestOptions({
...options,
region
});
if (!this.anonymous) {
// For non-anonymous https requests sha256sum is 'UNSIGNED-PAYLOAD' for signature calculation.
if (!this.enableSHA256) {
sha256sum = 'UNSIGNED-PAYLOAD';
}
const date = new Date();
reqOptions.headers['x-amz-date'] = makeDateLong(date);
reqOptions.headers['x-amz-content-sha256'] = sha256sum;
if (this.sessionToken) {
reqOptions.headers['x-amz-security-token'] = this.sessionToken;
}
reqOptions.headers.authorization = signV4(reqOptions, this.accessKey, this.secretKey, region, date, sha256sum);
}
const response = await requestWithRetry(this.transport, reqOptions, body);
if (!response.statusCode) {
throw new Error("BUG: response doesn't have a statusCode");
}
if (!statusCodes.includes(response.statusCode)) {
// For an incorrect region, S3 server always sends back 400.
// But we will do cache invalidation for all errors so that,
// in future, if AWS S3 decides to send a different status code or
// XML error code we will still work fine.
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
delete this.regionMap[options.bucketName];
const err = await xmlParsers.parseResponseError(response);
this.logHTTP(reqOptions, response, err);
throw err;
}
this.logHTTP(reqOptions, response);
return response;
}
/**
* gets the region of the bucket
*
* @param bucketName
*
* @internal
*/
async getBucketRegionAsync(bucketName) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError(`Invalid bucket name : ${bucketName}`);
}
// Region is set with constructor, return the region right here.
if (this.region) {
return this.region;
}
const cached = this.regionMap[bucketName];
if (cached) {
return cached;
}
const extractRegionAsync = async response => {
const body = await readAsString(response);
const region = xmlParsers.parseBucketRegion(body) || DEFAULT_REGION;
this.regionMap[bucketName] = region;
return region;
};
const method = 'GET';
const query = 'location';
// `getBucketLocation` behaves differently in following ways for
// different environments.
//
// - For nodejs env we default to path style requests.
// - For browser env path style requests on buckets yields CORS
// error. To circumvent this problem we make a virtual host
// style request signed with 'us-east-1'. This request fails
// with an error 'AuthorizationHeaderMalformed', additionally
// the error XML also provides Region of the bucket. To validate
// this region is proper we retry the same request with the newly
// obtained region.
const pathStyle = this.pathStyle && !isBrowser;
let region;
try {
const res = await this.makeRequestAsync({
method,
bucketName,
query,
pathStyle
}, '', [200], DEFAULT_REGION);
return extractRegionAsync(res);
} catch (e) {
// make alignment with mc cli
if (e instanceof errors.S3Error) {
const errCode = e.code;
const errRegion = e.region;
if (errCode === 'AccessDenied' && !errRegion) {
return DEFAULT_REGION;
}
}
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
if (!(e.name === 'AuthorizationHeaderMalformed')) {
throw e;
}
// @ts-expect-error we set extra properties on error object
region = e.Region;
if (!region) {
throw e;
}
}
const res = await this.makeRequestAsync({
method,
bucketName,
query,
pathStyle
}, '', [200], region);
return await extractRegionAsync(res);
}
/**
* makeRequest is the primitive used by the apis for making S3 requests.
* payload can be empty string in case of no payload.
* statusCode is the expected statusCode. If response.statusCode does not match
* we parse the XML error and call the callback with the error message.
* A valid region is passed by the calls - listBuckets, makeBucket and
* getBucketRegion.
*
* @deprecated use `makeRequestAsync` instead
*/
makeRequest(options, payload = '', expectedCodes = [200], region = '', returnResponse, cb) {
let prom;
if (returnResponse) {
prom = this.makeRequestAsync(options, payload, expectedCodes, region);
} else {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error compatible for old behaviour
prom = this.makeRequestAsyncOmit(options, payload, expectedCodes, region);
}
prom.then(result => cb(null, result), err => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
cb(err);
});
}
/**
* makeRequestStream will be used directly instead of makeRequest in case the payload
* is available as a stream. for ex. putObject
*
* @deprecated use `makeRequestStreamAsync` instead
*/
makeRequestStream(options, stream, sha256sum, statusCodes, region, returnResponse, cb) {
const executor = async () => {
const res = await this.makeRequestStreamAsync(options, stream, sha256sum, statusCodes, region);
if (!returnResponse) {
await drainResponse(res);
}
return res;
};
executor().then(result => cb(null, result),
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
err => cb(err));
}
/**
* @deprecated use `getBucketRegionAsync` instead
*/
getBucketRegion(bucketName, cb) {
return this.getBucketRegionAsync(bucketName).then(result => cb(null, result),
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
err => cb(err));
}
// Bucket operations
/**
* Creates the bucket `bucketName`.
*
*/
async makeBucket(bucketName, region = '', makeOpts) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
// Backward Compatibility
if (isObject(region)) {
makeOpts = region;
region = '';
}
if (!isString(region)) {
throw new TypeError('region should be of type "string"');
}
if (makeOpts && !isObject(makeOpts)) {
throw new TypeError('makeOpts should be of type "object"');
}
let payload = '';
// Region already set in constructor, validate if
// caller requested bucket location is same.
if (region && this.region) {
if (region !== this.region) {
throw new errors.InvalidArgumentError(`Configured region ${this.region}, requested ${region}`);
}
}
// sending makeBucket request with XML containing 'us-east-1' fails. For
// default region server expects the request without body
if (region && region !== DEFAULT_REGION) {
payload = xml.buildObject({
CreateBucketConfiguration: {
$: {
xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
},
LocationConstraint: region
}
});
}
const method = 'PUT';
const headers = {};
if (makeOpts && makeOpts.ObjectLocking) {
headers['x-amz-bucket-object-lock-enabled'] = true;
}
// For custom region clients default to custom region specified in client constructor
const finalRegion = this.region || region || DEFAULT_REGION;
const requestOpt = {
method,
bucketName,
headers
};
try {
await this.makeRequestAsyncOmit(requestOpt, payload, [200], finalRegion);
} catch (err) {
if (region === '' || region === DEFAULT_REGION) {
if (err instanceof errors.S3Error) {
const errCode = err.code;
const errRegion = err.region;
if (errCode === 'AuthorizationHeaderMalformed' && errRegion !== '') {
// Retry with region returned as part of error
await this.makeRequestAsyncOmit(requestOpt, payload, [200], errCode);
}
}
}
throw err;
}
}
/**
* To check if a bucket already exists.
*/
async bucketExists(bucketName) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
const method = 'HEAD';
try {
await this.makeRequestAsyncOmit({
method,
bucketName
});
} catch (err) {
// @ts-ignore
if (err.code === 'NoSuchBucket' || err.code === 'NotFound') {
return false;
}
throw err;
}
return true;
}
/**
* @deprecated use promise style API
*/
async removeBucket(bucketName) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
const method = 'DELETE';
await this.makeRequestAsyncOmit({
method,
bucketName
}, '', [204]);
delete this.regionMap[bucketName];
}
/**
* Callback is called with readable stream of the object content.
*/
async getObject(bucketName, objectName, getOpts) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
return this.getPartialObject(bucketName, objectName, 0, 0, getOpts);
}
/**
* Callback is called with readable stream of the partial object content.
* @param bucketName
* @param objectName
* @param offset
* @param length - length of the object that will be read in the stream (optional, if not specified we read the rest of the file from the offset)
* @param getOpts
*/
async getPartialObject(bucketName, objectName, offset, length = 0, getOpts) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!isNumber(offset)) {
throw new TypeError('offset should be of type "number"');
}
if (!isNumber(length)) {
throw new TypeError('length should be of type "number"');
}
let range = '';
if (offset || length) {
if (offset) {
range = `bytes=${+offset}-`;
} else {
range = 'bytes=0-';
offset = 0;
}
if (length) {
range += `${+length + offset - 1}`;
}
}
let query = '';
let headers = {
...(range !== '' && {
range
})
};
if (getOpts) {
const sseHeaders = {
...(getOpts.SSECustomerAlgorithm && {
'X-Amz-Server-Side-Encryption-Customer-Algorithm': getOpts.SSECustomerAlgorithm
}),
...(getOpts.SSECustomerKey && {
'X-Amz-Server-Side-Encryption-Customer-Key': getOpts.SSECustomerKey
}),
...(getOpts.SSECustomerKeyMD5 && {
'X-Amz-Server-Side-Encryption-Customer-Key-MD5': getOpts.SSECustomerKeyMD5
})
};
query = qs.stringify(getOpts);
headers = {
...prependXAMZMeta(sseHeaders),
...headers
};
}
const expectedStatusCodes = [200];
if (range) {
expectedStatusCodes.push(206);
}
const method = 'GET';
return await this.makeRequestAsync({
method,
bucketName,
objectName,
headers,
query
}, '', expectedStatusCodes);
}
/**
* download object content to a file.
* This method will create a temp file named `${filename}.${base64(etag)}.part.minio` when downloading.
*
* @param bucketName - name of the bucket
* @param objectName - name of the object
* @param filePath - path to which the object data will be written to
* @param getOpts - Optional object get option
*/
async fGetObject(bucketName, objectName, filePath, getOpts) {
// Input validation.
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!isString(filePath)) {
throw new TypeError('filePath should be of type "string"');
}
const downloadToTmpFile = async () => {
let partFileStream;
const objStat = await this.statObject(bucketName, objectName, getOpts);
const encodedEtag = Buffer.from(objStat.etag).toString('base64');
const partFile = `${filePath}.${encodedEtag}.part.minio`;
await fsp.mkdir(path.dirname(filePath), {
recursive: true
});
let offset = 0;
try {
const stats = await fsp.stat(partFile);
if (objStat.size === stats.size) {
return partFile;
}
offset = stats.size;
partFileStream = fs.createWriteStream(partFile, {
flags: 'a'
});
} catch (e) {
if (e instanceof Error && e.code === 'ENOENT') {
// file not exist
partFileStream = fs.createWriteStream(partFile, {
flags: 'w'
});
} else {
// other error, maybe access deny
throw e;
}
}
const downloadStream = await this.getPartialObject(bucketName, objectName, offset, 0, getOpts);
await streamPromise.pipeline(downloadStream, partFileStream);
const stats = await fsp.stat(partFile);
if (stats.size === objStat.size) {
return partFile;
}
throw new Error('Size mismatch between downloaded file and the object');
};
const partFile = await downloadToTmpFile();
await fsp.rename(partFile, filePath);
}
/**
* Stat information of the object.
*/
async statObject(bucketName, objectName, statOpts) {
const statOptDef = statOpts || {};
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!isObject(statOptDef)) {
throw new errors.InvalidArgumentError('statOpts should be of type "object"');
}
const query = qs.stringify(statOptDef);
const method = 'HEAD';
const res = await this.makeRequestAsyncOmit({
method,
bucketName,
objectName,
query
});
return {
size: parseInt(res.headers['content-length']),
metaData: extractMetadata(res.headers),
lastModified: new Date(res.headers['last-modified']),
versionId: getVersionId(res.headers),
etag: sanitizeETag(res.headers.etag)
};
}
async removeObject(bucketName, objectName, removeOpts) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError(`Invalid bucket name: ${bucketName}`);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (removeOpts && !isObject(removeOpts)) {
throw new errors.InvalidArgumentError('removeOpts should be of type "object"');
}
const method = 'DELETE';
const headers = {};
if (removeOpts !== null && removeOpts !== void 0 && removeOpts.governanceBypass) {
headers['X-Amz-Bypass-Governance-Retention'] = true;
}
if (removeOpts !== null && removeOpts !== void 0 && removeOpts.forceDelete) {
headers['x-minio-force-delete'] = true;
}
const queryParams = {};
if (removeOpts !== null && removeOpts !== void 0 && removeOpts.versionId) {
queryParams.versionId = `${removeOpts.versionId}`;
}
const query = qs.stringify(queryParams);
await this.makeRequestAsyncOmit({
method,
bucketName,
objectName,
headers,
query
}, '', [200, 204]);
}
// Calls implemented below are related to multipart.
listIncompleteUploads(bucket, prefix, recursive) {
if (prefix === undefined) {
prefix = '';
}
if (recursive === undefined) {
recursive = false;
}
if (!isValidBucketName(bucket)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucket);
}
if (!isValidPrefix(prefix)) {
throw new errors.InvalidPrefixError(`Invalid prefix : ${prefix}`);
}
if (!isBoolean(recursive)) {
throw new TypeError('recursive should be of type "boolean"');
}
const delimiter = recursive ? '' : '/';
let keyMarker = '';
let uploadIdMarker = '';
const uploads = [];
let ended = false;
// TODO: refactor this with async/await and `stream.Readable.from`
const readStream = new stream.Readable({
objectMode: true
});
readStream._read = () => {
// push one upload info per _read()
if (uploads.length) {
return readStream.push(uploads.shift());
}
if (ended) {
return readStream.push(null);
}
this.listIncompleteUploadsQuery(bucket, prefix, keyMarker, uploadIdMarker, delimiter).then(result => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
result.prefixes.forEach(prefix => uploads.push(prefix));
async.eachSeries(result.uploads, (upload, cb) => {
// for each incomplete upload add the sizes of its uploaded parts
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
this.listParts(bucket, upload.key, upload.uploadId).then(parts => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
upload.size = parts.reduce((acc, item) => acc + item.size, 0);
uploads.push(upload);
cb();
}, err => cb(err));
}, err => {
if (err) {
readStream.emit('error', err);
return;
}
if (result.isTruncated) {
keyMarker = result.nextKeyMarker;
uploadIdMarker = result.nextUploadIdMarker;
} else {
ended = true;
}
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
readStream._read();
});
}, e => {
readStream.emit('error', e);
});
};
return readStream;
}
/**
* Called by listIncompleteUploads to fetch a batch of incomplete uploads.
*/
async listIncompleteUploadsQuery(bucketName, prefix, keyMarker, uploadIdMarker, delimiter) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isString(prefix)) {
throw new TypeError('prefix should be of type "string"');
}
if (!isString(keyMarker)) {
throw new TypeError('keyMarker should be of type "string"');
}
if (!isString(uploadIdMarker)) {
throw new TypeError('uploadIdMarker should be of type "string"');
}
if (!isString(delimiter)) {
throw new TypeError('delimiter should be of type "string"');
}
const queries = [];
queries.push(`prefix=${uriEscape(prefix)}`);
queries.push(`delimiter=${uriEscape(delimiter)}`);
if (keyMarker) {
queries.push(`key-marker=${uriEscape(keyMarker)}`);
}
if (uploadIdMarker) {
queries.push(`upload-id-marker=${uploadIdMarker}`);
}
const maxUploads = 1000;
queries.push(`max-uploads=${maxUploads}`);
queries.sort();
queries.unshift('uploads');
let query = '';
if (queries.length > 0) {
query = `${queries.join('&')}`;
}
const method = 'GET';
const res = await this.makeRequestAsync({
method,
bucketName,
query
});
const body = await readAsString(res);
return xmlParsers.parseListMultipart(body);
}
/**
* Initiate a new multipart upload.
* @internal
*/
async initiateNewMultipartUpload(bucketName, objectName, headers) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!isObject(headers)) {
throw new errors.InvalidObjectNameError('contentType should be of type "object"');
}
const method = 'POST';
const query = 'uploads';
const res = await this.makeRequestAsync({
method,
bucketName,
objectName,
query,
headers
});
const body = await readAsBuffer(res);
return parseInitiateMultipart(body.toString());
}
/**
* Internal Method to abort a multipart upload request in case of any errors.
*
* @param bucketName - Bucket Name
* @param objectName - Object Name
* @param uploadId - id of a multipart upload to cancel during compose object sequence.
*/
async abortMultipartUpload(bucketName, objectName, uploadId) {
const method = 'DELETE';
const query = `uploadId=${uploadId}`;
const requestOptions = {
method,
bucketName,
objectName: objectName,
query
};
await this.makeRequestAsyncOmit(requestOptions, '', [204]);
}
async findUploadId(bucketName, objectName) {
var _latestUpload;
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
let latestUpload;
let keyMarker = '';
let uploadIdMarker = '';
for (;;) {
const result = await this.listIncompleteUploadsQuery(bucketName, objectName, keyMarker, uploadIdMarker, '');
for (const upload of result.uploads) {
if (upload.key === objectName) {
if (!latestUpload || upload.initiated.getTime() > latestUpload.initiated.getTime()) {
latestUpload = upload;
}
}
}
if (result.isTruncated) {
keyMarker = result.nextKeyMarker;
uploadIdMarker = result.nextUploadIdMarker;
continue;
}
break;
}
return (_latestUpload = latestUpload) === null || _latestUpload === void 0 ? void 0 : _latestUpload.uploadId;
}
/**
* this call will aggregate the parts on the server into a single object.
*/
async completeMultipartUpload(bucketName, objectName, uploadId, etags) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!isString(uploadId)) {
throw new TypeError('uploadId should be of type "string"');
}
if (!isObject(etags)) {
throw new TypeError('etags should be of type "Array"');
}
if (!uploadId) {
throw new errors.InvalidArgumentError('uploadId cannot be empty');
}
const method = 'POST';
const query = `uploadId=${uriEscape(uploadId)}`;
const builder = new xml2js.Builder();
const payload = builder.buildObject({
CompleteMultipartUpload: {
$: {
xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
},
Part: etags.map(etag => {
return {
PartNumber: etag.part,
ETag: etag.etag
};
})
}
});
const res = await this.makeRequestAsync({
method,
bucketName,
objectName,
query
}, payload);
const body = await readAsBuffer(res);
const result = parseCompleteMultipart(body.toString());
if (!result) {
throw new Error('BUG: failed to parse server response');
}
if (result.errCode) {
// Multipart Complete API returns an error XML after a 200 http status
throw new errors.S3Error(result.errMessage);
}
return {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
etag: result.etag,
versionId: getVersionId(res.headers)
};
}
/**
* Get part-info of all parts of an incomplete upload specified by uploadId.
*/
async listParts(bucketName, objectName, uploadId) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!isString(uploadId)) {
throw new TypeError('uploadId should be of type "string"');
}
if (!uploadId) {
throw new errors.InvalidArgumentError('uploadId cannot be empty');
}
const parts = [];
let marker = 0;
let result;
do {
result = await this.listPartsQuery(bucketName, objectName, uploadId, marker);
marker = result.marker;
parts.push(...result.parts);
} while (result.isTruncated);
return parts;
}
/**
* Called by listParts to fetch a batch of part-info
*/
async listPartsQuery(bucketName, objectName, uploadId, marker) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!isString(uploadId)) {
throw new TypeError('uploadId should be of type "string"');
}
if (!isNumber(marker)) {
throw new TypeError('marker should be of type "number"');
}
if (!uploadId) {
throw new errors.InvalidArgumentError('uploadId cannot be empty');
}
let query = `uploadId=${uriEscape(uploadId)}`;
if (marker) {
query += `&part-number-marker=${marker}`;
}
const method = 'GET';
const res = await this.makeRequestAsync({
method,
bucketName,
objectName,
query
});
return xmlParsers.parseListParts(await readAsString(res));
}
async listBuckets() {
const method = 'GET';
const regionConf = this.region || DEFAULT_REGION;
const httpRes = await this.makeRequestAsync({
method
}, '', [200], regionConf);
const xmlResult = await readAsString(httpRes);
return xmlParsers.parseListBucket(xmlResult);
}
/**
* Calculate part size given the object size. Part size will be atleast this.partSize
*/
calculatePartSize(size) {
if (!isNumber(size)) {
throw new TypeError('size should be of type "number"');
}
if (size > this.maxObjectSize) {
throw new TypeError(`size should not be more than ${this.maxObjectSize}`);
}
if (this.overRidePartSize) {
return this.partSize;
}
let partSize = this.partSize;
for (;;) {
// while(true) {...} throws linting error.
// If partSize is big enough to accomodate the object size, then use it.
if (partSize * 10000 > size) {
return partSize;
}
// Try part sizes as 64MB, 80MB, 96MB etc.
partSize += 16 * 1024 * 1024;
}
}
/**
* Uploads the object using contents from a file
*/
async fPutObject(bucketName, objectName, filePath, metaData) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
if (!isString(filePath)) {
throw new TypeError('filePath should be of type "string"');
}
if (metaData && !isObject(metaData)) {
throw new TypeError('metaData should be of type "object"');
}
// Inserts correct `content-type` attribute based on metaData and filePath
metaData = insertContentType(metaData || {}, filePath);
const stat = await fsp.lstat(filePath);
return await this.putObject(bucketName, objectName, fs.createReadStream(filePath), stat.size, metaData);
}
/**
* Uploading a stream, "Buffer" or "string".
* It's recommended to pass `size` argument with stream.
*/
async putObject(bucketName, objectName, stream, size, metaData) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError(`Invalid bucket name: ${bucketName}`);
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`);
}
// We'll need to shift arguments to the left because of metaData
// and size being optional.
if (isObject(size)) {
metaData = size;
}
// Ensures Metadata has appropriate prefix for A3 API
const headers = prependXAMZMeta(metaData);
if (typeof stream === 'string' || stream instanceof Buffer) {
// Adapts the non-stream interface into a stream.
size = stream.length;
stream = readableStream(stream);
} else if (!isReadableStream(stream)) {
throw new TypeError('third argument should be of type "stream.Readable" or "Buffer" or "string"');
}
if (isNumber(size) && size < 0) {
throw new errors.InvalidArgumentError(`size cannot be negative, given size: ${size}`);
}
// Get the part size and forward that to the BlockStream. Default to the
// largest block size possible if necessary.
if (!isNumber(size)) {
size = this.maxObjectSize;
}
// Get the part size and forward that to the BlockStream. Default to the
// largest block size possible if necessary.
if (size === undefined) {
const statSize = await getContentLength(stream);
if (statSize !== null) {
size = statSize;
}
}
if (!isNumber(size)) {
// Backward compatibility
size = this.maxObjectSize;
}
const partSize = this.calculatePartSize(size);
if (typeof stream === 'string' || stream.readableLength === 0 || Buffer.isBuffer(stream) || size <= partSize) {
const buf = isReadableStream(stream) ? await readAsBuffer(stream) : Buffer.from(stream);
return this.uploadBuffer(bucketName, objectName, headers, buf);
}
return this.uploadStream(bucketName, objectName, headers, stream, partSize);
}
/**
* method to upload buffer in one call
* @private
*/
async uploadBuffer(bucketName, objectName, headers, buf) {
const {
md5sum,
sha256sum
} = hashBinary(buf, this.enableSHA256);
headers['Content-Length'] = buf.length;
if (!this.enableSHA256) {
headers['Content-MD5'] = md5sum;
}
const res = await this.makeRequestStreamAsync({
method: 'PUT',
bucketName,
objectName,
headers
}, buf, sha256sum, [200], '');
await drainResponse(res);
return {
etag: sanitizeETag(res.headers.etag),
versionId: getVersionId(res.headers)
};
}
/**
* upload stream with MultipartUpload
* @private
*/
async uploadStream(bucketName, objectName, headers, body, partSize) {
// A map of the previously uploaded chunks, for resuming a file upload. This
// will be null if we aren't resuming an upload.
const oldParts = {};
// Keep track of the etags for aggregating the chunks together later. Each
// etag represents a single chunk of the file.
const eTags = [];
const previousUploadId = await this.findUploadId(bucketName, objectName);
let uploadId;
if (!previousUploadId) {
uploadId = await this.initiateNewMultipartUpload(bucketName, objectName, headers);
} else {
uploadId = previousUploadId;
const oldTags = await this.listParts(bucketName, objectName, previousUploadId);
oldTags.forEach(e => {
oldParts[e.part] = e;
});
}
const chunkier = new BlockStream2({
size: partSize,
zeroPadding: false
});
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [_, o] = await Promise.all([new Promise((resolve, reject) => {
body.pipe(chunkier).on('error', reject);
chunkier.on('end', resolve).on('error', reject);
}), (async () => {
let partNumber = 1;
for await (const chunk of chunkier) {
const md5 = crypto.createHash('md5').update(chunk).digest();
const oldPart = oldParts[partNumber];
if (oldPart) {
if (oldPart.etag === md5.toString('hex')) {
eTags.push({
part: partNumber,
etag: oldPart.etag
});
partNumber++;
continue;