minio
Version:
S3 Compatible Cloud Storage client
752 lines (668 loc) • 23.9 kB
text/typescript
import type * as http from 'node:http'
import type stream from 'node:stream'
import crc32 from 'buffer-crc32'
import { XMLParser } from 'fast-xml-parser'
import * as errors from '../errors.ts'
import { SelectResults } from '../helpers.ts'
import { isObject, parseXml, readableStream, sanitizeETag, sanitizeObjectKey, sanitizeSize, toArray } from './helper.ts'
import { readAsString } from './response.ts'
import type {
BucketItemFromList,
BucketItemWithMetadata,
CommonPrefix,
CopyObjectResultV1,
ListBucketResultV1,
ObjectInfo,
ObjectLockInfo,
ObjectRowEntry,
ReplicationConfig,
Tags,
} from './type.ts'
import { RETENTION_VALIDITY_UNITS } from './type.ts'
// parse XML response for bucket region
export function parseBucketRegion(xml: string): string {
// return region information
return parseXml(xml).LocationConstraint
}
const fxp = new XMLParser()
const fxpWithoutNumParser = new XMLParser({
// @ts-ignore
numberParseOptions: {
skipLike: /./,
},
})
// Parse XML and return information as Javascript types
// parse error XML response
export function parseError(xml: string, headerInfo: Record<string, unknown>) {
let xmlErr = {}
const xmlObj = fxp.parse(xml)
if (xmlObj.Error) {
xmlErr = xmlObj.Error
}
const e = new errors.S3Error() as unknown as Record<string, unknown>
Object.entries(xmlErr).forEach(([key, value]) => {
e[key.toLowerCase()] = value
})
Object.entries(headerInfo).forEach(([key, value]) => {
e[key] = value
})
return e
}
// Generates an Error object depending on http statusCode and XML body
export async function parseResponseError(response: http.IncomingMessage): Promise<Record<string, string>> {
const statusCode = response.statusCode
let code = '',
message = ''
if (statusCode === 301) {
code = 'MovedPermanently'
message = 'Moved Permanently'
} else if (statusCode === 307) {
code = 'TemporaryRedirect'
message = 'Are you using the correct endpoint URL?'
} else if (statusCode === 403) {
code = 'AccessDenied'
message = 'Valid and authorized credentials required'
} else if (statusCode === 404) {
code = 'NotFound'
message = 'Not Found'
} else if (statusCode === 405) {
code = 'MethodNotAllowed'
message = 'Method Not Allowed'
} else if (statusCode === 501) {
code = 'MethodNotAllowed'
message = 'Method Not Allowed'
} else if (statusCode === 503) {
code = 'SlowDown'
message = 'Please reduce your request rate.'
} else {
const hErrCode = response.headers['x-minio-error-code'] as string
const hErrDesc = response.headers['x-minio-error-desc'] as string
if (hErrCode && hErrDesc) {
code = hErrCode
message = hErrDesc
}
}
const headerInfo: Record<string, string | undefined | null> = {}
// A value created by S3 compatible server that uniquely identifies the request.
headerInfo.amzRequestid = response.headers['x-amz-request-id'] as string | undefined
// A special token that helps troubleshoot API replies and issues.
headerInfo.amzId2 = response.headers['x-amz-id-2'] as string | undefined
// Region where the bucket is located. This header is returned only
// in HEAD bucket and ListObjects response.
headerInfo.amzBucketRegion = response.headers['x-amz-bucket-region'] as string | undefined
const xmlString = await readAsString(response)
if (xmlString) {
throw parseError(xmlString, headerInfo)
}
// Message should be instantiated for each S3Errors.
const e = new errors.S3Error(message, { cause: headerInfo })
// S3 Error code.
e.code = code
Object.entries(headerInfo).forEach(([key, value]) => {
// @ts-expect-error force set error properties
e[key] = value
})
throw e
}
/**
* parse XML response for list objects v2 with metadata in a bucket
*/
export function parseListObjectsV2WithMetadata(xml: string) {
const result: {
objects: Array<BucketItemWithMetadata>
isTruncated: boolean
nextContinuationToken: string
} = {
objects: [],
isTruncated: false,
nextContinuationToken: '',
}
let xmlobj = parseXml(xml)
if (!xmlobj.ListBucketResult) {
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
}
xmlobj = xmlobj.ListBucketResult
if (xmlobj.IsTruncated) {
result.isTruncated = xmlobj.IsTruncated
}
if (xmlobj.NextContinuationToken) {
result.nextContinuationToken = xmlobj.NextContinuationToken
}
if (xmlobj.Contents) {
toArray(xmlobj.Contents).forEach((content) => {
const name = sanitizeObjectKey(content.Key)
const lastModified = new Date(content.LastModified)
const etag = sanitizeETag(content.ETag)
const size = content.Size
let tags: Tags = {}
if (content.UserTags != null) {
toArray(content.UserTags.split('&')).forEach((tag) => {
const [key, value] = tag.split('=')
tags[key] = value
})
} else {
tags = {}
}
let metadata
if (content.UserMetadata != null) {
metadata = toArray(content.UserMetadata)[0]
} else {
metadata = null
}
result.objects.push({ name, lastModified, etag, size, metadata, tags })
})
}
if (xmlobj.CommonPrefixes) {
toArray(xmlobj.CommonPrefixes).forEach((commonPrefix) => {
result.objects.push({ prefix: sanitizeObjectKey(toArray(commonPrefix.Prefix)[0]), size: 0 })
})
}
return result
}
export type UploadedPart = {
part: number
lastModified?: Date
etag: string
size: number
}
// parse XML response for list parts of an in progress multipart upload
export function parseListParts(xml: string): {
isTruncated: boolean
marker: number
parts: UploadedPart[]
} {
let xmlobj = parseXml(xml)
const result: {
isTruncated: boolean
marker: number
parts: UploadedPart[]
} = {
isTruncated: false,
parts: [],
marker: 0,
}
if (!xmlobj.ListPartsResult) {
throw new errors.InvalidXMLError('Missing tag: "ListPartsResult"')
}
xmlobj = xmlobj.ListPartsResult
if (xmlobj.IsTruncated) {
result.isTruncated = xmlobj.IsTruncated
}
if (xmlobj.NextPartNumberMarker) {
result.marker = toArray(xmlobj.NextPartNumberMarker)[0] || ''
}
if (xmlobj.Part) {
toArray(xmlobj.Part).forEach((p) => {
const part = parseInt(toArray(p.PartNumber)[0], 10)
const lastModified = new Date(p.LastModified)
const etag = p.ETag.replace(/^"/g, '')
.replace(/"$/g, '')
.replace(/^"/g, '')
.replace(/"$/g, '')
.replace(/^"/g, '')
.replace(/"$/g, '')
result.parts.push({ part, lastModified, etag, size: parseInt(p.Size, 10) })
})
}
return result
}
export function parseListBucket(xml: string): BucketItemFromList[] {
let result: BucketItemFromList[] = []
const listBucketResultParser = new XMLParser({
parseTagValue: true, // Enable parsing of values
numberParseOptions: {
leadingZeros: false, // Disable number parsing for values with leading zeros
hex: false, // Disable hex number parsing - Invalid bucket name
skipLike: /^[0-9]+$/, // Skip number parsing if the value consists entirely of digits
},
tagValueProcessor: (tagName, tagValue = '') => {
// Ensure that the Name tag is always treated as a string
if (tagName === 'Name') {
return tagValue.toString()
}
return tagValue
},
ignoreAttributes: false, // Ensure that all attributes are parsed
})
const parsedXmlRes = listBucketResultParser.parse(xml)
if (!parsedXmlRes.ListAllMyBucketsResult) {
throw new errors.InvalidXMLError('Missing tag: "ListAllMyBucketsResult"')
}
const { ListAllMyBucketsResult: { Buckets = {} } = {} } = parsedXmlRes
if (Buckets.Bucket) {
result = toArray(Buckets.Bucket).map((bucket = {}) => {
const { Name: bucketName, CreationDate } = bucket
const creationDate = new Date(CreationDate)
return { name: bucketName, creationDate }
})
}
return result
}
export function parseInitiateMultipart(xml: string): string {
let xmlobj = parseXml(xml)
if (!xmlobj.InitiateMultipartUploadResult) {
throw new errors.InvalidXMLError('Missing tag: "InitiateMultipartUploadResult"')
}
xmlobj = xmlobj.InitiateMultipartUploadResult
if (xmlobj.UploadId) {
return xmlobj.UploadId
}
throw new errors.InvalidXMLError('Missing tag: "UploadId"')
}
export function parseReplicationConfig(xml: string): ReplicationConfig {
const xmlObj = parseXml(xml)
const { Role, Rule } = xmlObj.ReplicationConfiguration
return {
ReplicationConfiguration: {
role: Role,
rules: toArray(Rule),
},
}
}
export function parseObjectLegalHoldConfig(xml: string) {
const xmlObj = parseXml(xml)
return xmlObj.LegalHold
}
export function parseTagging(xml: string) {
const xmlObj = parseXml(xml)
let result = []
if (xmlObj.Tagging && xmlObj.Tagging.TagSet && xmlObj.Tagging.TagSet.Tag) {
const tagResult = xmlObj.Tagging.TagSet.Tag
// if it is a single tag convert into an array so that the return value is always an array.
if (isObject(tagResult)) {
result.push(tagResult)
} else {
result = tagResult
}
}
return result
}
// parse XML response when a multipart upload is completed
export function parseCompleteMultipart(xml: string) {
const xmlobj = parseXml(xml).CompleteMultipartUploadResult
if (xmlobj.Location) {
const location = toArray(xmlobj.Location)[0]
const bucket = toArray(xmlobj.Bucket)[0]
const key = xmlobj.Key
const etag = xmlobj.ETag.replace(/^"/g, '')
.replace(/"$/g, '')
.replace(/^"/g, '')
.replace(/"$/g, '')
.replace(/^"/g, '')
.replace(/"$/g, '')
return { location, bucket, key, etag }
}
// Complete Multipart can return XML Error after a 200 OK response
if (xmlobj.Code && xmlobj.Message) {
const errCode = toArray(xmlobj.Code)[0]
const errMessage = toArray(xmlobj.Message)[0]
return { errCode, errMessage }
}
}
type UploadID = string
export type ListMultipartResult = {
uploads: {
key: string
uploadId: UploadID
initiator?: { id: string; displayName: string }
owner?: { id: string; displayName: string }
storageClass: unknown
initiated: Date
}[]
prefixes: {
prefix: string
}[]
isTruncated: boolean
nextKeyMarker: string
nextUploadIdMarker: string
}
// parse XML response for listing in-progress multipart uploads
export function parseListMultipart(xml: string): ListMultipartResult {
const result: ListMultipartResult = {
prefixes: [],
uploads: [],
isTruncated: false,
nextKeyMarker: '',
nextUploadIdMarker: '',
}
let xmlobj = parseXml(xml)
if (!xmlobj.ListMultipartUploadsResult) {
throw new errors.InvalidXMLError('Missing tag: "ListMultipartUploadsResult"')
}
xmlobj = xmlobj.ListMultipartUploadsResult
if (xmlobj.IsTruncated) {
result.isTruncated = xmlobj.IsTruncated
}
if (xmlobj.NextKeyMarker) {
result.nextKeyMarker = xmlobj.NextKeyMarker
}
if (xmlobj.NextUploadIdMarker) {
result.nextUploadIdMarker = xmlobj.nextUploadIdMarker || ''
}
if (xmlobj.CommonPrefixes) {
toArray(xmlobj.CommonPrefixes).forEach((prefix) => {
// @ts-expect-error index check
result.prefixes.push({ prefix: sanitizeObjectKey(toArray<string>(prefix.Prefix)[0]) })
})
}
if (xmlobj.Upload) {
toArray(xmlobj.Upload).forEach((upload) => {
const uploadItem: ListMultipartResult['uploads'][number] = {
key: upload.Key,
uploadId: upload.UploadId,
storageClass: upload.StorageClass,
initiated: new Date(upload.Initiated),
}
if (upload.Initiator) {
uploadItem.initiator = { id: upload.Initiator.ID, displayName: upload.Initiator.DisplayName }
}
if (upload.Owner) {
uploadItem.owner = { id: upload.Owner.ID, displayName: upload.Owner.DisplayName }
}
result.uploads.push(uploadItem)
})
}
return result
}
export function parseObjectLockConfig(xml: string): ObjectLockInfo {
const xmlObj = parseXml(xml)
let lockConfigResult = {} as ObjectLockInfo
if (xmlObj.ObjectLockConfiguration) {
lockConfigResult = {
objectLockEnabled: xmlObj.ObjectLockConfiguration.ObjectLockEnabled,
} as ObjectLockInfo
let retentionResp
if (
xmlObj.ObjectLockConfiguration &&
xmlObj.ObjectLockConfiguration.Rule &&
xmlObj.ObjectLockConfiguration.Rule.DefaultRetention
) {
retentionResp = xmlObj.ObjectLockConfiguration.Rule.DefaultRetention || {}
lockConfigResult.mode = retentionResp.Mode
}
if (retentionResp) {
const isUnitYears = retentionResp.Years
if (isUnitYears) {
lockConfigResult.validity = isUnitYears
lockConfigResult.unit = RETENTION_VALIDITY_UNITS.YEARS
} else {
lockConfigResult.validity = retentionResp.Days
lockConfigResult.unit = RETENTION_VALIDITY_UNITS.DAYS
}
}
}
return lockConfigResult
}
export function parseBucketVersioningConfig(xml: string) {
const xmlObj = parseXml(xml)
return xmlObj.VersioningConfiguration
}
// Used only in selectObjectContent API.
// extractHeaderType extracts the first half of the header message, the header type.
function extractHeaderType(stream: stream.Readable): string | undefined {
const headerNameLen = Buffer.from(stream.read(1)).readUInt8()
const headerNameWithSeparator = Buffer.from(stream.read(headerNameLen)).toString()
const splitBySeparator = (headerNameWithSeparator || '').split(':')
return splitBySeparator.length >= 1 ? splitBySeparator[1] : ''
}
function extractHeaderValue(stream: stream.Readable) {
const bodyLen = Buffer.from(stream.read(2)).readUInt16BE()
return Buffer.from(stream.read(bodyLen)).toString()
}
export function parseSelectObjectContentResponse(res: Buffer) {
const selectResults = new SelectResults({}) // will be returned
const responseStream = readableStream(res) // convert byte array to a readable responseStream
// @ts-ignore
while (responseStream._readableState.length) {
// Top level responseStream read tracker.
let msgCrcAccumulator // accumulate from start of the message till the message crc start.
const totalByteLengthBuffer = Buffer.from(responseStream.read(4))
msgCrcAccumulator = crc32(totalByteLengthBuffer)
const headerBytesBuffer = Buffer.from(responseStream.read(4))
msgCrcAccumulator = crc32(headerBytesBuffer, msgCrcAccumulator)
const calculatedPreludeCrc = msgCrcAccumulator.readInt32BE() // use it to check if any CRC mismatch in header itself.
const preludeCrcBuffer = Buffer.from(responseStream.read(4)) // read 4 bytes i.e 4+4 =8 + 4 = 12 ( prelude + prelude crc)
msgCrcAccumulator = crc32(preludeCrcBuffer, msgCrcAccumulator)
const totalMsgLength = totalByteLengthBuffer.readInt32BE()
const headerLength = headerBytesBuffer.readInt32BE()
const preludeCrcByteValue = preludeCrcBuffer.readInt32BE()
if (preludeCrcByteValue !== calculatedPreludeCrc) {
// Handle Header CRC mismatch Error
throw new Error(
`Header Checksum Mismatch, Prelude CRC of ${preludeCrcByteValue} does not equal expected CRC of ${calculatedPreludeCrc}`,
)
}
const headers: Record<string, unknown> = {}
if (headerLength > 0) {
const headerBytes = Buffer.from(responseStream.read(headerLength))
msgCrcAccumulator = crc32(headerBytes, msgCrcAccumulator)
const headerReaderStream = readableStream(headerBytes)
// @ts-ignore
while (headerReaderStream._readableState.length) {
const headerTypeName = extractHeaderType(headerReaderStream)
headerReaderStream.read(1) // just read and ignore it.
if (headerTypeName) {
headers[headerTypeName] = extractHeaderValue(headerReaderStream)
}
}
}
let payloadStream
const payLoadLength = totalMsgLength - headerLength - 16
if (payLoadLength > 0) {
const payLoadBuffer = Buffer.from(responseStream.read(payLoadLength))
msgCrcAccumulator = crc32(payLoadBuffer, msgCrcAccumulator)
// read the checksum early and detect any mismatch so we can avoid unnecessary further processing.
const messageCrcByteValue = Buffer.from(responseStream.read(4)).readInt32BE()
const calculatedCrc = msgCrcAccumulator.readInt32BE()
// Handle message CRC Error
if (messageCrcByteValue !== calculatedCrc) {
throw new Error(
`Message Checksum Mismatch, Message CRC of ${messageCrcByteValue} does not equal expected CRC of ${calculatedCrc}`,
)
}
payloadStream = readableStream(payLoadBuffer)
}
const messageType = headers['message-type']
switch (messageType) {
case 'error': {
const errorMessage = headers['error-code'] + ':"' + headers['error-message'] + '"'
throw new Error(errorMessage)
}
case 'event': {
const contentType = headers['content-type']
const eventType = headers['event-type']
switch (eventType) {
case 'End': {
selectResults.setResponse(res)
return selectResults
}
case 'Records': {
const readData = payloadStream?.read(payLoadLength)
selectResults.setRecords(readData)
break
}
case 'Progress':
{
switch (contentType) {
case 'text/xml': {
const progressData = payloadStream?.read(payLoadLength)
selectResults.setProgress(progressData.toString())
break
}
default: {
const errorMessage = `Unexpected content-type ${contentType} sent for event-type Progress`
throw new Error(errorMessage)
}
}
}
break
case 'Stats':
{
switch (contentType) {
case 'text/xml': {
const statsData = payloadStream?.read(payLoadLength)
selectResults.setStats(statsData.toString())
break
}
default: {
const errorMessage = `Unexpected content-type ${contentType} sent for event-type Stats`
throw new Error(errorMessage)
}
}
}
break
default: {
// Continuation message: Not sure if it is supported. did not find a reference or any message in response.
// It does not have a payload.
const warningMessage = `Un implemented event detected ${messageType}.`
// eslint-disable-next-line no-console
console.warn(warningMessage)
}
}
}
}
}
}
export function parseLifecycleConfig(xml: string) {
const xmlObj = parseXml(xml)
return xmlObj.LifecycleConfiguration
}
export function parseBucketEncryptionConfig(xml: string) {
return parseXml(xml)
}
export function parseObjectRetentionConfig(xml: string) {
const xmlObj = parseXml(xml)
const retentionConfig = xmlObj.Retention
return {
mode: retentionConfig.Mode,
retainUntilDate: retentionConfig.RetainUntilDate,
}
}
export function removeObjectsParser(xml: string) {
const xmlObj = parseXml(xml)
if (xmlObj.DeleteResult && xmlObj.DeleteResult.Error) {
// return errors as array always. as the response is object in case of single object passed in removeObjects
return toArray(xmlObj.DeleteResult.Error)
}
return []
}
// parse XML response for copy object
export function parseCopyObject(xml: string): CopyObjectResultV1 {
const result: CopyObjectResultV1 = {
etag: '',
lastModified: '',
}
let xmlobj = parseXml(xml)
if (!xmlobj.CopyObjectResult) {
throw new errors.InvalidXMLError('Missing tag: "CopyObjectResult"')
}
xmlobj = xmlobj.CopyObjectResult
if (xmlobj.ETag) {
result.etag = xmlobj.ETag.replace(/^"/g, '')
.replace(/"$/g, '')
.replace(/^"/g, '')
.replace(/"$/g, '')
.replace(/^"/g, '')
.replace(/"$/g, '')
}
if (xmlobj.LastModified) {
result.lastModified = new Date(xmlobj.LastModified)
}
return result
}
const formatObjInfo = (content: ObjectRowEntry, opts: { IsDeleteMarker?: boolean } = {}) => {
const { Key, LastModified, ETag, Size, VersionId, IsLatest } = content
if (!isObject(opts)) {
opts = {}
}
const name = sanitizeObjectKey(toArray(Key)[0] || '')
const lastModified = LastModified ? new Date(toArray(LastModified)[0] || '') : undefined
const etag = sanitizeETag(toArray(ETag)[0] || '')
const size = sanitizeSize(Size || '')
return {
name,
lastModified,
etag,
size,
versionId: VersionId,
isLatest: IsLatest,
isDeleteMarker: opts.IsDeleteMarker ? opts.IsDeleteMarker : false,
}
}
// parse XML response for list objects in a bucket
export function parseListObjects(xml: string) {
const result: { objects: ObjectInfo[]; isTruncated?: boolean; nextMarker?: string; versionIdMarker?: string } = {
objects: [],
isTruncated: false,
nextMarker: undefined,
versionIdMarker: undefined,
}
let isTruncated = false
let nextMarker, nextVersionKeyMarker
const xmlobj = fxpWithoutNumParser.parse(xml)
const parseCommonPrefixesEntity = (commonPrefixEntry: CommonPrefix[]) => {
if (commonPrefixEntry) {
toArray(commonPrefixEntry).forEach((commonPrefix) => {
result.objects.push({ prefix: sanitizeObjectKey(toArray(commonPrefix.Prefix)[0] || ''), size: 0 })
})
}
}
const listBucketResult: ListBucketResultV1 = xmlobj.ListBucketResult
const listVersionsResult: ListBucketResultV1 = xmlobj.ListVersionsResult
if (listBucketResult) {
if (listBucketResult.IsTruncated) {
isTruncated = listBucketResult.IsTruncated
}
if (listBucketResult.Contents) {
toArray(listBucketResult.Contents).forEach((content) => {
const name = sanitizeObjectKey(toArray(content.Key)[0] || '')
const lastModified = new Date(toArray(content.LastModified)[0] || '')
const etag = sanitizeETag(toArray(content.ETag)[0] || '')
const size = sanitizeSize(content.Size || '')
result.objects.push({ name, lastModified, etag, size })
})
}
if (listBucketResult.Marker) {
nextMarker = listBucketResult.Marker
} else if (isTruncated && result.objects.length > 0) {
nextMarker = result.objects[result.objects.length - 1]?.name
}
if (listBucketResult.CommonPrefixes) {
parseCommonPrefixesEntity(listBucketResult.CommonPrefixes)
}
}
if (listVersionsResult) {
if (listVersionsResult.IsTruncated) {
isTruncated = listVersionsResult.IsTruncated
}
if (listVersionsResult.Version) {
toArray(listVersionsResult.Version).forEach((content) => {
result.objects.push(formatObjInfo(content))
})
}
if (listVersionsResult.DeleteMarker) {
toArray(listVersionsResult.DeleteMarker).forEach((content) => {
result.objects.push(formatObjInfo(content, { IsDeleteMarker: true }))
})
}
if (listVersionsResult.NextKeyMarker) {
nextVersionKeyMarker = listVersionsResult.NextKeyMarker
}
if (listVersionsResult.NextVersionIdMarker) {
result.versionIdMarker = listVersionsResult.NextVersionIdMarker
}
if (listVersionsResult.CommonPrefixes) {
parseCommonPrefixesEntity(listVersionsResult.CommonPrefixes)
}
}
result.isTruncated = isTruncated
if (isTruncated) {
result.nextMarker = nextVersionKeyMarker || nextMarker
}
return result
}
export function uploadPartParser(xml: string) {
const xmlObj = parseXml(xml)
const respEl = xmlObj.CopyPartResult
return respEl
}