UNPKG

s3mini

Version:

👶 Tiny & fast S3 client for node and edge computing platforms

1,297 lines (1,178 loc) • 97.8 kB
'use strict'; import * as C from './consts.js'; import { hexFromBuffer, sha256, hmac, uriResourceEscape, getByteSize, sanitizeETag, uriEscape, parseXml, escapeXml, base64FromBuffer, extractErrCode, S3NetworkError, S3ServiceError, generateParts, toUint8Array, isBun, byCodePoint, } from './utils.js'; import type * as IT from './types.js'; /** * S3 class for interacting with S3-compatible object storage services. * This class provides methods for common S3 operations such as uploading, downloading, * and deleting objects, as well as multipart uploads. * * @class * @example * const s3 = new S3mini({ * accessKeyId: 'your-access-key', * secretAccessKey: 'your-secret-key', * endpoint: 'https://your-s3-endpoint.com/bucket-name', * region: 'auto' // by default is auto * }); * * // Upload a file * await s3.putObject('example.txt', 'Hello, World!'); * * // Download a file * const content = await s3.getObject('example.txt'); * * // Delete a file * await s3.deleteObject('example.txt'); */ class S3mini { /** * Creates an instance of the S3 class. * * @constructor * @param {Object} config - Configuration options for the S3 instance. * @param {string} config.accessKeyId - The access key ID for authentication. * @param {string} config.secretAccessKey - The secret access key for authentication. * @param {string} config.endpoint - The endpoint URL of the S3-compatible service. * @param {string} [config.region='auto'] - The region of the S3 service. * @param {number} [config.requestSizeInBytes=8388608] - The request size of a single request in bytes (AWS S3 is 8MB). * @param {number} [config.requestAbortTimeout=undefined] - The timeout in milliseconds after which a request should be aborted (careful on streamed requests). * @param {Object} [config.logger=null] - A logger object with methods like info, warn, error. * @param {typeof fetch} [config.fetch=globalThis.fetch] - Custom fetch implementation to use for HTTP requests. * @param {number} [config.minPartSize=8388608] - The minimum part size for multipart uploads in bytes (default is 8MB). * @throws {TypeError} Will throw an error if required parameters are missing or of incorrect type. */ readonly #accessKeyId: string; readonly #secretAccessKey: string; readonly endpoint: URL; readonly region: string; readonly bucketName: string; readonly requestSizeInBytes: number; readonly requestAbortTimeout?: number; readonly logger?: IT.Logger; readonly _fetch: typeof fetch; readonly minPartSize: number; private readonly _bun?: IT.NativeS3Client; private signingKeyDate?: string; private signingKey?: ArrayBuffer; constructor({ accessKeyId, secretAccessKey, endpoint, region = 'auto', requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES, requestAbortTimeout = undefined, logger = undefined, fetch = globalThis.fetch, minPartSize = C.MIN_PART_SIZE, }: IT.S3Config) { this._validateConstructorParams(accessKeyId, secretAccessKey, endpoint); this.#accessKeyId = accessKeyId; this.#secretAccessKey = secretAccessKey; this.endpoint = new URL(this._ensureValidUrl(endpoint)); this.region = region; this.bucketName = this._extractBucketName(); this.requestSizeInBytes = requestSizeInBytes; this.requestAbortTimeout = requestAbortTimeout; this.logger = logger; this._fetch = (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => fetch(input, init); this.minPartSize = minPartSize; // Bun's native client has its own transport, so a caller-supplied fetch would be silently // bypassed: only take the native path when the default fetch is in use. It also refuses empty // credentials, which anonymous access to public buckets relies on. if (isBun && fetch === globalThis.fetch && this._hasCredentials()) { // Bun's client is an origin plus a bucket name, so an endpoint carrying anything past the // bucket (host/bucket/prefix) cannot be expressed: the extra segments would be dropped and // every native request would silently land in the parent bucket, while the signed path stays // inside the prefix. Decline the native path rather than read and write the wrong location. const segments = this.endpoint.pathname.split('/').filter(Boolean); if (segments.length < 2) { const { S3Client } = ( globalThis as unknown as { Bun: { S3Client: new (o: Record<string, unknown>) => IT.NativeS3Client } } ).Bun; this._bun = new S3Client({ accessKeyId, secretAccessKey, endpoint: this.endpoint.origin, region: this.region, bucket: this.bucketName, // Bucket in the path means path-style; otherwise it is in the host and Bun has to be // told, or it would repeat the bucket in the path (bucket.host/bucket/key). virtualHostedStyle: segments.length === 0, }); } } } private _sanitize(obj: unknown): unknown { if (typeof obj !== 'object' || obj === null) { return obj; } return Object.keys(obj).reduce( (acc: Record<string, unknown>, key) => { if (C.SENSITIVE_KEYS_REDACTED.has(key.toLowerCase())) { acc[key] = '[REDACTED]'; } else if ( typeof (obj as Record<string, unknown>)[key] === 'object' && (obj as Record<string, unknown>)[key] !== null ) { acc[key] = this._sanitize((obj as Record<string, unknown>)[key]); } else { acc[key] = (obj as Record<string, unknown>)[key]; } return acc; }, Array.isArray(obj) ? [] : {}, ); } private _log( level: 'info' | 'warn' | 'error', message: string, additionalData: Record<string, unknown> | string = {}, ): void { if (this.logger && typeof this.logger[level] === 'function') { // Function to recursively sanitize an object // Sanitize the additional data const sanitizedData = this._sanitize(additionalData); // Prepare the log entry const logEntry = { timestamp: new Date().toISOString(), level, message, details: sanitizedData, // Include some general context, but sanitize sensitive parts context: this._sanitize({ region: this.region, endpoint: this.endpoint.toString(), // Only include the first few characters of the access key, if it exists accessKeyId: this.#accessKeyId ? `${this.#accessKeyId.substring(0, 4)}...` : undefined, }), }; // Log the sanitized entry this.logger[level](JSON.stringify(logEntry)); } } // S3 returns repeated elements as either an array or a single scalar object // (e.g. a lone <Contents> is not wrapped in a 1-element array). Normalize to array. private _asArray(value: unknown): unknown[] { return Array.isArray(value) ? value : [value]; } private _validateConstructorParams(accessKeyId: string, secretAccessKey: string, endpoint: string): void { if (typeof accessKeyId !== 'string') { throw new TypeError(C.ERROR_ACCESS_KEY_REQUIRED); } if (typeof secretAccessKey !== 'string') { throw new TypeError(C.ERROR_SECRET_KEY_REQUIRED); } if (typeof endpoint !== 'string' || endpoint.trim().length === 0) { throw new TypeError(C.ERROR_ENDPOINT_REQUIRED); } } /** * Check if credentials are configured (non-empty). * @returns true if both accessKeyId and secretAccessKey are non-empty. */ private _hasCredentials(): boolean { return this.#accessKeyId.trim().length > 0 && this.#secretAccessKey.trim().length > 0; } /** * Re-shape a Bun S3Error as the S3ServiceError the signed path throws, so callers see one error * type on every runtime. Bun does not expose the HTTP status, so it is recovered from the error * code where the S3 API pins it and left as 0 (unknown) otherwise. */ private _bunError(e: unknown): unknown { const err = e as { name?: string; code?: string; message?: string }; if (err?.name !== 'S3Error') { return e; } const status = err.code ? (C.S3_CODE_STATUS[err.code] ?? 0) : 0; const message = status ? `S3 returned ${status} – ${err.code}` : (err.message ?? String(e)); // The provider's wording goes where the signed path puts the error body. return new S3ServiceError(message, status, err.code, err.message); } /** True for a Bun S3Error the signed path would have absorbed as a tolerated 404. */ private _isBunNotFound(e: unknown): boolean { const code = (e as { code?: string })?.code; return !!code && C.S3_CODE_STATUS[code] === 404; } /** Run a read op via Bun-native S3, returning null when the object or its bucket is absent. */ private async _bunRead<T>(key: string, op: (f: IT.NativeS3File) => Promise<T>): Promise<T | null> { try { return await op(this._bun!.file(key)); } catch (e) { // Every signed reader tolerates 404 and answers null, so match on the status the code maps // to rather than on NoSuchKey alone: a missing *bucket* is equally a 404, and singling out // the key left it throwing here while returning null on the signed path. if (this._isBunNotFound(e)) { return null; } throw this._bunError(e); } } private _ensureValidUrl(raw: string): string { const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}`; try { new URL(candidate); // Find the last non-slash character let endIndex = candidate.length; while (endIndex > 0 && candidate[endIndex - 1] === '/') { endIndex--; } return endIndex === candidate.length ? candidate : candidate.substring(0, endIndex); } catch { const msg = `${C.ERROR_ENDPOINT_FORMAT} But provided: "${raw}"`; this._log('error', msg); throw new TypeError(msg); } } private _validateMethodIsGetOrHead(method: string): void { if (method !== 'GET' && method !== 'HEAD') { this._log('error', `${C.ERROR_PREFIX}method must be either GET or HEAD`); throw new Error(`${C.ERROR_PREFIX}method must be either GET or HEAD`); } } private _checkKey(key: string): void { if (typeof key !== 'string' || key.trim().length === 0) { this._log('error', C.ERROR_KEY_REQUIRED); throw new TypeError(C.ERROR_KEY_REQUIRED); } } private _checkDelimiter(delimiter: string): void { if (typeof delimiter !== 'string' || delimiter.trim().length === 0) { this._log('error', C.ERROR_DELIMITER_REQUIRED); throw new TypeError(C.ERROR_DELIMITER_REQUIRED); } } private _checkPrefix(prefix: string): void { if (typeof prefix !== 'string') { this._log('error', C.ERROR_PREFIX_TYPE); throw new TypeError(C.ERROR_PREFIX_TYPE); } } // private _checkMaxKeys(maxKeys: number): void { // if (typeof maxKeys !== 'number' || maxKeys <= 0) { // this._log('error', C.ERROR_MAX_KEYS_TYPE); // throw new TypeError(C.ERROR_MAX_KEYS_TYPE); // } // } private _checkOpts(opts: object): void { if (typeof opts !== 'object') { this._log('error', `${C.ERROR_PREFIX}opts must be an object`); throw new TypeError(`${C.ERROR_PREFIX}opts must be an object`); } } private _filterIfHeaders(opts: Record<string, unknown>): { filteredOpts: Record<string, string>; conditionalHeaders: Record<string, unknown>; } { const filteredOpts: Record<string, string> = {}; const conditionalHeaders: Record<string, unknown> = {}; for (const [key, value] of Object.entries(opts)) { if (C.IFHEADERS.has(key.toLowerCase())) { conditionalHeaders[key] = value; } else { filteredOpts[key] = value as string; } } return { filteredOpts, conditionalHeaders }; } // private _validateData(data: unknown): BodyInit { // if (data instanceof ArrayBuffer) { // return data; // } // if (data instanceof Uint8Array) { // return data as unknown as BodyInit; // } // if ((globalThis.Buffer && data instanceof globalThis.Buffer) || typeof data === 'string') { // return data as BodyInit; // } // this._log('error', C.ERROR_DATA_BUFFER_REQUIRED); // throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED); // } private _validateUploadPartParams( key: string, uploadId: string, data: IT.DataInput, partNumber: number, opts: object, ): BodyInit { this._checkKey(key); if (typeof uploadId !== 'string' || uploadId.trim().length === 0) { this._log('error', C.ERROR_UPLOAD_ID_REQUIRED); throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED); } if (!Number.isInteger(partNumber) || partNumber <= 0) { this._log('error', `${C.ERROR_PREFIX}partNumber must be a positive integer`); throw new TypeError(`${C.ERROR_PREFIX}partNumber must be a positive integer`); } this._checkOpts(opts); return data as BodyInit; } private async _sign( method: IT.HttpMethod, keyPath: string, query: Record<string, unknown> = {}, headers: Record<string, string | number> = {}, ): Promise<{ url: string; headers: Record<string, string | number> }> { // Create URL without appending keyPath first const url = new URL(this.endpoint); // Properly format the pathname to avoid double slashes if (keyPath && keyPath.length > 0) { url.pathname = url.pathname === '/' ? `/${keyPath.replace(/^\/+/, '')}` : `${url.pathname}/${keyPath.replace(/^\/+/, '')}`; } // If no credentials, return unsigned request (for public bucket access) if (!this._hasCredentials()) { headers[C.HEADER_HOST] = url.host; return { url: url.toString(), headers }; } const d = new Date(); const year = d.getUTCFullYear(); const month = String(d.getUTCMonth() + 1).padStart(2, '0'); const day = String(d.getUTCDate()).padStart(2, '0'); const shortDatetime = `${year}${month}${day}`; const fullDatetime = `${shortDatetime}T${String(d.getUTCHours()).padStart(2, '0')}${String(d.getUTCMinutes()).padStart(2, '0')}${String(d.getUTCSeconds()).padStart(2, '0')}Z`; const credentialScope = `${shortDatetime}/${this.region}/${C.S3_SERVICE}/${C.AWS_REQUEST_TYPE}`; headers[C.HEADER_AMZ_CONTENT_SHA256] = C.UNSIGNED_PAYLOAD; headers[C.HEADER_AMZ_DATE] = fullDatetime; headers[C.HEADER_HOST] = url.host; const ignoredHeaders = new Set(['authorization', 'content-length', 'content-type', 'user-agent']); const sortedHeaders = Object.entries(headers) .map(([key, value]): [string, string] => [key.toLowerCase(), String(value).trim()]) .filter(([lowerKey]) => !ignoredHeaders.has(lowerKey)) .sort(([a], [b]) => byCodePoint(a, b)); const canonicalHeaders = sortedHeaders.map(([k, v]) => `${k}:${v}`).join('\n'); const signedHeaders = sortedHeaders.map(([k]) => k).join(';'); const canonicalRequest = `${method}\n${url.pathname}\n${this._buildCanonicalQueryString(query)}\n${canonicalHeaders}\n\n${signedHeaders}\n${C.UNSIGNED_PAYLOAD}`; const stringToSign = `${C.AWS_ALGORITHM}\n${fullDatetime}\n${credentialScope}\n${hexFromBuffer(await sha256(canonicalRequest))}`; if (shortDatetime !== this.signingKeyDate || !this.signingKey) { this.signingKeyDate = shortDatetime; this.signingKey = await this._getSignatureKey(shortDatetime); } const signature = hexFromBuffer(await hmac(this.signingKey, stringToSign)); headers[C.HEADER_AUTHORIZATION] = `${C.AWS_ALGORITHM} Credential=${this.#accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`; return { url: url.toString(), headers }; } private async _signedRequest( method: IT.HttpMethod, // 'GET' | 'HEAD' | 'PUT' | 'POST' | 'DELETE' key: string, // ‘’ allowed for bucket‑level ops { query = {}, // ?query=string body = '', // BodyInit | undefined headers = {}, // extra/override headers tolerated = [], // [200, 404] etc. withQuery = false, // append query string to signed URL }: { query?: Record<string, unknown>; body?: BodyInit; headers?: Record<string, string | number | undefined> | IT.SSECHeaders | IT.AWSHeaders; tolerated?: number[]; withQuery?: boolean; } = {}, ): Promise<Response> { // Basic validation // if (!['GET', 'HEAD', 'PUT', 'POST', 'DELETE'].includes(method)) { // throw new Error(`${C.ERROR_PREFIX}Unsupported HTTP method ${method as string}`); // } const { filteredOpts, conditionalHeaders } = ['GET', 'HEAD'].includes(method) ? this._filterIfHeaders(query) : { filteredOpts: query, conditionalHeaders: {} }; const baseHeaders: Record<string, string | number> = { [C.HEADER_AMZ_CONTENT_SHA256]: C.UNSIGNED_PAYLOAD, // ...(['GET', 'HEAD'].includes(method) ? { [C.HEADER_CONTENT_TYPE]: C.JSON_CONTENT_TYPE } : {}), ...headers, ...conditionalHeaders, }; const encodedKey = key ? uriResourceEscape(key) : ''; const { url, headers: signedHeaders } = await this._sign(method, encodedKey, filteredOpts, baseHeaders); if (Object.keys(query).length > 0) { withQuery = true; // append query string to signed URL } const finalUrl = withQuery && Object.keys(filteredOpts).length ? `${url}?${this._buildCanonicalQueryString(filteredOpts)}` : url; const signedHeadersString = Object.fromEntries( Object.entries(signedHeaders).map(([k, v]) => [k, String(v)]), ) as Record<string, string>; return this._sendRequest(finalUrl, method, signedHeadersString, body, tolerated); } /** * Sanitizes an ETag value by removing surrounding quotes and whitespace. * Still returns RFC compliant ETag. https://www.rfc-editor.org/rfc/rfc9110#section-8.8.3 * @param {string} etag - The ETag value to sanitize. * @returns {string} The sanitized ETag value. * @example * const cleanEtag = s3.sanitizeETag('"abc123"'); // Returns: 'abc123' */ public sanitizeETag(etag: string): string { return sanitizeETag(etag); } /** * Creates a new bucket. * This method sends a request to create a new bucket in the specified in endpoint. * @returns A promise that resolves to true if the bucket was created successfully, false otherwise. */ public async createBucket(): Promise<boolean> { const xmlBody = ` <CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <LocationConstraint>${this.region}</LocationConstraint> </CreateBucketConfiguration> `; const headers = { [C.HEADER_CONTENT_TYPE]: C.XML_CONTENT_TYPE, [C.HEADER_CONTENT_LENGTH]: getByteSize(xmlBody), }; const res = await this._signedRequest('PUT', '', { body: xmlBody, headers, tolerated: [200, 404, 403, 409], // don’t throw on 404/403 // 409 = bucket already exists }); return res.status === 200; } private _extractBucketName(): string { const url = this.endpoint; // Path-style: bucket is the first non-empty path segment const firstSegment = url.pathname.split('/').find(Boolean); if (firstSegment) { return firstSegment; } // Virtual-hosted style: bucket is the first subdomain label const hostname = url.hostname; // IP addresses (v4: digits+dots, v6: contains colons) can't carry a bucket subdomain if (/^\d+\.\d+\.\d+\.\d+$/.test(hostname) || hostname.includes(':')) { return ''; } const labels = hostname.split('.'); // Need ≥3 labels for virtual-hosted (bucket.service.tld) // Single-label (localhost) or two-label (example.com) have no room for a bucket subdomain if (labels.length < 3) { return ''; } return labels[0]!; } /** * Checks if a bucket exists. * This method sends a request to check if the specified bucket exists in the S3-compatible service. * @returns A promise that resolves to true if the bucket exists, false otherwise. */ public async bucketExists(): Promise<boolean> { const res = await this._signedRequest('HEAD', '', { tolerated: [200, 404, 403] }); return res.status === 200; } /** * Sets bucket versioning status (PutBucketVersioning). * Required before object versioning APIs (`listObjectVersions`, versioned delete/copy) are useful. * @param {'Enabled' | 'Suspended'} status - Versioning status to apply. * @returns {Promise<boolean>} True when the service accepts the configuration (HTTP 200). * @example * await s3.setBucketVersioning('Enabled'); */ public async setBucketVersioning(status: 'Enabled' | 'Suspended'): Promise<boolean> { if (status !== 'Enabled' && status !== 'Suspended') { throw new TypeError(`${C.ERROR_PREFIX}status must be 'Enabled' or 'Suspended'`); } const xmlBody = '<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' + `<Status>${status}</Status>` + '</VersioningConfiguration>'; const res = await this._signedRequest('PUT', '', { query: { versioning: '' }, body: xmlBody, headers: { [C.HEADER_CONTENT_TYPE]: C.XML_CONTENT_TYPE, [C.HEADER_CONTENT_LENGTH]: getByteSize(xmlBody), }, withQuery: true, tolerated: [200], }); return res.status === 200; } /** * Gets bucket versioning status (GetBucketVersioning). * @returns {Promise<'Enabled' | 'Suspended' | 'Off'>} Current status. `'Off'` when the config is empty/unset. */ public async getBucketVersioning(): Promise<'Enabled' | 'Suspended' | 'Off'> { const res = await this._signedRequest('GET', '', { query: { versioning: '' }, withQuery: true, tolerated: [200, 404], }); if (res.status !== 200) { void res.body?.cancel(); return 'Off'; } const raw = parseXml(await res.text()) as Record<string, unknown>; const cfg = (raw.VersioningConfiguration || raw.versioningConfiguration || raw) as Record<string, unknown>; const status = cfg.Status ?? cfg.status; if (status === 'Enabled' || status === 'Suspended') { return status; } return 'Off'; } /** * Lists objects in the bucket with optional filtering and no pagination. * This method retrieves all objects matching the criteria (not paginated like listObjectsV2). * Pass `{ versions: true }` in opts to list object versions (ListObjectVersions API). * @param {string} [delimiter='/'] - The delimiter to use for grouping objects. * @param {string} [prefix=''] - The prefix to filter objects by. * @param {number} [maxKeys] - The maximum number of keys to return. If not provided, all keys will be returned. * @param {Record<string, unknown>} [opts={}] - Additional options for the request. Use `{ versions: true }` for version listing. * @returns {Promise<IT.ListObject[] | null>} A promise that resolves to an array of objects, or null if the bucket does not exist. An empty bucket resolves to an empty array. * @example * // List all objects * const objects = await s3.listObjects(); * * // List objects with prefix * const photos = await s3.listObjects('/', 'photos/', 100); * * // List object versions (includes VersionId / IsLatest; may include delete markers) * const versions = await s3.listObjects('/', 'photos/', undefined, { versions: true }); */ public async listObjects( delimiter: string = '/', prefix: string = '', maxKeys?: number, opts: Record<string, unknown> = {}, ): Promise<IT.ListObject[] | null> { this._checkDelimiter(delimiter); this._checkPrefix(prefix); this._checkOpts(opts); if (this._bun && delimiter === '/' && !this._isVersionsMode(opts)) { const extraKeys = Object.keys(opts).filter(k => k !== 'delimiter'); if (extraKeys.length === 0) { return this._bunListAll(prefix, maxKeys, opts.delimiter as string | undefined); } } const keyPath = delimiter === '/' ? delimiter : uriEscape(delimiter); const unlimited = !(maxKeys && maxKeys > 0); let remaining = unlimited ? Infinity : maxKeys; let token: string | undefined; const all: IT.ListObject[] = []; do { const batchResult = await this._fetchObjectBatch(keyPath, prefix, remaining, token, opts); if (batchResult === null) { return null; // 404 - bucket not found } all.push(...batchResult.objects); if (!unlimited) { remaining -= batchResult.objects.length; } token = batchResult.continuationToken; } while (token && remaining > 0); return all; } /** * Lists objects in the bucket with optional filtering and pagination using a continuation token. * This method retrieves objects matching the criteria (paginated like listObjectsV2). * Pass `{ versions: true }` in opts to list object versions (uses key-marker / version-id-marker under the hood; * the returned token is opaque and only valid with the same opts). * @param {string} [delimiter='/'] - The delimiter to use for grouping objects. * @param {string} [prefix=''] - The prefix to filter objects by. * @param {number} [maxKeys] - The maximum number of keys to return. Uses a default value of 100. * @param {string} [nextContinuationToken] - The nextContinuationToken to continue previous results. If not provided, starts from the beginning. * @param {Record<string, unknown>} [opts={}] - Additional options for the request. Use `{ versions: true }` for version listing. * @returns {Promise<{objects: IT.ListObject[] | null; nextContinuationToken?: string } | undefined | null>} A promise that resolves to an array of objects, along with nextContinuationToken if there are more reccords, or null if the bucket does not exist. * @example * // List all objects * const { objects, nextContinuationToken } = await s3.listObjectsPaged(); * * // List 200 objects with prefix * const photos = await s3.listObjectsPaged('/', 'photos/', 200, "token..."); */ public async listObjectsPaged( delimiter: string = '/', prefix: string = '', maxKeys: number = 100, nextContinuationToken?: string, opts: Record<string, unknown> = {}, ): Promise<{ objects: IT.ListObject[] | null; nextContinuationToken?: string } | undefined | null> { this._checkDelimiter(delimiter); this._checkPrefix(prefix); this._checkOpts(opts); const keyPath = delimiter === '/' ? delimiter : uriEscape(delimiter); let token: string | undefined = nextContinuationToken; let remaining = maxKeys; const all: IT.ListObject[] = []; do { const batchResult = await this._fetchObjectBatch(keyPath, prefix, remaining, token, opts); if (batchResult === null) { return null; // 404 - bucket not found } all.push(...batchResult.objects); remaining -= batchResult.objects.length; token = batchResult.continuationToken; } while (token && remaining > 0); return { objects: all, nextContinuationToken: token }; } /** * Lists all versions (and delete markers) of a specific object key. * Auto-paginates until every version is returned (or maxKeys is reached). * Entries include `VersionId`, `IsLatest`, and optionally `IsDeleteMarker`. * * @param {string} key - Exact object key whose versions to list. * @param {number} [maxKeys] - Optional cap on how many version entries to return. * @returns {Promise<IT.ListObject[] | null>} All versions for the key, or null if the bucket is not found. * @example * const versions = await s3.listObjectVersions('file.jpg'); * const latest = versions?.find(v => v.IsLatest); * const older = versions?.filter(v => !v.IsLatest && !v.IsDeleteMarker); */ public async listObjectVersions(key: string, maxKeys?: number): Promise<IT.ListObject[] | null> { this._checkKey(key); // Narrow server-side with prefix=key, then filter exact Key match (prefix is a string prefix). const listed = await this.listObjects('/', key, maxKeys, { versions: true }); if (listed === null) { return null; } return listed.filter(obj => obj.Key === key); } private async _fetchObjectBatch( keyPath: string, prefix: string, remaining: number, token: string | undefined, opts: Record<string, unknown>, ): Promise<{ objects: IT.ListObject[]; continuationToken?: string } | null> { const query = this._buildListObjectsQuery(prefix, remaining, token, opts); const res = await this._signedRequest('GET', keyPath, { query, withQuery: true, tolerated: [200, 404], }); if (res.status === 404) { void res.body?.cancel(); return null; } if (res.status !== 200) { await this._handleListObjectsError(res); } const xmlText = await res.text(); return this._parseListObjectsResponse(xmlText, this._isVersionsMode(opts)); } private _isVersionsMode(opts: Record<string, unknown>): boolean { const v = opts.versions; return v === true || v === '' || v === 'true' || v === 1; } private _encodeVersionListToken(keyMarker: string, versionIdMarker: string): string { return JSON.stringify({ k: keyMarker, v: versionIdMarker }); } private _decodeVersionListToken(token: string): { keyMarker: string; versionIdMarker: string } { try { const parsed = JSON.parse(token) as { k?: unknown; v?: unknown }; if (parsed && typeof parsed.k === 'string') { return { keyMarker: parsed.k, versionIdMarker: typeof parsed.v === 'string' ? parsed.v : '', }; } } catch { // fall through — treat raw token as key-marker only } return { keyMarker: token, versionIdMarker: '' }; } private _buildListObjectsQuery( prefix: string, remaining: number, token: string | undefined, opts: Record<string, unknown>, ): Record<string, unknown> { const batchSize = Math.min(remaining, 1000); // S3 ceiling const versionsMode = this._isVersionsMode(opts); // Do not forward control key that we map ourselves const restOpts: Record<string, unknown> = { ...opts }; delete restOpts.versions; if (versionsMode) { const markers = token ? this._decodeVersionListToken(token) : undefined; return { versions: '', 'max-keys': String(batchSize), ...(prefix ? { prefix } : {}), ...(markers ? { 'key-marker': markers.keyMarker, 'version-id-marker': markers.versionIdMarker, } : {}), ...restOpts, }; } return { 'list-type': C.LIST_TYPE, // =2 for V2 'max-keys': String(batchSize), ...(prefix ? { prefix } : {}), ...(token ? { 'continuation-token': token } : {}), ...restOpts, }; } private async _handleListObjectsError(res: Response): Promise<never> { const errorBody = await res.text(); const parsedErrorBody = this._parseErrorXml(res.headers, errorBody); const errorCode = res.headers.get('x-amz-error-code') ?? parsedErrorBody.svcCode ?? 'Unknown'; const errorMessage = res.headers.get('x-amz-error-message') ?? parsedErrorBody.errorMessage ?? res.statusText; this._log( 'error', `${C.ERROR_PREFIX}Request failed with status ${res.status}: ${errorCode} - ${errorMessage}, err body: ${errorBody}`, ); throw new Error( `${C.ERROR_PREFIX}Request failed with status ${res.status}: ${errorCode} - ${errorMessage}, err body: ${errorBody}`, ); } private _parseListObjectsResponse( xmlText: string, versionsMode = false, ): { objects: IT.ListObject[]; continuationToken?: string; } { const raw = parseXml(xmlText) as Record<string, unknown>; if (typeof raw !== 'object' || !raw || 'error' in raw) { this._log('error', `${C.ERROR_PREFIX}Unexpected listObjects response shape: ${JSON.stringify(raw)}`); throw new Error(`${C.ERROR_PREFIX}Unexpected listObjects response shape`); } const out = (raw.ListVersionsResult || raw.listVersionsResult || raw.ListBucketResult || raw.listBucketResult || raw) as Record<string, unknown>; const objects = this._extractObjectsFromResponse(out); const continuationToken = versionsMode ? this._extractVersionListToken(out) : this._extractContinuationToken(out); return { objects, continuationToken }; } private _mapListEntry(item: Record<string, unknown>, isDeleteMarker = false): IT.ListObject { const keyRaw = item.Key ?? item.key ?? ''; const key = typeof keyRaw === 'string' ? keyRaw : ''; const versionId = item.VersionId ?? item.versionId; const isLatestRaw = item.IsLatest ?? item.isLatest; const etagRaw = item.ETag ?? item.etag ?? item.eTag ?? ''; const storageRaw = item.StorageClass ?? item.storageClass ?? ''; const lmRaw = item.LastModified ?? item.lastModified ?? 0; const entry: IT.ListObject = { Key: key, Size: Number(item.Size ?? item.size ?? 0), LastModified: new Date(typeof lmRaw === 'string' || typeof lmRaw === 'number' ? lmRaw : 0), ETag: typeof etagRaw === 'string' ? etagRaw : '', StorageClass: typeof storageRaw === 'string' ? storageRaw : '', }; if (typeof versionId === 'string' && versionId !== '') { entry.VersionId = versionId; } if (isLatestRaw !== undefined && isLatestRaw !== null && isLatestRaw !== '') { entry.IsLatest = isLatestRaw === true || isLatestRaw === 'true'; } if (isDeleteMarker) { entry.IsDeleteMarker = true; } return entry; } private _pushListEntries(raw: unknown, isDeleteMarker: boolean, out: IT.ListObject[]): void { if (!raw) { return; } for (const item of this._asArray(raw)) { out.push(this._mapListEntry(item as Record<string, unknown>, isDeleteMarker)); } } private _pushCommonPrefixes(raw: unknown, out: IT.ListObject[]): void { if (!raw) { return; } for (const item of this._asArray(raw)) { const entry = item as Record<string, unknown>; const prefix = entry.Prefix || entry.prefix; if (typeof prefix === 'string') { out.push({ Key: prefix, Size: 0, LastModified: new Date(0), ETag: '', StorageClass: '' }); } } } private _extractObjectsFromResponse(response: Record<string, unknown>): IT.ListObject[] { const objects: IT.ListObject[] = []; this._pushListEntries(response.Contents || response.contents, false, objects); this._pushListEntries(response.Version || response.version, false, objects); this._pushListEntries(response.DeleteMarker || response.deleteMarker, true, objects); this._pushCommonPrefixes(response.CommonPrefixes || response.commonPrefixes, objects); return objects; } private _extractContinuationToken(response: Record<string, unknown>): string | undefined { const truncated = response.IsTruncated === 'true' || response.isTruncated === 'true' || false; if (!truncated) { return undefined; } return (response.NextContinuationToken || response.nextContinuationToken || response.NextMarker || response.nextMarker) as string | undefined; } private _extractVersionListToken(response: Record<string, unknown>): string | undefined { const truncated = response.IsTruncated === 'true' || response.isTruncated === 'true' || false; if (!truncated) { return undefined; } const keyMarker = (response.NextKeyMarker ?? response.nextKeyMarker ?? '') as string; const versionIdMarker = (response.NextVersionIdMarker ?? response.nextVersionIdMarker ?? '') as string; // Always encode when truncated so the next request can resume correctly return this._encodeVersionListToken(String(keyMarker), String(versionIdMarker)); } private async _bunListAll( prefix: string, maxKeys: number | undefined, delimiter: string | undefined, ): Promise<IT.ListObject[] | null> { const unlimited = !(maxKeys && maxKeys > 0); let remaining = unlimited ? Infinity : maxKeys; let token: string | undefined; const all: IT.ListObject[] = []; try { do { const batchSize = Math.min(remaining === Infinity ? 1000 : remaining, 1000); const res = await this._bunFetchPage(prefix, delimiter, batchSize, token); const mapped = this._bunMapListResult(res); const prev = token; token = res.nextContinuationToken; all.push(...mapped); if (!unlimited) { remaining -= mapped.length; } // Only a page we still need to follow can stall: a missing or repeated token means the // next request would either be skipped or replay this one forever. if (res.isTruncated && remaining > 0 && (!token || token === prev)) { throw new Error(C.ERROR_BUN_PAGINATION_STALLED); } } while (token && remaining > 0); } catch (e) { // _fetchObjectBatch tolerates 404 whatever the provider calls it, so a bucket addressed // through a path the provider reads as a key (NoSuchKey) must land on null here too. if (this._isBunNotFound(e)) { return null; } throw this._bunError(e); } return all; } private _bunFetchPage( prefix: string, delimiter: string | undefined, maxKeys: number, continuationToken?: string, ): Promise<IT.NativeS3ListResult> { return this._bun!.list({ prefix: prefix || undefined, // No delimiter means a flat listing, matching the signed-request path. delimiter, maxKeys, ...(continuationToken ? { continuationToken } : {}), }); } private _bunMapListResult(res: IT.NativeS3ListResult): IT.ListObject[] { const objects: IT.ListObject[] = []; if (res.contents) { for (const item of res.contents) { objects.push({ Key: item.key, Size: item.size, LastModified: item.lastModified instanceof Date ? item.lastModified : new Date(item.lastModified), ETag: item.eTag ?? '', StorageClass: item.storageClass ?? '', }); } } if (res.commonPrefixes) { for (const item of res.commonPrefixes) { objects.push({ Key: item.prefix, Size: 0, LastModified: new Date(0), ETag: '', StorageClass: '', }); } } return objects; } /** * Lists multipart uploads in the bucket. * This method sends a request to list multipart uploads in the specified bucket. * @param {string} [delimiter='/'] - The delimiter to use for grouping uploads. * @param {string} [prefix=''] - The prefix to filter uploads by. * @param {IT.HttpMethod} [method='GET'] - The HTTP method to use for the request (GET or HEAD). * @param {Record<string, string | number | boolean | undefined>} [opts={}] - Additional options for the request. * @returns A promise that resolves to a list of multipart uploads or an error. */ public async listMultipartUploads( delimiter: string = '/', prefix: string = '', method: IT.HttpMethod = 'GET', opts: Record<string, string | number | boolean | undefined> = {}, ): Promise<IT.ListMultipartUploadSuccess | IT.MultipartUploadError> { this._checkDelimiter(delimiter); this._checkPrefix(prefix); this._validateMethodIsGetOrHead(method); this._checkOpts(opts); const query = { uploads: '', ...opts }; const keyPath = delimiter === '/' ? delimiter : uriEscape(delimiter); const res = await this._signedRequest(method, keyPath, { query, withQuery: true, }); // doublecheck if this is needed // if (method === 'HEAD') { // return { // size: +(res.headers.get(C.HEADER_CONTENT_LENGTH) ?? '0'), // mtime: res.headers.get(C.HEADER_LAST_MODIFIED) ? new Date(res.headers.get(C.HEADER_LAST_MODIFIED)!) : undefined, // etag: res.headers.get(C.HEADER_ETAG) ?? '', // }; // } const raw = parseXml(await res.text()) as unknown; if (typeof raw !== 'object' || raw === null) { throw new Error(`${C.ERROR_PREFIX}Unexpected listMultipartUploads response shape`); } if ('listMultipartUploadsResult' in raw) { return raw.listMultipartUploadsResult as IT.ListMultipartUploadSuccess; } return raw as IT.MultipartUploadError; } /** * Get an object from the S3-compatible service. * This method sends a request to retrieve the specified object from the S3-compatible service. * @param {string} key - The key of the object to retrieve. * @param {Record<string, unknown>} [opts] - Additional options for the request. * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any. * @returns A promise that resolves to the object data (string) or null if not found. */ public async getObject( key: string, opts: Record<string, unknown> = {}, ssecHeaders?: IT.SSECHeaders, ): Promise<string | null> { if (this._bun && !ssecHeaders && !Object.keys(opts).length) { return this._bunRead(key, f => f.text()); } const res = await this._signedRequest('GET', key, { query: opts, // use opts.query if it exists, otherwise use an empty object tolerated: [200, 404, 412, 304], headers: ssecHeaders ? { ...ssecHeaders } : undefined, }); const s = res.status; if (s === 200) { return res.text(); } void res.body?.cancel(); return null; } /** * Get an object response from the S3-compatible service. * This method sends a request to retrieve the specified object and returns the full response. * @param {string} key - The key of the object to retrieve. * @param {Record<string, unknown>} [opts={}] - Additional options for the request. * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any. * @returns A promise that resolves to the Response object or null if not found. */ public async getObjectResponse( key: string, opts: Record<string, unknown> = {}, ssecHeaders?: IT.SSECHeaders, ): Promise<Response | null> { const res = await this._signedRequest('GET', key, { query: opts, tolerated: [200, 404, 412, 304], headers: ssecHeaders ? { ...ssecHeaders } : undefined, }); if (res.status === 200) { return res; } void res.body?.cancel(); return null; } /** * Get an object as an ArrayBuffer from the S3-compatible service. * This method sends a request to retrieve the specified object and returns it as an ArrayBuffer. * @param {string} key - The key of the object to retrieve. * @param {Record<string, unknown>} [opts={}] - Additional options for the request. * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any. * @returns A promise that resolves to the object data as an ArrayBuffer or null if not found. */ public async getObjectArrayBuffer( key: string, opts: Record<string, unknown> = {}, ssecHeaders?: IT.SSECHeaders, ): Promise<ArrayBuffer | null> { if (this._bun && !ssecHeaders && !Object.keys(opts).length) { return this._bunRead(key, f => f.arrayBuffer()); } const res = await this._signedRequest('GET', key, { query: opts, tolerated: [200, 404, 412, 304], headers: ssecHeaders ? { ...ssecHeaders } : undefined, }); if (res.status === 200) { return res.arrayBuffer(); } void res.body?.cancel(); return null; } /** * Get an object as JSON from the S3-compatible service. * This method sends a request to retrieve the specified object and returns it as JSON. * @param {string} key - The key of the object to retrieve. * @param {Record<string, unknown>} [opts={}] - Additional options for the request. * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any. * @returns A promise that resolves to the object data as JSON or null if not found. */ public async getObjectJSON<T = unknown>( key: string, opts: Record<string, unknown> = {}, ssecHeaders?: IT.SSECHeaders, ): Promise<T | null> { if (this._bun && !ssecHeaders && !Object.keys(opts).length) { return this._bunRead(key, f => f.json()) as Promise<T | null>; } const res = await this._signedRequest('GET', key, { query: opts, tolerated: [200, 404, 412, 304], headers: ssecHeaders ? { ...ssecHeaders } : undefined, }); if (res.status === 200) { return res.json() as Promise<T>; } void res.body?.cancel(); return null; } /** * Get an object with its ETag from the S3-compatible service. * This method sends a request to retrieve the specified object and its ETag. * @param {string} key - The key of the object to retrieve. * @param {Record<string, unknown>} [opts={}] - Additional options for the request. * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any. * @returns A promise that resolves to an object containing the ETag and the object data as an ArrayBuffer or null if not found. */ public async getObjectWithETag( key: string, opts: Record<string, unknown> = {}, ssecHeaders?: IT.SSECHeaders, ): Promise<{ etag: string | null; data: ArrayBuffer | null }> { try { const res = await this._signedRequest('GET', key, { query: opts, tolerated: [200, 404, 412, 304], headers: ssecHeaders ? { ...ssecHeaders } : undefined, }); const s = res.status; if (s === 404 || s === 412 || s === 304) { void res.body?.cancel(); return { etag: null, data: null }; } const etag = res.headers.get(C.HEADER_ETAG); if (!etag) { throw new Error(`${C.ERROR_PREFIX}ETag not found in response headers`); } return { etag: sanitizeETag(etag), data: await res.arrayBuffer() }; } catch (err) { this._log('error', `Error getting object ${key} with ETag: ${String(err)}`); throw err; } } /** * Get an object as a raw response from the S3-compatible service. * This method sends a request to retrieve the specified object and returns the raw response. * @param {string} key - The key of the object to retrieve. * @param {boolean} [wholeFile=true] - Whether to retrieve the whole file or a range. * @param {number} [rangeFrom=0] - The starting byte for the range (if not whole file). * @param {number} [rangeTo=this.requestSizeInBytes] - The ending byte for the range (if not whole file). * @param {Record<string, unknown>} [opts={}] - Additional options for the request. * @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any. * @returns A promise that resolves to the Response object. */ public async getObjectRaw( key: string, wholeFile = true, rangeFrom = 0, rangeTo?: number, opts: Record<string, unknown> = {}, ssecHeaders?: IT.SSECHeaders, ): Promise<Response> { let rangeHdr: Record<string, string | number> = {}; if (!wholeFile) { rangeHdr = rangeTo === undefined ? { range: `bytes=${rangeFrom}-` } : { range: `bytes=${rangeFrom}-${rangeTo - 1}` }; } return this._signedRequest('GET', key, { query: { ...opts }, headers: { ...rangeHdr, ...ssecHeaders }, withQuery: true, // keep ?query=string behaviour }); } /** * Get the content length of an object. * This method sends a HEAD request to retrieve the content length of the specified object. * @param {string} key - The key of the object to retrieve the content length for. * @returns A promise that resolves to the content length of the object in bytes; 0 when the object exists but the response carries no content-length header. * @throws {Error} If the object does not exist (HTTP 404) or the request otherwise fails; the underlying S3ServiceError is attached as `.cause`. */ public async getContentLength(key: string, ssecHeaders?: IT.SSECHeaders): Promise<number> { try { if (this._bun && !ssecHeaders) { try { return (await this._bun.file(key).stat()).size; } catch (e) { throw this._bunError(e); } } const res = await this._signedRequest('HEAD', key, { headers: ssecHeaders ? { ...ssecHeaders } : undefined, }); const len = res.headers.get(C.HEADER_CONTENT_LENGTH); return len ? +len : 0; } catch (err) { this._log('error', `Error getting content length for object ${key}: ${String(err)}`); throw new Error(`${C.ERROR_PREFIX}Error getting content length for object ${key}: ${String(err)}`, { cause: err, }); } } /** * Checks if an object exists in the S3-compatible service. * This method sends a HEAD request to check if the specified object exists. * @param {string} key - The key of the object to check. * @param {Record<string, unknown>} [opts={}] - Additional options for the request. * @returns A promise that resolves to true if the object exists, false if not found, or null if ETag mismatch. */ public async objectExists(key: string, opts: Record<string, unknown> = {}): Promise<IT.ExistResponseCode> { if (this._bun && !Object.keys(opts).length) { try { return await this._bun.file(key).exists(); } catch (e) { throw this._bunError(e); } } const res = await this._signedRequest('HEAD', key, { query: opts, tolerated: [200, 404, 412, 304], }); if (res.status === 404) { return false; // not found } if (res.status === 412 || res.status === 304) { return null; // ETag mismatch } return true; // found (200) } /** * Retrie