s3mini
Version:
👶 Tiny & fast S3 client for node and edge computing platforms
1,297 lines (1,178 loc) • 97.8 kB
text/typescript
'use strict';
import * as C from './consts.js';
import {
hexFromBuffer,
sha256,
hmac,
uriResourceEscape,
getByteSize,
sanitizeETag,
uriEscape,
parseXml,
escapeXml,
base64FromBuffer,
extractErrCode,
S3NetworkError,
S3ServiceError,
generateParts,
toUint8Array,
isBun,
byCodePoint,
} from './utils.js';
import type * as IT from './types.js';
/**
* S3 class for interacting with S3-compatible object storage services.
* This class provides methods for common S3 operations such as uploading, downloading,
* and deleting objects, as well as multipart uploads.
*
* @class
* @example
* const s3 = new S3mini({
* accessKeyId: 'your-access-key',
* secretAccessKey: 'your-secret-key',
* endpoint: 'https://your-s3-endpoint.com/bucket-name',
* region: 'auto' // by default is auto
* });
*
* // Upload a file
* await s3.putObject('example.txt', 'Hello, World!');
*
* // Download a file
* const content = await s3.getObject('example.txt');
*
* // Delete a file
* await s3.deleteObject('example.txt');
*/
class S3mini {
/**
* Creates an instance of the S3 class.
*
* @constructor
* @param {Object} config - Configuration options for the S3 instance.
* @param {string} config.accessKeyId - The access key ID for authentication.
* @param {string} config.secretAccessKey - The secret access key for authentication.
* @param {string} config.endpoint - The endpoint URL of the S3-compatible service.
* @param {string} [config.region='auto'] - The region of the S3 service.
* @param {number} [config.requestSizeInBytes=8388608] - The request size of a single request in bytes (AWS S3 is 8MB).
* @param {number} [config.requestAbortTimeout=undefined] - The timeout in milliseconds after which a request should be aborted (careful on streamed requests).
* @param {Object} [config.logger=null] - A logger object with methods like info, warn, error.
* @param {typeof fetch} [config.fetch=globalThis.fetch] - Custom fetch implementation to use for HTTP requests.
* @param {number} [config.minPartSize=8388608] - The minimum part size for multipart uploads in bytes (default is 8MB).
* @throws {TypeError} Will throw an error if required parameters are missing or of incorrect type.
*/
readonly #accessKeyId: string;
readonly #secretAccessKey: string;
readonly endpoint: URL;
readonly region: string;
readonly bucketName: string;
readonly requestSizeInBytes: number;
readonly requestAbortTimeout?: number;
readonly logger?: IT.Logger;
readonly _fetch: typeof fetch;
readonly minPartSize: number;
private readonly _bun?: IT.NativeS3Client;
private signingKeyDate?: string;
private signingKey?: ArrayBuffer;
constructor({
accessKeyId,
secretAccessKey,
endpoint,
region = 'auto',
requestSizeInBytes = C.DEFAULT_REQUEST_SIZE_IN_BYTES,
requestAbortTimeout = undefined,
logger = undefined,
fetch = globalThis.fetch,
minPartSize = C.MIN_PART_SIZE,
}: IT.S3Config) {
this._validateConstructorParams(accessKeyId, secretAccessKey, endpoint);
this.#accessKeyId = accessKeyId;
this.#secretAccessKey = secretAccessKey;
this.endpoint = new URL(this._ensureValidUrl(endpoint));
this.region = region;
this.bucketName = this._extractBucketName();
this.requestSizeInBytes = requestSizeInBytes;
this.requestAbortTimeout = requestAbortTimeout;
this.logger = logger;
this._fetch = (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => fetch(input, init);
this.minPartSize = minPartSize;
// Bun's native client has its own transport, so a caller-supplied fetch would be silently
// bypassed: only take the native path when the default fetch is in use. It also refuses empty
// credentials, which anonymous access to public buckets relies on.
if (isBun && fetch === globalThis.fetch && this._hasCredentials()) {
// Bun's client is an origin plus a bucket name, so an endpoint carrying anything past the
// bucket (host/bucket/prefix) cannot be expressed: the extra segments would be dropped and
// every native request would silently land in the parent bucket, while the signed path stays
// inside the prefix. Decline the native path rather than read and write the wrong location.
const segments = this.endpoint.pathname.split('/').filter(Boolean);
if (segments.length < 2) {
const { S3Client } = (
globalThis as unknown as { Bun: { S3Client: new (o: Record<string, unknown>) => IT.NativeS3Client } }
).Bun;
this._bun = new S3Client({
accessKeyId,
secretAccessKey,
endpoint: this.endpoint.origin,
region: this.region,
bucket: this.bucketName,
// Bucket in the path means path-style; otherwise it is in the host and Bun has to be
// told, or it would repeat the bucket in the path (bucket.host/bucket/key).
virtualHostedStyle: segments.length === 0,
});
}
}
}
private _sanitize(obj: unknown): unknown {
if (typeof obj !== 'object' || obj === null) {
return obj;
}
return Object.keys(obj).reduce(
(acc: Record<string, unknown>, key) => {
if (C.SENSITIVE_KEYS_REDACTED.has(key.toLowerCase())) {
acc[key] = '[REDACTED]';
} else if (
typeof (obj as Record<string, unknown>)[key] === 'object' &&
(obj as Record<string, unknown>)[key] !== null
) {
acc[key] = this._sanitize((obj as Record<string, unknown>)[key]);
} else {
acc[key] = (obj as Record<string, unknown>)[key];
}
return acc;
},
Array.isArray(obj) ? [] : {},
);
}
private _log(
level: 'info' | 'warn' | 'error',
message: string,
additionalData: Record<string, unknown> | string = {},
): void {
if (this.logger && typeof this.logger[level] === 'function') {
// Function to recursively sanitize an object
// Sanitize the additional data
const sanitizedData = this._sanitize(additionalData);
// Prepare the log entry
const logEntry = {
timestamp: new Date().toISOString(),
level,
message,
details: sanitizedData,
// Include some general context, but sanitize sensitive parts
context: this._sanitize({
region: this.region,
endpoint: this.endpoint.toString(),
// Only include the first few characters of the access key, if it exists
accessKeyId: this.#accessKeyId ? `${this.#accessKeyId.substring(0, 4)}...` : undefined,
}),
};
// Log the sanitized entry
this.logger[level](JSON.stringify(logEntry));
}
}
// S3 returns repeated elements as either an array or a single scalar object
// (e.g. a lone <Contents> is not wrapped in a 1-element array). Normalize to array.
private _asArray(value: unknown): unknown[] {
return Array.isArray(value) ? value : [value];
}
private _validateConstructorParams(accessKeyId: string, secretAccessKey: string, endpoint: string): void {
if (typeof accessKeyId !== 'string') {
throw new TypeError(C.ERROR_ACCESS_KEY_REQUIRED);
}
if (typeof secretAccessKey !== 'string') {
throw new TypeError(C.ERROR_SECRET_KEY_REQUIRED);
}
if (typeof endpoint !== 'string' || endpoint.trim().length === 0) {
throw new TypeError(C.ERROR_ENDPOINT_REQUIRED);
}
}
/**
* Check if credentials are configured (non-empty).
* @returns true if both accessKeyId and secretAccessKey are non-empty.
*/
private _hasCredentials(): boolean {
return this.#accessKeyId.trim().length > 0 && this.#secretAccessKey.trim().length > 0;
}
/**
* Re-shape a Bun S3Error as the S3ServiceError the signed path throws, so callers see one error
* type on every runtime. Bun does not expose the HTTP status, so it is recovered from the error
* code where the S3 API pins it and left as 0 (unknown) otherwise.
*/
private _bunError(e: unknown): unknown {
const err = e as { name?: string; code?: string; message?: string };
if (err?.name !== 'S3Error') {
return e;
}
const status = err.code ? (C.S3_CODE_STATUS[err.code] ?? 0) : 0;
const message = status ? `S3 returned ${status} – ${err.code}` : (err.message ?? String(e));
// The provider's wording goes where the signed path puts the error body.
return new S3ServiceError(message, status, err.code, err.message);
}
/** True for a Bun S3Error the signed path would have absorbed as a tolerated 404. */
private _isBunNotFound(e: unknown): boolean {
const code = (e as { code?: string })?.code;
return !!code && C.S3_CODE_STATUS[code] === 404;
}
/** Run a read op via Bun-native S3, returning null when the object or its bucket is absent. */
private async _bunRead<T>(key: string, op: (f: IT.NativeS3File) => Promise<T>): Promise<T | null> {
try {
return await op(this._bun!.file(key));
} catch (e) {
// Every signed reader tolerates 404 and answers null, so match on the status the code maps
// to rather than on NoSuchKey alone: a missing *bucket* is equally a 404, and singling out
// the key left it throwing here while returning null on the signed path.
if (this._isBunNotFound(e)) {
return null;
}
throw this._bunError(e);
}
}
private _ensureValidUrl(raw: string): string {
const candidate = /^(https?:)?\/\//i.test(raw) ? raw : `https://${raw}`;
try {
new URL(candidate);
// Find the last non-slash character
let endIndex = candidate.length;
while (endIndex > 0 && candidate[endIndex - 1] === '/') {
endIndex--;
}
return endIndex === candidate.length ? candidate : candidate.substring(0, endIndex);
} catch {
const msg = `${C.ERROR_ENDPOINT_FORMAT} But provided: "${raw}"`;
this._log('error', msg);
throw new TypeError(msg);
}
}
private _validateMethodIsGetOrHead(method: string): void {
if (method !== 'GET' && method !== 'HEAD') {
this._log('error', `${C.ERROR_PREFIX}method must be either GET or HEAD`);
throw new Error(`${C.ERROR_PREFIX}method must be either GET or HEAD`);
}
}
private _checkKey(key: string): void {
if (typeof key !== 'string' || key.trim().length === 0) {
this._log('error', C.ERROR_KEY_REQUIRED);
throw new TypeError(C.ERROR_KEY_REQUIRED);
}
}
private _checkDelimiter(delimiter: string): void {
if (typeof delimiter !== 'string' || delimiter.trim().length === 0) {
this._log('error', C.ERROR_DELIMITER_REQUIRED);
throw new TypeError(C.ERROR_DELIMITER_REQUIRED);
}
}
private _checkPrefix(prefix: string): void {
if (typeof prefix !== 'string') {
this._log('error', C.ERROR_PREFIX_TYPE);
throw new TypeError(C.ERROR_PREFIX_TYPE);
}
}
// private _checkMaxKeys(maxKeys: number): void {
// if (typeof maxKeys !== 'number' || maxKeys <= 0) {
// this._log('error', C.ERROR_MAX_KEYS_TYPE);
// throw new TypeError(C.ERROR_MAX_KEYS_TYPE);
// }
// }
private _checkOpts(opts: object): void {
if (typeof opts !== 'object') {
this._log('error', `${C.ERROR_PREFIX}opts must be an object`);
throw new TypeError(`${C.ERROR_PREFIX}opts must be an object`);
}
}
private _filterIfHeaders(opts: Record<string, unknown>): {
filteredOpts: Record<string, string>;
conditionalHeaders: Record<string, unknown>;
} {
const filteredOpts: Record<string, string> = {};
const conditionalHeaders: Record<string, unknown> = {};
for (const [key, value] of Object.entries(opts)) {
if (C.IFHEADERS.has(key.toLowerCase())) {
conditionalHeaders[key] = value;
} else {
filteredOpts[key] = value as string;
}
}
return { filteredOpts, conditionalHeaders };
}
// private _validateData(data: unknown): BodyInit {
// if (data instanceof ArrayBuffer) {
// return data;
// }
// if (data instanceof Uint8Array) {
// return data as unknown as BodyInit;
// }
// if ((globalThis.Buffer && data instanceof globalThis.Buffer) || typeof data === 'string') {
// return data as BodyInit;
// }
// this._log('error', C.ERROR_DATA_BUFFER_REQUIRED);
// throw new TypeError(C.ERROR_DATA_BUFFER_REQUIRED);
// }
private _validateUploadPartParams(
key: string,
uploadId: string,
data: IT.DataInput,
partNumber: number,
opts: object,
): BodyInit {
this._checkKey(key);
if (typeof uploadId !== 'string' || uploadId.trim().length === 0) {
this._log('error', C.ERROR_UPLOAD_ID_REQUIRED);
throw new TypeError(C.ERROR_UPLOAD_ID_REQUIRED);
}
if (!Number.isInteger(partNumber) || partNumber <= 0) {
this._log('error', `${C.ERROR_PREFIX}partNumber must be a positive integer`);
throw new TypeError(`${C.ERROR_PREFIX}partNumber must be a positive integer`);
}
this._checkOpts(opts);
return data as BodyInit;
}
private async _sign(
method: IT.HttpMethod,
keyPath: string,
query: Record<string, unknown> = {},
headers: Record<string, string | number> = {},
): Promise<{ url: string; headers: Record<string, string | number> }> {
// Create URL without appending keyPath first
const url = new URL(this.endpoint);
// Properly format the pathname to avoid double slashes
if (keyPath && keyPath.length > 0) {
url.pathname =
url.pathname === '/' ? `/${keyPath.replace(/^\/+/, '')}` : `${url.pathname}/${keyPath.replace(/^\/+/, '')}`;
}
// If no credentials, return unsigned request (for public bucket access)
if (!this._hasCredentials()) {
headers[C.HEADER_HOST] = url.host;
return { url: url.toString(), headers };
}
const d = new Date();
const year = d.getUTCFullYear();
const month = String(d.getUTCMonth() + 1).padStart(2, '0');
const day = String(d.getUTCDate()).padStart(2, '0');
const shortDatetime = `${year}${month}${day}`;
const fullDatetime = `${shortDatetime}T${String(d.getUTCHours()).padStart(2, '0')}${String(d.getUTCMinutes()).padStart(2, '0')}${String(d.getUTCSeconds()).padStart(2, '0')}Z`;
const credentialScope = `${shortDatetime}/${this.region}/${C.S3_SERVICE}/${C.AWS_REQUEST_TYPE}`;
headers[C.HEADER_AMZ_CONTENT_SHA256] = C.UNSIGNED_PAYLOAD;
headers[C.HEADER_AMZ_DATE] = fullDatetime;
headers[C.HEADER_HOST] = url.host;
const ignoredHeaders = new Set(['authorization', 'content-length', 'content-type', 'user-agent']);
const sortedHeaders = Object.entries(headers)
.map(([key, value]): [string, string] => [key.toLowerCase(), String(value).trim()])
.filter(([lowerKey]) => !ignoredHeaders.has(lowerKey))
.sort(([a], [b]) => byCodePoint(a, b));
const canonicalHeaders = sortedHeaders.map(([k, v]) => `${k}:${v}`).join('\n');
const signedHeaders = sortedHeaders.map(([k]) => k).join(';');
const canonicalRequest = `${method}\n${url.pathname}\n${this._buildCanonicalQueryString(query)}\n${canonicalHeaders}\n\n${signedHeaders}\n${C.UNSIGNED_PAYLOAD}`;
const stringToSign = `${C.AWS_ALGORITHM}\n${fullDatetime}\n${credentialScope}\n${hexFromBuffer(await sha256(canonicalRequest))}`;
if (shortDatetime !== this.signingKeyDate || !this.signingKey) {
this.signingKeyDate = shortDatetime;
this.signingKey = await this._getSignatureKey(shortDatetime);
}
const signature = hexFromBuffer(await hmac(this.signingKey, stringToSign));
headers[C.HEADER_AUTHORIZATION] =
`${C.AWS_ALGORITHM} Credential=${this.#accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
return { url: url.toString(), headers };
}
private async _signedRequest(
method: IT.HttpMethod, // 'GET' | 'HEAD' | 'PUT' | 'POST' | 'DELETE'
key: string, // ‘’ allowed for bucket‑level ops
{
query = {}, // ?query=string
body = '', // BodyInit | undefined
headers = {}, // extra/override headers
tolerated = [], // [200, 404] etc.
withQuery = false, // append query string to signed URL
}: {
query?: Record<string, unknown>;
body?: BodyInit;
headers?: Record<string, string | number | undefined> | IT.SSECHeaders | IT.AWSHeaders;
tolerated?: number[];
withQuery?: boolean;
} = {},
): Promise<Response> {
// Basic validation
// if (!['GET', 'HEAD', 'PUT', 'POST', 'DELETE'].includes(method)) {
// throw new Error(`${C.ERROR_PREFIX}Unsupported HTTP method ${method as string}`);
// }
const { filteredOpts, conditionalHeaders } = ['GET', 'HEAD'].includes(method)
? this._filterIfHeaders(query)
: { filteredOpts: query, conditionalHeaders: {} };
const baseHeaders: Record<string, string | number> = {
[C.HEADER_AMZ_CONTENT_SHA256]: C.UNSIGNED_PAYLOAD,
// ...(['GET', 'HEAD'].includes(method) ? { [C.HEADER_CONTENT_TYPE]: C.JSON_CONTENT_TYPE } : {}),
...headers,
...conditionalHeaders,
};
const encodedKey = key ? uriResourceEscape(key) : '';
const { url, headers: signedHeaders } = await this._sign(method, encodedKey, filteredOpts, baseHeaders);
if (Object.keys(query).length > 0) {
withQuery = true; // append query string to signed URL
}
const finalUrl =
withQuery && Object.keys(filteredOpts).length ? `${url}?${this._buildCanonicalQueryString(filteredOpts)}` : url;
const signedHeadersString = Object.fromEntries(
Object.entries(signedHeaders).map(([k, v]) => [k, String(v)]),
) as Record<string, string>;
return this._sendRequest(finalUrl, method, signedHeadersString, body, tolerated);
}
/**
* Sanitizes an ETag value by removing surrounding quotes and whitespace.
* Still returns RFC compliant ETag. https://www.rfc-editor.org/rfc/rfc9110#section-8.8.3
* @param {string} etag - The ETag value to sanitize.
* @returns {string} The sanitized ETag value.
* @example
* const cleanEtag = s3.sanitizeETag('"abc123"'); // Returns: 'abc123'
*/
public sanitizeETag(etag: string): string {
return sanitizeETag(etag);
}
/**
* Creates a new bucket.
* This method sends a request to create a new bucket in the specified in endpoint.
* @returns A promise that resolves to true if the bucket was created successfully, false otherwise.
*/
public async createBucket(): Promise<boolean> {
const xmlBody = `
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<LocationConstraint>${this.region}</LocationConstraint>
</CreateBucketConfiguration>
`;
const headers = {
[C.HEADER_CONTENT_TYPE]: C.XML_CONTENT_TYPE,
[C.HEADER_CONTENT_LENGTH]: getByteSize(xmlBody),
};
const res = await this._signedRequest('PUT', '', {
body: xmlBody,
headers,
tolerated: [200, 404, 403, 409], // don’t throw on 404/403 // 409 = bucket already exists
});
return res.status === 200;
}
private _extractBucketName(): string {
const url = this.endpoint;
// Path-style: bucket is the first non-empty path segment
const firstSegment = url.pathname.split('/').find(Boolean);
if (firstSegment) {
return firstSegment;
}
// Virtual-hosted style: bucket is the first subdomain label
const hostname = url.hostname;
// IP addresses (v4: digits+dots, v6: contains colons) can't carry a bucket subdomain
if (/^\d+\.\d+\.\d+\.\d+$/.test(hostname) || hostname.includes(':')) {
return '';
}
const labels = hostname.split('.');
// Need ≥3 labels for virtual-hosted (bucket.service.tld)
// Single-label (localhost) or two-label (example.com) have no room for a bucket subdomain
if (labels.length < 3) {
return '';
}
return labels[0]!;
}
/**
* Checks if a bucket exists.
* This method sends a request to check if the specified bucket exists in the S3-compatible service.
* @returns A promise that resolves to true if the bucket exists, false otherwise.
*/
public async bucketExists(): Promise<boolean> {
const res = await this._signedRequest('HEAD', '', { tolerated: [200, 404, 403] });
return res.status === 200;
}
/**
* Sets bucket versioning status (PutBucketVersioning).
* Required before object versioning APIs (`listObjectVersions`, versioned delete/copy) are useful.
* @param {'Enabled' | 'Suspended'} status - Versioning status to apply.
* @returns {Promise<boolean>} True when the service accepts the configuration (HTTP 200).
* @example
* await s3.setBucketVersioning('Enabled');
*/
public async setBucketVersioning(status: 'Enabled' | 'Suspended'): Promise<boolean> {
if (status !== 'Enabled' && status !== 'Suspended') {
throw new TypeError(`${C.ERROR_PREFIX}status must be 'Enabled' or 'Suspended'`);
}
const xmlBody =
'<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' +
`<Status>${status}</Status>` +
'</VersioningConfiguration>';
const res = await this._signedRequest('PUT', '', {
query: { versioning: '' },
body: xmlBody,
headers: {
[C.HEADER_CONTENT_TYPE]: C.XML_CONTENT_TYPE,
[C.HEADER_CONTENT_LENGTH]: getByteSize(xmlBody),
},
withQuery: true,
tolerated: [200],
});
return res.status === 200;
}
/**
* Gets bucket versioning status (GetBucketVersioning).
* @returns {Promise<'Enabled' | 'Suspended' | 'Off'>} Current status. `'Off'` when the config is empty/unset.
*/
public async getBucketVersioning(): Promise<'Enabled' | 'Suspended' | 'Off'> {
const res = await this._signedRequest('GET', '', {
query: { versioning: '' },
withQuery: true,
tolerated: [200, 404],
});
if (res.status !== 200) {
void res.body?.cancel();
return 'Off';
}
const raw = parseXml(await res.text()) as Record<string, unknown>;
const cfg = (raw.VersioningConfiguration || raw.versioningConfiguration || raw) as Record<string, unknown>;
const status = cfg.Status ?? cfg.status;
if (status === 'Enabled' || status === 'Suspended') {
return status;
}
return 'Off';
}
/**
* Lists objects in the bucket with optional filtering and no pagination.
* This method retrieves all objects matching the criteria (not paginated like listObjectsV2).
* Pass `{ versions: true }` in opts to list object versions (ListObjectVersions API).
* @param {string} [delimiter='/'] - The delimiter to use for grouping objects.
* @param {string} [prefix=''] - The prefix to filter objects by.
* @param {number} [maxKeys] - The maximum number of keys to return. If not provided, all keys will be returned.
* @param {Record<string, unknown>} [opts={}] - Additional options for the request. Use `{ versions: true }` for version listing.
* @returns {Promise<IT.ListObject[] | null>} A promise that resolves to an array of objects, or null if the bucket does not exist. An empty bucket resolves to an empty array.
* @example
* // List all objects
* const objects = await s3.listObjects();
*
* // List objects with prefix
* const photos = await s3.listObjects('/', 'photos/', 100);
*
* // List object versions (includes VersionId / IsLatest; may include delete markers)
* const versions = await s3.listObjects('/', 'photos/', undefined, { versions: true });
*/
public async listObjects(
delimiter: string = '/',
prefix: string = '',
maxKeys?: number,
opts: Record<string, unknown> = {},
): Promise<IT.ListObject[] | null> {
this._checkDelimiter(delimiter);
this._checkPrefix(prefix);
this._checkOpts(opts);
if (this._bun && delimiter === '/' && !this._isVersionsMode(opts)) {
const extraKeys = Object.keys(opts).filter(k => k !== 'delimiter');
if (extraKeys.length === 0) {
return this._bunListAll(prefix, maxKeys, opts.delimiter as string | undefined);
}
}
const keyPath = delimiter === '/' ? delimiter : uriEscape(delimiter);
const unlimited = !(maxKeys && maxKeys > 0);
let remaining = unlimited ? Infinity : maxKeys;
let token: string | undefined;
const all: IT.ListObject[] = [];
do {
const batchResult = await this._fetchObjectBatch(keyPath, prefix, remaining, token, opts);
if (batchResult === null) {
return null; // 404 - bucket not found
}
all.push(...batchResult.objects);
if (!unlimited) {
remaining -= batchResult.objects.length;
}
token = batchResult.continuationToken;
} while (token && remaining > 0);
return all;
}
/**
* Lists objects in the bucket with optional filtering and pagination using a continuation token.
* This method retrieves objects matching the criteria (paginated like listObjectsV2).
* Pass `{ versions: true }` in opts to list object versions (uses key-marker / version-id-marker under the hood;
* the returned token is opaque and only valid with the same opts).
* @param {string} [delimiter='/'] - The delimiter to use for grouping objects.
* @param {string} [prefix=''] - The prefix to filter objects by.
* @param {number} [maxKeys] - The maximum number of keys to return. Uses a default value of 100.
* @param {string} [nextContinuationToken] - The nextContinuationToken to continue previous results. If not provided, starts from the beginning.
* @param {Record<string, unknown>} [opts={}] - Additional options for the request. Use `{ versions: true }` for version listing.
* @returns {Promise<{objects: IT.ListObject[] | null; nextContinuationToken?: string } | undefined | null>} A promise that resolves to an array of objects, along with nextContinuationToken if there are more reccords, or null if the bucket does not exist.
* @example
* // List all objects
* const { objects, nextContinuationToken } = await s3.listObjectsPaged();
*
* // List 200 objects with prefix
* const photos = await s3.listObjectsPaged('/', 'photos/', 200, "token...");
*/
public async listObjectsPaged(
delimiter: string = '/',
prefix: string = '',
maxKeys: number = 100,
nextContinuationToken?: string,
opts: Record<string, unknown> = {},
): Promise<{ objects: IT.ListObject[] | null; nextContinuationToken?: string } | undefined | null> {
this._checkDelimiter(delimiter);
this._checkPrefix(prefix);
this._checkOpts(opts);
const keyPath = delimiter === '/' ? delimiter : uriEscape(delimiter);
let token: string | undefined = nextContinuationToken;
let remaining = maxKeys;
const all: IT.ListObject[] = [];
do {
const batchResult = await this._fetchObjectBatch(keyPath, prefix, remaining, token, opts);
if (batchResult === null) {
return null; // 404 - bucket not found
}
all.push(...batchResult.objects);
remaining -= batchResult.objects.length;
token = batchResult.continuationToken;
} while (token && remaining > 0);
return { objects: all, nextContinuationToken: token };
}
/**
* Lists all versions (and delete markers) of a specific object key.
* Auto-paginates until every version is returned (or maxKeys is reached).
* Entries include `VersionId`, `IsLatest`, and optionally `IsDeleteMarker`.
*
* @param {string} key - Exact object key whose versions to list.
* @param {number} [maxKeys] - Optional cap on how many version entries to return.
* @returns {Promise<IT.ListObject[] | null>} All versions for the key, or null if the bucket is not found.
* @example
* const versions = await s3.listObjectVersions('file.jpg');
* const latest = versions?.find(v => v.IsLatest);
* const older = versions?.filter(v => !v.IsLatest && !v.IsDeleteMarker);
*/
public async listObjectVersions(key: string, maxKeys?: number): Promise<IT.ListObject[] | null> {
this._checkKey(key);
// Narrow server-side with prefix=key, then filter exact Key match (prefix is a string prefix).
const listed = await this.listObjects('/', key, maxKeys, { versions: true });
if (listed === null) {
return null;
}
return listed.filter(obj => obj.Key === key);
}
private async _fetchObjectBatch(
keyPath: string,
prefix: string,
remaining: number,
token: string | undefined,
opts: Record<string, unknown>,
): Promise<{ objects: IT.ListObject[]; continuationToken?: string } | null> {
const query = this._buildListObjectsQuery(prefix, remaining, token, opts);
const res = await this._signedRequest('GET', keyPath, {
query,
withQuery: true,
tolerated: [200, 404],
});
if (res.status === 404) {
void res.body?.cancel();
return null;
}
if (res.status !== 200) {
await this._handleListObjectsError(res);
}
const xmlText = await res.text();
return this._parseListObjectsResponse(xmlText, this._isVersionsMode(opts));
}
private _isVersionsMode(opts: Record<string, unknown>): boolean {
const v = opts.versions;
return v === true || v === '' || v === 'true' || v === 1;
}
private _encodeVersionListToken(keyMarker: string, versionIdMarker: string): string {
return JSON.stringify({ k: keyMarker, v: versionIdMarker });
}
private _decodeVersionListToken(token: string): { keyMarker: string; versionIdMarker: string } {
try {
const parsed = JSON.parse(token) as { k?: unknown; v?: unknown };
if (parsed && typeof parsed.k === 'string') {
return {
keyMarker: parsed.k,
versionIdMarker: typeof parsed.v === 'string' ? parsed.v : '',
};
}
} catch {
// fall through — treat raw token as key-marker only
}
return { keyMarker: token, versionIdMarker: '' };
}
private _buildListObjectsQuery(
prefix: string,
remaining: number,
token: string | undefined,
opts: Record<string, unknown>,
): Record<string, unknown> {
const batchSize = Math.min(remaining, 1000); // S3 ceiling
const versionsMode = this._isVersionsMode(opts);
// Do not forward control key that we map ourselves
const restOpts: Record<string, unknown> = { ...opts };
delete restOpts.versions;
if (versionsMode) {
const markers = token ? this._decodeVersionListToken(token) : undefined;
return {
versions: '',
'max-keys': String(batchSize),
...(prefix ? { prefix } : {}),
...(markers
? {
'key-marker': markers.keyMarker,
'version-id-marker': markers.versionIdMarker,
}
: {}),
...restOpts,
};
}
return {
'list-type': C.LIST_TYPE, // =2 for V2
'max-keys': String(batchSize),
...(prefix ? { prefix } : {}),
...(token ? { 'continuation-token': token } : {}),
...restOpts,
};
}
private async _handleListObjectsError(res: Response): Promise<never> {
const errorBody = await res.text();
const parsedErrorBody = this._parseErrorXml(res.headers, errorBody);
const errorCode = res.headers.get('x-amz-error-code') ?? parsedErrorBody.svcCode ?? 'Unknown';
const errorMessage = res.headers.get('x-amz-error-message') ?? parsedErrorBody.errorMessage ?? res.statusText;
this._log(
'error',
`${C.ERROR_PREFIX}Request failed with status ${res.status}: ${errorCode} - ${errorMessage}, err body: ${errorBody}`,
);
throw new Error(
`${C.ERROR_PREFIX}Request failed with status ${res.status}: ${errorCode} - ${errorMessage}, err body: ${errorBody}`,
);
}
private _parseListObjectsResponse(
xmlText: string,
versionsMode = false,
): {
objects: IT.ListObject[];
continuationToken?: string;
} {
const raw = parseXml(xmlText) as Record<string, unknown>;
if (typeof raw !== 'object' || !raw || 'error' in raw) {
this._log('error', `${C.ERROR_PREFIX}Unexpected listObjects response shape: ${JSON.stringify(raw)}`);
throw new Error(`${C.ERROR_PREFIX}Unexpected listObjects response shape`);
}
const out = (raw.ListVersionsResult ||
raw.listVersionsResult ||
raw.ListBucketResult ||
raw.listBucketResult ||
raw) as Record<string, unknown>;
const objects = this._extractObjectsFromResponse(out);
const continuationToken = versionsMode
? this._extractVersionListToken(out)
: this._extractContinuationToken(out);
return { objects, continuationToken };
}
private _mapListEntry(item: Record<string, unknown>, isDeleteMarker = false): IT.ListObject {
const keyRaw = item.Key ?? item.key ?? '';
const key = typeof keyRaw === 'string' ? keyRaw : '';
const versionId = item.VersionId ?? item.versionId;
const isLatestRaw = item.IsLatest ?? item.isLatest;
const etagRaw = item.ETag ?? item.etag ?? item.eTag ?? '';
const storageRaw = item.StorageClass ?? item.storageClass ?? '';
const lmRaw = item.LastModified ?? item.lastModified ?? 0;
const entry: IT.ListObject = {
Key: key,
Size: Number(item.Size ?? item.size ?? 0),
LastModified: new Date(typeof lmRaw === 'string' || typeof lmRaw === 'number' ? lmRaw : 0),
ETag: typeof etagRaw === 'string' ? etagRaw : '',
StorageClass: typeof storageRaw === 'string' ? storageRaw : '',
};
if (typeof versionId === 'string' && versionId !== '') {
entry.VersionId = versionId;
}
if (isLatestRaw !== undefined && isLatestRaw !== null && isLatestRaw !== '') {
entry.IsLatest = isLatestRaw === true || isLatestRaw === 'true';
}
if (isDeleteMarker) {
entry.IsDeleteMarker = true;
}
return entry;
}
private _pushListEntries(raw: unknown, isDeleteMarker: boolean, out: IT.ListObject[]): void {
if (!raw) {
return;
}
for (const item of this._asArray(raw)) {
out.push(this._mapListEntry(item as Record<string, unknown>, isDeleteMarker));
}
}
private _pushCommonPrefixes(raw: unknown, out: IT.ListObject[]): void {
if (!raw) {
return;
}
for (const item of this._asArray(raw)) {
const entry = item as Record<string, unknown>;
const prefix = entry.Prefix || entry.prefix;
if (typeof prefix === 'string') {
out.push({ Key: prefix, Size: 0, LastModified: new Date(0), ETag: '', StorageClass: '' });
}
}
}
private _extractObjectsFromResponse(response: Record<string, unknown>): IT.ListObject[] {
const objects: IT.ListObject[] = [];
this._pushListEntries(response.Contents || response.contents, false, objects);
this._pushListEntries(response.Version || response.version, false, objects);
this._pushListEntries(response.DeleteMarker || response.deleteMarker, true, objects);
this._pushCommonPrefixes(response.CommonPrefixes || response.commonPrefixes, objects);
return objects;
}
private _extractContinuationToken(response: Record<string, unknown>): string | undefined {
const truncated = response.IsTruncated === 'true' || response.isTruncated === 'true' || false;
if (!truncated) {
return undefined;
}
return (response.NextContinuationToken ||
response.nextContinuationToken ||
response.NextMarker ||
response.nextMarker) as string | undefined;
}
private _extractVersionListToken(response: Record<string, unknown>): string | undefined {
const truncated = response.IsTruncated === 'true' || response.isTruncated === 'true' || false;
if (!truncated) {
return undefined;
}
const keyMarker = (response.NextKeyMarker ?? response.nextKeyMarker ?? '') as string;
const versionIdMarker = (response.NextVersionIdMarker ?? response.nextVersionIdMarker ?? '') as string;
// Always encode when truncated so the next request can resume correctly
return this._encodeVersionListToken(String(keyMarker), String(versionIdMarker));
}
private async _bunListAll(
prefix: string,
maxKeys: number | undefined,
delimiter: string | undefined,
): Promise<IT.ListObject[] | null> {
const unlimited = !(maxKeys && maxKeys > 0);
let remaining = unlimited ? Infinity : maxKeys;
let token: string | undefined;
const all: IT.ListObject[] = [];
try {
do {
const batchSize = Math.min(remaining === Infinity ? 1000 : remaining, 1000);
const res = await this._bunFetchPage(prefix, delimiter, batchSize, token);
const mapped = this._bunMapListResult(res);
const prev = token;
token = res.nextContinuationToken;
all.push(...mapped);
if (!unlimited) {
remaining -= mapped.length;
}
// Only a page we still need to follow can stall: a missing or repeated token means the
// next request would either be skipped or replay this one forever.
if (res.isTruncated && remaining > 0 && (!token || token === prev)) {
throw new Error(C.ERROR_BUN_PAGINATION_STALLED);
}
} while (token && remaining > 0);
} catch (e) {
// _fetchObjectBatch tolerates 404 whatever the provider calls it, so a bucket addressed
// through a path the provider reads as a key (NoSuchKey) must land on null here too.
if (this._isBunNotFound(e)) {
return null;
}
throw this._bunError(e);
}
return all;
}
private _bunFetchPage(
prefix: string,
delimiter: string | undefined,
maxKeys: number,
continuationToken?: string,
): Promise<IT.NativeS3ListResult> {
return this._bun!.list({
prefix: prefix || undefined,
// No delimiter means a flat listing, matching the signed-request path.
delimiter,
maxKeys,
...(continuationToken ? { continuationToken } : {}),
});
}
private _bunMapListResult(res: IT.NativeS3ListResult): IT.ListObject[] {
const objects: IT.ListObject[] = [];
if (res.contents) {
for (const item of res.contents) {
objects.push({
Key: item.key,
Size: item.size,
LastModified: item.lastModified instanceof Date ? item.lastModified : new Date(item.lastModified),
ETag: item.eTag ?? '',
StorageClass: item.storageClass ?? '',
});
}
}
if (res.commonPrefixes) {
for (const item of res.commonPrefixes) {
objects.push({
Key: item.prefix,
Size: 0,
LastModified: new Date(0),
ETag: '',
StorageClass: '',
});
}
}
return objects;
}
/**
* Lists multipart uploads in the bucket.
* This method sends a request to list multipart uploads in the specified bucket.
* @param {string} [delimiter='/'] - The delimiter to use for grouping uploads.
* @param {string} [prefix=''] - The prefix to filter uploads by.
* @param {IT.HttpMethod} [method='GET'] - The HTTP method to use for the request (GET or HEAD).
* @param {Record<string, string | number | boolean | undefined>} [opts={}] - Additional options for the request.
* @returns A promise that resolves to a list of multipart uploads or an error.
*/
public async listMultipartUploads(
delimiter: string = '/',
prefix: string = '',
method: IT.HttpMethod = 'GET',
opts: Record<string, string | number | boolean | undefined> = {},
): Promise<IT.ListMultipartUploadSuccess | IT.MultipartUploadError> {
this._checkDelimiter(delimiter);
this._checkPrefix(prefix);
this._validateMethodIsGetOrHead(method);
this._checkOpts(opts);
const query = { uploads: '', ...opts };
const keyPath = delimiter === '/' ? delimiter : uriEscape(delimiter);
const res = await this._signedRequest(method, keyPath, {
query,
withQuery: true,
});
// doublecheck if this is needed
// if (method === 'HEAD') {
// return {
// size: +(res.headers.get(C.HEADER_CONTENT_LENGTH) ?? '0'),
// mtime: res.headers.get(C.HEADER_LAST_MODIFIED) ? new Date(res.headers.get(C.HEADER_LAST_MODIFIED)!) : undefined,
// etag: res.headers.get(C.HEADER_ETAG) ?? '',
// };
// }
const raw = parseXml(await res.text()) as unknown;
if (typeof raw !== 'object' || raw === null) {
throw new Error(`${C.ERROR_PREFIX}Unexpected listMultipartUploads response shape`);
}
if ('listMultipartUploadsResult' in raw) {
return raw.listMultipartUploadsResult as IT.ListMultipartUploadSuccess;
}
return raw as IT.MultipartUploadError;
}
/**
* Get an object from the S3-compatible service.
* This method sends a request to retrieve the specified object from the S3-compatible service.
* @param {string} key - The key of the object to retrieve.
* @param {Record<string, unknown>} [opts] - Additional options for the request.
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
* @returns A promise that resolves to the object data (string) or null if not found.
*/
public async getObject(
key: string,
opts: Record<string, unknown> = {},
ssecHeaders?: IT.SSECHeaders,
): Promise<string | null> {
if (this._bun && !ssecHeaders && !Object.keys(opts).length) {
return this._bunRead(key, f => f.text());
}
const res = await this._signedRequest('GET', key, {
query: opts, // use opts.query if it exists, otherwise use an empty object
tolerated: [200, 404, 412, 304],
headers: ssecHeaders ? { ...ssecHeaders } : undefined,
});
const s = res.status;
if (s === 200) {
return res.text();
}
void res.body?.cancel();
return null;
}
/**
* Get an object response from the S3-compatible service.
* This method sends a request to retrieve the specified object and returns the full response.
* @param {string} key - The key of the object to retrieve.
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
* @returns A promise that resolves to the Response object or null if not found.
*/
public async getObjectResponse(
key: string,
opts: Record<string, unknown> = {},
ssecHeaders?: IT.SSECHeaders,
): Promise<Response | null> {
const res = await this._signedRequest('GET', key, {
query: opts,
tolerated: [200, 404, 412, 304],
headers: ssecHeaders ? { ...ssecHeaders } : undefined,
});
if (res.status === 200) {
return res;
}
void res.body?.cancel();
return null;
}
/**
* Get an object as an ArrayBuffer from the S3-compatible service.
* This method sends a request to retrieve the specified object and returns it as an ArrayBuffer.
* @param {string} key - The key of the object to retrieve.
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
* @returns A promise that resolves to the object data as an ArrayBuffer or null if not found.
*/
public async getObjectArrayBuffer(
key: string,
opts: Record<string, unknown> = {},
ssecHeaders?: IT.SSECHeaders,
): Promise<ArrayBuffer | null> {
if (this._bun && !ssecHeaders && !Object.keys(opts).length) {
return this._bunRead(key, f => f.arrayBuffer());
}
const res = await this._signedRequest('GET', key, {
query: opts,
tolerated: [200, 404, 412, 304],
headers: ssecHeaders ? { ...ssecHeaders } : undefined,
});
if (res.status === 200) {
return res.arrayBuffer();
}
void res.body?.cancel();
return null;
}
/**
* Get an object as JSON from the S3-compatible service.
* This method sends a request to retrieve the specified object and returns it as JSON.
* @param {string} key - The key of the object to retrieve.
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
* @returns A promise that resolves to the object data as JSON or null if not found.
*/
public async getObjectJSON<T = unknown>(
key: string,
opts: Record<string, unknown> = {},
ssecHeaders?: IT.SSECHeaders,
): Promise<T | null> {
if (this._bun && !ssecHeaders && !Object.keys(opts).length) {
return this._bunRead(key, f => f.json()) as Promise<T | null>;
}
const res = await this._signedRequest('GET', key, {
query: opts,
tolerated: [200, 404, 412, 304],
headers: ssecHeaders ? { ...ssecHeaders } : undefined,
});
if (res.status === 200) {
return res.json() as Promise<T>;
}
void res.body?.cancel();
return null;
}
/**
* Get an object with its ETag from the S3-compatible service.
* This method sends a request to retrieve the specified object and its ETag.
* @param {string} key - The key of the object to retrieve.
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
* @returns A promise that resolves to an object containing the ETag and the object data as an ArrayBuffer or null if not found.
*/
public async getObjectWithETag(
key: string,
opts: Record<string, unknown> = {},
ssecHeaders?: IT.SSECHeaders,
): Promise<{ etag: string | null; data: ArrayBuffer | null }> {
try {
const res = await this._signedRequest('GET', key, {
query: opts,
tolerated: [200, 404, 412, 304],
headers: ssecHeaders ? { ...ssecHeaders } : undefined,
});
const s = res.status;
if (s === 404 || s === 412 || s === 304) {
void res.body?.cancel();
return { etag: null, data: null };
}
const etag = res.headers.get(C.HEADER_ETAG);
if (!etag) {
throw new Error(`${C.ERROR_PREFIX}ETag not found in response headers`);
}
return { etag: sanitizeETag(etag), data: await res.arrayBuffer() };
} catch (err) {
this._log('error', `Error getting object ${key} with ETag: ${String(err)}`);
throw err;
}
}
/**
* Get an object as a raw response from the S3-compatible service.
* This method sends a request to retrieve the specified object and returns the raw response.
* @param {string} key - The key of the object to retrieve.
* @param {boolean} [wholeFile=true] - Whether to retrieve the whole file or a range.
* @param {number} [rangeFrom=0] - The starting byte for the range (if not whole file).
* @param {number} [rangeTo=this.requestSizeInBytes] - The ending byte for the range (if not whole file).
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
* @param {IT.SSECHeaders} [ssecHeaders] - Server-Side Encryption headers, if any.
* @returns A promise that resolves to the Response object.
*/
public async getObjectRaw(
key: string,
wholeFile = true,
rangeFrom = 0,
rangeTo?: number,
opts: Record<string, unknown> = {},
ssecHeaders?: IT.SSECHeaders,
): Promise<Response> {
let rangeHdr: Record<string, string | number> = {};
if (!wholeFile) {
rangeHdr =
rangeTo === undefined ? { range: `bytes=${rangeFrom}-` } : { range: `bytes=${rangeFrom}-${rangeTo - 1}` };
}
return this._signedRequest('GET', key, {
query: { ...opts },
headers: { ...rangeHdr, ...ssecHeaders },
withQuery: true, // keep ?query=string behaviour
});
}
/**
* Get the content length of an object.
* This method sends a HEAD request to retrieve the content length of the specified object.
* @param {string} key - The key of the object to retrieve the content length for.
* @returns A promise that resolves to the content length of the object in bytes; 0 when the object exists but the response carries no content-length header.
* @throws {Error} If the object does not exist (HTTP 404) or the request otherwise fails; the underlying S3ServiceError is attached as `.cause`.
*/
public async getContentLength(key: string, ssecHeaders?: IT.SSECHeaders): Promise<number> {
try {
if (this._bun && !ssecHeaders) {
try {
return (await this._bun.file(key).stat()).size;
} catch (e) {
throw this._bunError(e);
}
}
const res = await this._signedRequest('HEAD', key, {
headers: ssecHeaders ? { ...ssecHeaders } : undefined,
});
const len = res.headers.get(C.HEADER_CONTENT_LENGTH);
return len ? +len : 0;
} catch (err) {
this._log('error', `Error getting content length for object ${key}: ${String(err)}`);
throw new Error(`${C.ERROR_PREFIX}Error getting content length for object ${key}: ${String(err)}`, {
cause: err,
});
}
}
/**
* Checks if an object exists in the S3-compatible service.
* This method sends a HEAD request to check if the specified object exists.
* @param {string} key - The key of the object to check.
* @param {Record<string, unknown>} [opts={}] - Additional options for the request.
* @returns A promise that resolves to true if the object exists, false if not found, or null if ETag mismatch.
*/
public async objectExists(key: string, opts: Record<string, unknown> = {}): Promise<IT.ExistResponseCode> {
if (this._bun && !Object.keys(opts).length) {
try {
return await this._bun.file(key).exists();
} catch (e) {
throw this._bunError(e);
}
}
const res = await this._signedRequest('HEAD', key, {
query: opts,
tolerated: [200, 404, 412, 304],
});
if (res.status === 404) {
return false; // not found
}
if (res.status === 412 || res.status === 304) {
return null; // ETag mismatch
}
return true; // found (200)
}
/**
* Retrie