lean-s3
Version:
A server-side S3 API for the regular user.
1,578 lines (1,562 loc) • 49.1 kB
JavaScript
// src/S3File.ts
import { Readable } from "stream";
// src/S3Stat.ts
var S3Stat = class _S3Stat {
etag;
lastModified;
size;
type;
constructor(etag, lastModified, size, type) {
this.etag = etag;
this.lastModified = lastModified;
this.size = size;
this.type = type;
}
static tryParseFromHeaders(headers) {
const lm = headers["last-modified"];
if (lm === null || typeof lm !== "string") {
return void 0;
}
const etag = headers.etag;
if (etag === null || typeof etag !== "string") {
return void 0;
}
const cl = headers["content-length"];
if (cl === null) {
return void 0;
}
const size = Number(cl);
if (!Number.isSafeInteger(size)) {
return void 0;
}
const ct = headers["content-type"];
if (ct === null || typeof ct !== "string") {
return void 0;
}
return new _S3Stat(etag, new Date(lm), size, ct);
}
};
// src/S3Client.ts
import { request, Agent } from "undici";
import { XMLParser as XMLParser2, XMLBuilder } from "fast-xml-parser";
// src/S3Error.ts
var S3Error = class extends Error {
code;
path;
message;
requestId;
hostId;
constructor(code, path, {
message = void 0,
requestId = void 0,
hostId = void 0,
cause = void 0
} = {}) {
super(message, { cause });
this.code = code;
this.path = path;
this.message = message ?? "Some unknown error occurred.";
this.requestId = requestId;
this.hostId = hostId;
}
};
// src/S3BucketEntry.ts
var S3BucketEntry = class _S3BucketEntry {
key;
size;
lastModified;
etag;
storageClass;
checksumAlgorithm;
checksumType;
constructor(key, size, lastModified, etag, storageClass, checksumAlgorithm, checksumType) {
this.key = key;
this.size = size;
this.lastModified = lastModified;
this.etag = etag;
this.storageClass = storageClass;
this.checksumAlgorithm = checksumAlgorithm;
this.checksumType = checksumType;
}
/**
* @internal
*/
// biome-ignore lint/suspicious/noExplicitAny: internal use only, any is ok here
static parse(source) {
return new _S3BucketEntry(
source.Key,
source.Size,
new Date(source.LastModified),
source.ETag,
source.StorageClass,
source.ChecksumAlgorithm,
source.ChecksumType
);
}
};
// src/sign.ts
import { createHmac, createHash } from "crypto";
function deriveSigningKey(date, region, secretAccessKey) {
const key = `AWS4${secretAccessKey}`;
const signedDate = createHmac("sha256", key).update(date).digest();
const signedDateRegion = createHmac("sha256", signedDate).update(region).digest();
const signedDateRegionService = createHmac("sha256", signedDateRegion).update("s3").digest();
return createHmac("sha256", signedDateRegionService).update("aws4_request").digest();
}
function signCanonicalDataHash(signinKey, canonicalDataHash, date, region) {
return createHmac("sha256", signinKey).update(
`AWS4-HMAC-SHA256
${date.dateTime}
${date.date}/${region}/s3/aws4_request
${canonicalDataHash}`
).digest("hex");
}
var unsignedPayload = "UNSIGNED-PAYLOAD";
function createCanonicalDataDigestHostOnly(method, path, query, host) {
return createHash("sha256").update(
`${method}
${path}
${query}
host:${host}
host
UNSIGNED-PAYLOAD`
).digest("hex");
}
function createCanonicalDataDigest(method, path, query, sortedHeaders, contentHashStr) {
const sortedHeaderNames = Object.keys(sortedHeaders);
let canonData = `${method}
${path}
${query}
`;
for (const header of sortedHeaderNames) {
canonData += `${header}:${sortedHeaders[header]}
`;
}
canonData += "\n";
canonData += sortedHeaderNames.length > 0 ? sortedHeaderNames[0] : "";
for (let i = 1; i < sortedHeaderNames.length; ++i) {
canonData += `;${sortedHeaderNames[i]}`;
}
canonData += `
${contentHashStr}`;
return createHash("sha256").update(canonData).digest("hex");
}
function sha256(data) {
return createHash("sha256").update(data).digest();
}
function md5Base64(data) {
return createHash("md5").update(data).digest("base64");
}
// src/KeyCache.ts
var KeyCache = class {
#lastNumericDay = -1;
#keys = /* @__PURE__ */ new Map();
computeIfAbsent(date, region, accessKeyId, secretAccessKey) {
if (date.numericDayStart !== this.#lastNumericDay) {
this.#keys.clear();
this.#lastNumericDay = date.numericDayStart;
}
const cacheKey = `${date.date}:${region}:${accessKeyId}`;
const key = this.#keys.get(cacheKey);
if (key) {
return key;
}
const newKey = deriveSigningKey(date.date, region, secretAccessKey);
this.#keys.set(cacheKey, newKey);
return newKey;
}
};
// src/AmzDate.ts
var ONE_DAY = 1e3 * 60 * 60 * 24;
function getAmzDate(dateTime) {
const date = pad4(dateTime.getUTCFullYear()) + pad2(dateTime.getUTCMonth() + 1) + pad2(dateTime.getUTCDate());
const time = pad2(dateTime.getUTCHours()) + pad2(dateTime.getUTCMinutes()) + pad2(dateTime.getUTCSeconds());
return {
numericDayStart: dateTime.getTime() / ONE_DAY | 0,
date,
dateTime: `${date}T${time}Z`
};
}
function now() {
return getAmzDate(/* @__PURE__ */ new Date());
}
function pad4(v) {
return v < 10 ? `000${v}` : v < 100 ? `00${v}` : v < 1e3 ? `0${v}` : v.toString();
}
function pad2(v) {
return v < 10 ? `0${v}` : v.toString();
}
// src/url.ts
function buildRequestUrl(endpoint, bucket, region, path) {
const normalizedBucket = normalizePath(bucket);
const [endpointWithBucketAndRegion, replacedBucket] = replaceDomainPlaceholders(endpoint, normalizedBucket, region);
const result = new URL(endpointWithBucketAndRegion);
const pathPrefix = result.pathname.endsWith("/") ? result.pathname : `${result.pathname}/`;
const pathSuffix = replacedBucket ? normalizePath(path) : `${normalizedBucket}/${normalizePath(path)}`;
result.pathname = pathPrefix + pathSuffix.replaceAll(":", "%3A").replaceAll("+", "%2B").replaceAll("(", "%28").replaceAll(")", "%29");
return result;
}
function replaceDomainPlaceholders(endpoint, bucket, region) {
const replacedBucket = endpoint.includes("{bucket}");
return [
endpoint.replaceAll("{bucket}", bucket).replaceAll("{region}", region),
replacedBucket
];
}
function normalizePath(path) {
const start = path[0] === "/" ? 1 : 0;
const end = path[path.length - 1] === "/" ? path.length - 1 : path.length;
return path.substring(start, end);
}
function prepareHeadersForSigning(unfilteredHeadersUnsorted) {
const result = {};
for (const header of Object.keys(unfilteredHeadersUnsorted).sort()) {
const v = unfilteredHeadersUnsorted[header];
if (v !== void 0 && v !== null) {
result[header] = v;
}
}
return result;
}
function getRangeHeader(start, endExclusive) {
return typeof start === "number" || typeof endExclusive === "number" ? (
// Http-ranges are end-inclusive, we are exclusiv ein our slice
`bytes=${start ?? 0}-${typeof endExclusive === "number" ? endExclusive - 1 : ""}`
) : void 0;
}
// src/error.ts
import { XMLParser } from "fast-xml-parser";
var xmlParser = new XMLParser();
async function getResponseError(response, path) {
let body;
try {
body = await response.body.text();
} catch (cause) {
return new S3Error("Unknown", path, {
message: "Could not read response body.",
cause
});
}
if (response.headers["content-type"] === "application/xml") {
return parseAndGetXmlError(body, path);
}
return new S3Error("Unknown", path, {
message: "Unknown error during S3 request."
});
}
function fromStatusCode(code, path) {
switch (code) {
case 404:
return new S3Error("NoSuchKey", path, {
message: "The specified key does not exist."
});
case 403:
return new S3Error("AccessDenied", path, {
message: "Access denied to the key."
});
// TODO: Add more status codes as needed
default:
return void 0;
}
}
function parseAndGetXmlError(body, path) {
let error;
try {
error = xmlParser.parse(body);
} catch (cause) {
return new S3Error("Unknown", path, {
message: "Could not parse XML error response.",
cause
});
}
if (error.Error) {
const e = error.Error;
return new S3Error(e.Code || "Unknown", path, {
message: e.Message || void 0
// Message might be "",
});
}
return new S3Error(error.Code || "Unknown", path, {
message: error.Message || void 0
// Message might be "",
});
}
// src/request.ts
function getAuthorizationHeader(keyCache, method, path, query, date, sortedSignedHeaders, region, contentHashStr, accessKeyId, secretAccessKey) {
const dataDigest = createCanonicalDataDigest(
method,
path,
query,
sortedSignedHeaders,
contentHashStr
);
const signingKey = keyCache.computeIfAbsent(
date,
region,
accessKeyId,
secretAccessKey
);
const signature = signCanonicalDataHash(
signingKey,
dataDigest,
date,
region
);
const signedHeadersSpec = Object.keys(sortedSignedHeaders).join(";");
const credentialSpec = `${accessKeyId}/${date.date}/${region}/s3/aws4_request`;
return `AWS4-HMAC-SHA256 Credential=${credentialSpec}, SignedHeaders=${signedHeadersSpec}, Signature=${signature}`;
}
// src/branded.ts
function ensureValidBucketName(name) {
if (name.length < 3 || name.length > 63) {
throw new Error("`name` must be between 3 and 63 characters long.");
}
if (name.startsWith(".") || name.endsWith(".")) {
throw new Error("`name` must not start or end with a period (.)");
}
if (!/^[a-z0-9.-]+$/.test(name)) {
throw new Error(
"`name` can only contain lowercase letters, numbers, periods (.), and hyphens (-)."
);
}
if (name.includes("..")) {
throw new Error("`name` must not contain two adjacent periods (..)");
}
return name;
}
function ensureValidPath(path) {
if (typeof path !== "string") {
throw new TypeError("`path` must be a `string`.");
}
if (path.length < 1) {
throw new RangeError("`path` must be at least 1 character long.");
}
return path;
}
// src/S3Client.ts
var write = Symbol("write");
var stream = Symbol("stream");
var signedRequest = Symbol("signedRequest");
var xmlParser2 = new XMLParser2({
ignoreAttributes: true,
isArray: (_, jPath) => jPath === "ListMultipartUploadsResult.Upload" || jPath === "ListBucketResult.Contents" || jPath === "ListPartsResult.Part" || jPath === "DeleteResult.Deleted" || jPath === "DeleteResult.Error"
});
var xmlBuilder = new XMLBuilder({
attributeNamePrefix: "$",
ignoreAttributes: false
});
var S3Client = class {
#options;
#keyCache = new KeyCache();
// TODO: pass options to this in client? Do we want to expose tjhe internal use of undici?
#dispatcher = new Agent();
/**
* Create a new instance of an S3 bucket so that credentials can be managed from a single instance instead of being passed to every method.
*
* @param options The default options to use for the S3 client.
*/
constructor(options) {
if (!options) {
throw new Error("`options` is required.");
}
const {
accessKeyId,
secretAccessKey,
endpoint,
region,
bucket,
sessionToken
} = options;
if (!accessKeyId || typeof accessKeyId !== "string") {
throw new Error("`accessKeyId` is required.");
}
if (!secretAccessKey || typeof secretAccessKey !== "string") {
throw new Error("`secretAccessKey` is required.");
}
if (!endpoint || typeof endpoint !== "string") {
throw new Error("`endpoint` is required.");
}
if (!region || typeof region !== "string") {
throw new Error("`region` is required.");
}
if (!bucket || typeof bucket !== "string") {
throw new Error("`bucket` is required.");
}
this.#options = {
accessKeyId,
secretAccessKey,
endpoint,
region,
bucket,
sessionToken
};
}
/**
* Creates an S3File instance for the given path.
*
* @param {string} path The path to the object in the bucket. Also known as [object key](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html).
* We recommend not using the following characters in a key name because of significant special character handling, which isn't consistent across all applications (see [AWS docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html)):
* - Backslash (`\\`)
* - Left brace (`{`)
* - Non-printable ASCII characters (128–255 decimal characters)
* - Caret or circumflex (`^`)
* - Right brace (`}`)
* - Percent character (`%`)
* - Grave accent or backtick (`\``)
* - Right bracket (`]`)
* - Quotation mark (`"`)
* - Greater than sign (`>`)
* - Left bracket (`[`)
* - Tilde (`~`)
* - Less than sign (`<`)
* - Pound sign (`#`)
* - Vertical bar or pipe (`|`)
*
* lean-s3 does not enforce these restrictions.
*
* @param {Partial<CreateFileInstanceOptions>} [_options] TODO
* @example
* ```js
* const file = client.file("image.jpg");
* await file.write(imageData);
*
* const configFile = client.file("config.json", {
* type: "application/json",
* acl: "private"
* });
* ```
*/
file(path, _options) {
return new S3File(
this,
ensureValidPath(path),
void 0,
void 0,
void 0
);
}
/**
* Generate a presigned URL for temporary access to a file.
* Useful for generating upload/download URLs without exposing credentials.
* @returns The operation on {@link S3Client#presign.path} as a pre-signed URL.
*
* @example
* ```js
* const downloadUrl = client.presign("file.pdf", {
* expiresIn: 3600 // 1 hour
* });
* ```
*/
presign(path, {
method = "GET",
expiresIn = 3600,
// TODO: Maybe rename this to expiresInSeconds
storageClass,
contentLength,
acl,
region: regionOverride,
bucket: bucketOverride,
endpoint: endpointOverride
} = {}) {
if (typeof contentLength === "number") {
if (contentLength < 0) {
throw new RangeError("`contentLength` must be >= 0.");
}
}
const now2 = /* @__PURE__ */ new Date();
const date = getAmzDate(now2);
const options = this.#options;
const region = regionOverride ?? options.region;
const bucket = bucketOverride ?? options.bucket;
const endpoint = endpointOverride ?? options.endpoint;
const res = buildRequestUrl(endpoint, bucket, region, path);
const query = buildSearchParams(
`${options.accessKeyId}/${date.date}/${region}/s3/aws4_request`,
date,
expiresIn,
typeof contentLength === "number" ? "content-length;host" : "host",
unsignedPayload,
storageClass,
options.sessionToken,
acl
);
const dataDigest = typeof contentLength === "number" ? createCanonicalDataDigest(
method,
res.pathname,
query,
{ "content-length": String(contentLength), host: res.host },
unsignedPayload
) : createCanonicalDataDigestHostOnly(
method,
res.pathname,
query,
res.host
);
const signingKey = this.#keyCache.computeIfAbsent(
date,
region,
options.accessKeyId,
options.secretAccessKey
);
const signature = signCanonicalDataHash(
signingKey,
dataDigest,
date,
region
);
res.search = `${query}&X-Amz-Signature=${signature}`;
return res.toString();
}
//#region multipart uploads
async createMultipartUpload(key, options = {}) {
if (key.length < 1) {
}
const response = await this[signedRequest](
"POST",
ensureValidPath(key),
"uploads=",
void 0,
void 0,
void 0,
void 0,
ensureValidBucketName(options.bucket ?? this.#options.bucket),
options.signal
);
if (response.statusCode !== 200) {
throw await getResponseError(response, key);
}
const text = await response.body.text();
const res = ensureParsedXml(text).InitiateMultipartUploadResult ?? {};
return {
bucket: res.Bucket,
key: res.Key,
uploadId: res.UploadId
};
}
/**
* @remarks Uses [`ListMultipartUploads`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html).
* @throws {RangeError} If `options.maxKeys` is not between `1` and `1000`.
*/
async listMultipartUploads(options = {}) {
const bucket = ensureValidBucketName(
options.bucket ?? this.#options.bucket
);
let query = "uploads=";
if (options.delimiter) {
if (typeof options.delimiter !== "string") {
throw new TypeError("`delimiter` must be a `string`.");
}
query += `&delimiter=${encodeURIComponent(options.delimiter)}`;
}
if (options.keyMarker) {
if (typeof options.keyMarker !== "string") {
throw new TypeError("`keyMarker` must be a `string`.");
}
query += `&key-marker=${encodeURIComponent(options.keyMarker)}`;
}
if (typeof options.maxUploads !== "undefined") {
if (typeof options.maxUploads !== "number") {
throw new TypeError("`maxUploads` must be a `number`.");
}
if (options.maxUploads < 1 || options.maxUploads > 1e3) {
throw new RangeError("`maxUploads` has to be between 1 and 1000.");
}
query += `&max-uploads=${options.maxUploads}`;
}
if (options.prefix) {
if (typeof options.prefix !== "string") {
throw new TypeError("`prefix` must be a `string`.");
}
query += `&prefix=${encodeURIComponent(options.prefix)}`;
}
const response = await this[signedRequest](
"GET",
"",
query,
void 0,
void 0,
void 0,
void 0,
bucket,
options.signal
);
if (response.statusCode !== 200) {
throw await getResponseError(response, "");
}
const text = await response.body.text();
const root = ensureParsedXml(text).ListMultipartUploadsResult ?? {};
return {
bucket: root.Bucket || void 0,
delimiter: root.Delimiter || void 0,
prefix: root.Prefix || void 0,
keyMarker: root.KeyMarker || void 0,
uploadIdMarker: root.UploadIdMarker || void 0,
nextKeyMarker: root.NextKeyMarker || void 0,
nextUploadIdMarker: root.NextUploadIdMarker || void 0,
maxUploads: root.MaxUploads ?? 1e3,
// not using || to not override 0; caution: minio supports 10000(!)
isTruncated: root.IsTruncated === "true",
uploads: root.Upload?.map(
// biome-ignore lint/suspicious/noExplicitAny: we're parsing here
(u) => ({
key: u.Key || void 0,
uploadId: u.UploadId || void 0,
// TODO: Initiator
// TODO: Owner
storageClass: u.StorageClass || void 0,
checksumAlgorithm: u.ChecksumAlgorithm || void 0,
checksumType: u.ChecksumType || void 0,
initiated: u.Initiated ? new Date(u.Initiated) : void 0
})
) ?? []
};
}
/**
* @remarks Uses [`AbortMultipartUpload`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html).
* @throws {RangeError} If `key` is not at least 1 character long.
* @throws {Error} If `uploadId` is not provided.
*/
async abortMultipartUpload(path, uploadId, options = {}) {
if (!uploadId) {
throw new Error("`uploadId` is required.");
}
const response = await this[signedRequest](
"DELETE",
ensureValidPath(path),
`uploadId=${encodeURIComponent(uploadId)}`,
void 0,
void 0,
void 0,
void 0,
ensureValidBucketName(options.bucket ?? this.#options.bucket),
options.signal
);
if (response.statusCode !== 204) {
throw await getResponseError(response, path);
}
}
/**
* @remarks Uses [`CompleteMultipartUpload`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html).
* @throws {RangeError} If `key` is not at least 1 character long.
* @throws {Error} If `uploadId` is not provided.
*/
async completeMultipartUpload(path, uploadId, parts, options = {}) {
if (!uploadId) {
throw new Error("`uploadId` is required.");
}
const body = xmlBuilder.build({
CompleteMultipartUpload: {
Part: parts.map((part) => ({
PartNumber: part.partNumber,
ETag: part.etag
}))
}
});
const response = await this[signedRequest](
"POST",
ensureValidPath(path),
`uploadId=${encodeURIComponent(uploadId)}`,
body,
void 0,
void 0,
void 0,
ensureValidBucketName(options.bucket ?? this.#options.bucket),
options.signal
);
if (response.statusCode !== 200) {
throw await getResponseError(response, path);
}
const text = await response.body.text();
const res = ensureParsedXml(text).CompleteMultipartUploadResult ?? {};
return {
location: res.Location || void 0,
bucket: res.Bucket || void 0,
key: res.Key || void 0,
etag: res.ETag || void 0,
checksumCRC32: res.ChecksumCRC32 || void 0,
checksumCRC32C: res.ChecksumCRC32C || void 0,
checksumCRC64NVME: res.ChecksumCRC64NVME || void 0,
checksumSHA1: res.ChecksumSHA1 || void 0,
checksumSHA256: res.ChecksumSHA256 || void 0,
checksumType: res.ChecksumType || void 0
};
}
/**
* @remarks Uses [`UploadPart`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html).
* @throws {RangeError} If `key` is not at least 1 character long.
* @throws {Error} If `uploadId` is not provided.
*/
async uploadPart(path, uploadId, data, partNumber, options = {}) {
if (!uploadId) {
throw new Error("`uploadId` is required.");
}
if (!data) {
throw new Error("`data` is required.");
}
if (typeof partNumber !== "number" || partNumber <= 0) {
throw new Error("`partNumber` has to be a `number` which is >= 1.");
}
const response = await this[signedRequest](
"PUT",
ensureValidPath(path),
`partNumber=${partNumber}&uploadId=${encodeURIComponent(uploadId)}`,
data,
void 0,
void 0,
void 0,
ensureValidBucketName(options.bucket ?? this.#options.bucket),
options.signal
);
if (response.statusCode === 200) {
await response.body.dump();
const etag = response.headers.etag;
if (typeof etag !== "string" || etag.length === 0) {
throw new S3Error("Unknown", "", {
message: "Response did not contain an etag."
});
}
return {
partNumber,
etag
};
}
throw await getResponseError(response, "");
}
/**
* @remarks Uses [`ListParts`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html).
* @throws {RangeError} If `key` is not at least 1 character long.
* @throws {Error} If `uploadId` is not provided.
* @throws {TypeError} If `options.maxParts` is not a `number`.
* @throws {RangeError} If `options.maxParts` is <= 0.
* @throws {TypeError} If `options.partNumberMarker` is not a `string`.
*/
async listParts(path, uploadId, options = {}) {
let query = "";
if (options.maxParts) {
if (typeof options.maxParts !== "number") {
throw new TypeError("`maxParts` must be a `number`.");
}
if (options.maxParts <= 0) {
throw new RangeError("`maxParts` must be >= 1.");
}
query += `&max-parts=${options.maxParts}`;
}
if (options.partNumberMarker) {
if (typeof options.partNumberMarker !== "string") {
throw new TypeError("`partNumberMarker` must be a `string`.");
}
query += `&part-number-marker=${encodeURIComponent(options.partNumberMarker)}`;
}
query += `&uploadId=${encodeURIComponent(uploadId)}`;
const response = await this[signedRequest](
"GET",
ensureValidPath(path),
// We always have a leading &, so we can slice the leading & away (this way, we have less conditionals on the hot path); see benchmark-operations.js
query.substring(1),
void 0,
void 0,
void 0,
void 0,
ensureValidBucketName(options.bucket ?? this.#options.bucket),
options?.signal
);
if (response.statusCode === 200) {
const text = await response.body.text();
const root = ensureParsedXml(text).ListPartsResult ?? {};
return {
bucket: root.Bucket,
key: root.Key,
uploadId: root.UploadId,
partNumberMarker: root.PartNumberMarker ?? void 0,
nextPartNumberMarker: root.NextPartNumberMarker ?? void 0,
maxParts: root.MaxParts ?? 1e3,
isTruncated: root.IsTruncated ?? false,
parts: (
// biome-ignore lint/suspicious/noExplicitAny: parsing code
root.Part?.map((part) => ({
etag: part.ETag,
lastModified: part.LastModified ? new Date(part.LastModified) : void 0,
partNumber: part.PartNumber ?? void 0,
size: part.Size ?? void 0
})) ?? []
)
};
}
throw await getResponseError(response, path);
}
//#endregion
//#region bucket operations
/**
* Creates a new bucket on the S3 server.
*
* @param name The name of the bucket to create. AWS the name according to [some rules](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html). The most important ones are:
* - Bucket names must be between `3` (min) and `63` (max) characters long.
* - Bucket names can consist only of lowercase letters, numbers, periods (`.`), and hyphens (`-`).
* - Bucket names must begin and end with a letter or number.
* - Bucket names must not contain two adjacent periods.
* - Bucket names must not be formatted as an IP address (for example, `192.168.5.4`).
*
* @throws {Error} If the bucket name is invalid.
* @throws {S3Error} If the bucket could not be created, e.g. if it already exists.
* @remarks Uses [`CreateBucket`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html)
*/
async createBucket(name, options) {
let body;
if (options) {
const location = options.location && (options.location.name || options.location.type) ? {
Name: options.location.name ?? void 0,
Type: options.location.type ?? void 0
} : void 0;
const bucket = options.info && (options.info.dataRedundancy || options.info.type) ? {
DataRedundancy: options.info.dataRedundancy ?? void 0,
Type: options.info.type ?? void 0
} : void 0;
body = location || bucket || options.locationConstraint ? xmlBuilder.build({
CreateBucketConfiguration: {
$xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
LocationConstraint: options.locationConstraint ?? void 0,
Location: location,
Bucket: bucket
}
}) : void 0;
}
const additionalSignedHeaders = body ? { "content-md5": md5Base64(body) } : void 0;
const response = await this[signedRequest](
"PUT",
"",
void 0,
body,
additionalSignedHeaders,
void 0,
void 0,
ensureValidBucketName(name),
options?.signal
);
if (400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
await response.body.dump();
if (response.statusCode === 200) {
return;
}
throw new Error(`Response code not supported: ${response.statusCode}`);
}
/**
* Deletes a bucket from the S3 server.
* @param name The name of the bucket to delete. Same restrictions as in {@link S3Client#createBucket}.
* @throws {Error} If the bucket name is invalid.
* @throws {S3Error} If the bucket could not be deleted, e.g. if it is not empty.
* @remarks Uses [`DeleteBucket`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html).
*/
async deleteBucket(name, options) {
const response = await this[signedRequest](
"DELETE",
"",
void 0,
void 0,
void 0,
void 0,
void 0,
ensureValidBucketName(name),
options?.signal
);
if (400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
await response.body.dump();
if (response.statusCode === 204) {
return;
}
throw new Error(`Response code not supported: ${response.statusCode}`);
}
/**
* Checks if a bucket exists.
* @param name The name of the bucket to delete. Same restrictions as in {@link S3Client#createBucket}.
* @throws {Error} If the bucket name is invalid.
* @remarks Uses [`HeadBucket`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html).
*/
async bucketExists(name, options) {
const response = await this[signedRequest](
"HEAD",
"",
void 0,
void 0,
void 0,
void 0,
void 0,
ensureValidBucketName(name),
options?.signal
);
if (response.statusCode !== 404 && 400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
await response.body.dump();
if (response.statusCode === 200) {
return true;
}
if (response.statusCode === 404) {
return false;
}
throw new Error(`Response code not supported: ${response.statusCode}`);
}
//#endregion
//#region list objects
/**
* Uses [`ListObjectsV2`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) to iterate over all keys. Pagination and continuation is handled internally.
*/
async *listIterating(options) {
const maxKeys = options?.internalPageSize ?? void 0;
let continuationToken;
do {
const res = await this.list({
...options,
maxKeys,
continuationToken
});
if (!res || res.contents.length === 0) {
break;
}
yield* res.contents;
continuationToken = res.nextContinuationToken;
} while (continuationToken);
}
/**
* Implements [`ListObjectsV2`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) to iterate over all keys.
*
* @throws {RangeError} If `maxKeys` is not between `1` and `1000`.
*/
async list(options = {}) {
let query = "";
if (typeof options.continuationToken !== "undefined") {
if (typeof options.continuationToken !== "string") {
throw new TypeError("`continuationToken` must be a `string`.");
}
query += `continuation-token=${encodeURIComponent(options.continuationToken)}&`;
}
query += "list-type=2";
if (typeof options.maxKeys !== "undefined") {
if (typeof options.maxKeys !== "number") {
throw new TypeError("`maxKeys` must be a `number`.");
}
if (options.maxKeys < 1 || options.maxKeys > 1e3) {
throw new RangeError("`maxKeys` has to be between 1 and 1000.");
}
query += `&max-keys=${options.maxKeys}`;
}
if (options.prefix) {
if (typeof options.prefix !== "string") {
throw new TypeError("`prefix` must be a `string`.");
}
query += `&prefix=${encodeURIComponent(options.prefix)}`;
}
if (typeof options.startAfter !== "undefined") {
if (typeof options.startAfter !== "string") {
throw new TypeError("`startAfter` must be a `string`.");
}
query += `&start-after=${encodeURIComponent(options.startAfter)}`;
}
const response = await this[signedRequest](
"GET",
"",
query,
void 0,
void 0,
void 0,
void 0,
ensureValidBucketName(options.bucket ?? this.#options.bucket),
options.signal
);
if (response.statusCode !== 200) {
response.body.dump();
throw new Error(
`Response code not implemented yet: ${response.statusCode}`
);
}
const text = await response.body.text();
const res = ensureParsedXml(text).ListBucketResult ?? {};
if (!res) {
throw new S3Error("Unknown", "", {
message: "Could not read bucket contents."
});
}
return {
name: res.Name,
prefix: res.Prefix,
startAfter: res.StartAfter,
isTruncated: res.IsTruncated,
continuationToken: res.ContinuationToken,
maxKeys: res.MaxKeys,
keyCount: res.KeyCount,
nextContinuationToken: res.NextContinuationToken,
contents: res.Contents?.map(S3BucketEntry.parse) ?? []
};
}
//#endregion
/**
* Uses [`DeleteObjects`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) to delete multiple objects in a single request.
*/
async deleteObjects(objects, options = {}) {
const body = xmlBuilder.build({
Delete: {
Quiet: true,
Object: objects.map((o) => ({
Key: typeof o === "string" ? o : o.key
}))
}
});
const response = await this[signedRequest](
"POST",
"",
"delete=",
// "=" is needed by minio for some reason
body,
{
"content-md5": md5Base64(body)
},
void 0,
void 0,
ensureValidBucketName(options.bucket ?? this.#options.bucket),
options.signal
);
if (response.statusCode === 200) {
const text = await response.body.text();
let deleteResult;
try {
deleteResult = ensureParsedXml(text).DeleteResult ?? {};
} catch (cause) {
throw new S3Error("Unknown", "", {
message: "S3 service responded with invalid XML.",
cause
});
}
const errors = (
// biome-ignore lint/suspicious/noExplicitAny: parsing
deleteResult.Error?.map((e) => ({
code: e.Code,
key: e.Key,
message: e.Message,
versionId: e.VersionId
})) ?? []
);
return errors.length > 0 ? { errors } : null;
}
if (400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
response.body.dump();
throw new Error(
`Response code not implemented yet: ${response.statusCode}`
);
}
/**
* Do not use this. This is an internal method.
* TODO: Maybe move this into a separate free function?
* @internal
*/
async [signedRequest](method, pathWithoutBucket, query, body, additionalSignedHeaders, additionalUnsignedHeaders, contentHash, bucket, signal = void 0) {
const endpoint = this.#options.endpoint;
const region = this.#options.region;
const effectiveBucket = bucket ?? this.#options.bucket;
const url = buildRequestUrl(
endpoint,
effectiveBucket,
region,
pathWithoutBucket
);
if (query) {
url.search = query;
}
const now2 = now();
const contentHashStr = contentHash?.toString("hex") ?? unsignedPayload;
const headersToBeSigned = prepareHeadersForSigning({
host: url.host,
"x-amz-date": now2.dateTime,
"x-amz-content-sha256": contentHashStr,
...additionalSignedHeaders
});
try {
return await request(url, {
method,
signal,
dispatcher: this.#dispatcher,
headers: {
...headersToBeSigned,
authorization: getAuthorizationHeader(
this.#keyCache,
method,
url.pathname,
query ?? "",
now2,
headersToBeSigned,
region,
contentHashStr,
this.#options.accessKeyId,
this.#options.secretAccessKey
),
...additionalUnsignedHeaders,
"user-agent": "lean-s3"
},
body
});
} catch (cause) {
signal?.throwIfAborted();
throw new S3Error("Unknown", pathWithoutBucket, {
message: "Unknown error during S3 request.",
cause
});
}
}
/**
* @internal
* @param {import("./index.d.ts").UndiciBodyInit} data TODO
*/
async [write](path, data, contentType, contentLength, contentHash, rageStart, rangeEndExclusive, signal = void 0) {
const bucket = this.#options.bucket;
const endpoint = this.#options.endpoint;
const region = this.#options.region;
const url = buildRequestUrl(endpoint, bucket, region, path);
const now2 = now();
const contentHashStr = contentHash?.toString("hex") ?? unsignedPayload;
const headersToBeSigned = prepareHeadersForSigning({
"content-length": contentLength?.toString() ?? void 0,
"content-type": contentType,
host: url.host,
range: getRangeHeader(rageStart, rangeEndExclusive),
"x-amz-content-sha256": contentHashStr,
"x-amz-date": now2.dateTime
});
let response;
try {
response = await request(url, {
method: "PUT",
signal,
dispatcher: this.#dispatcher,
headers: {
...headersToBeSigned,
authorization: getAuthorizationHeader(
this.#keyCache,
"PUT",
url.pathname,
url.search,
now2,
headersToBeSigned,
region,
contentHashStr,
this.#options.accessKeyId,
this.#options.secretAccessKey
),
"user-agent": "lean-s3"
},
body: data
});
} catch (cause) {
signal?.throwIfAborted();
throw new S3Error("Unknown", path, {
message: "Unknown error during S3 request.",
cause
});
}
const status = response.statusCode;
if (200 <= status && status < 300) {
return;
}
throw await getResponseError(response, path);
}
// TODO: Support abortSignal
/**
* @internal
*/
[stream](path, contentHash, rageStart, rangeEndExclusive) {
const bucket = this.#options.bucket;
const endpoint = this.#options.endpoint;
const region = this.#options.region;
const now2 = now();
const url = buildRequestUrl(endpoint, bucket, region, path);
const range = getRangeHeader(rageStart, rangeEndExclusive);
const contentHashStr = contentHash?.toString("hex") ?? unsignedPayload;
const headersToBeSigned = prepareHeadersForSigning({
"amz-sdk-invocation-id": crypto.randomUUID(),
// TODO: Maybe support retries and do "amz-sdk-request": attempt=1; max=3
host: url.host,
range,
// Hetzner doesnt care if the x-amz-content-sha256 header is missing, R2 requires it to be present
"x-amz-content-sha256": contentHashStr,
"x-amz-date": now2.dateTime
});
const ac = new AbortController();
return new ReadableStream({
type: "bytes",
start: (controller) => {
const onNetworkError = (cause) => {
controller.error(
new S3Error("Unknown", path, {
message: void 0,
cause
})
);
};
request(url, {
method: "GET",
signal: ac.signal,
dispatcher: this.#dispatcher,
headers: {
...headersToBeSigned,
authorization: getAuthorizationHeader(
this.#keyCache,
"GET",
url.pathname,
url.search,
now2,
headersToBeSigned,
region,
contentHashStr,
this.#options.accessKeyId,
this.#options.secretAccessKey
),
"user-agent": "lean-s3"
}
}).then((response) => {
const onData = controller.enqueue.bind(controller);
const onClose = controller.close.bind(controller);
const expectPartialResponse = range !== void 0;
const status = response.statusCode;
if (status === 200) {
if (expectPartialResponse) {
return controller.error(
new S3Error("Unknown", path, {
message: "Expected partial response to range request."
})
);
}
response.body.on("data", onData);
response.body.once("error", onNetworkError);
response.body.once("end", onClose);
return;
}
if (status === 206) {
if (!expectPartialResponse) {
return controller.error(
new S3Error("Unknown", path, {
message: "Received partial response but expected a full response."
})
);
}
response.body.on("data", onData);
response.body.once("error", onNetworkError);
response.body.once("end", onClose);
return;
}
if (400 <= status && status < 500) {
const responseText = void 0;
if (response.headers["content-type"] === "application/xml") {
return response.body.text().then((body) => {
let error;
try {
error = xmlParser2.parse(body);
} catch (cause) {
return controller.error(
new S3Error("Unknown", path, {
message: "Could not parse XML error response.",
cause
})
);
}
return controller.error(
new S3Error(error.Code || "Unknown", path, {
message: error.Message || void 0
// Message might be "",
})
);
}, onNetworkError);
}
return controller.error(
new S3Error("Unknown", path, {
message: void 0,
cause: responseText
})
);
}
return controller.error(
new Error(
`Handling for status code ${status} not implemented yet. You might want to open an issue and describe your situation.`
)
);
}, onNetworkError);
},
cancel(reason) {
ac.abort(reason);
}
});
}
};
function buildSearchParams(amzCredential, date, expiresIn, headerList, contentHashStr, storageClass, sessionToken, acl) {
let res = "";
if (acl) {
res += `X-Amz-Acl=${encodeURIComponent(acl)}&`;
}
res += "X-Amz-Algorithm=AWS4-HMAC-SHA256";
if (contentHashStr) {
res += `&X-Amz-Content-Sha256=${contentHashStr}`;
}
res += `&X-Amz-Credential=${encodeURIComponent(amzCredential)}`;
res += `&X-Amz-Date=${date.dateTime}`;
res += `&X-Amz-Expires=${expiresIn}`;
if (sessionToken) {
res += `&X-Amz-Security-Token=${encodeURIComponent(sessionToken)}`;
}
res += `&X-Amz-SignedHeaders=${encodeURIComponent(headerList)}`;
if (storageClass) {
res += `&X-Amz-Storage-Class=${storageClass}`;
}
return res;
}
function ensureParsedXml(text) {
try {
const r = xmlParser2.parse(text);
if (!r) {
throw new S3Error("Unknown", "", {
message: "S3 service responded with empty XML."
});
}
return r;
} catch (cause) {
throw new S3Error("Unknown", "", {
message: "S3 service responded with invalid XML.",
cause
});
}
}
// src/assertNever.ts
function assertNever(v) {
throw new TypeError(`Expected value not to have type ${typeof v}`);
}
// src/S3File.ts
var S3File = class _S3File {
#client;
#path;
#start;
#end;
#contentType;
/**
* @internal
*/
constructor(client, path, start, end, contentType) {
if (typeof start === "number" && start < 0) {
throw new Error("Invalid slice `start`.");
}
if (typeof end === "number" && (end < 0 || typeof start === "number" && end < start)) {
throw new Error("Invalid slice `end`.");
}
this.#client = client;
this.#path = path;
this.#start = start;
this.#end = end;
this.#contentType = contentType ?? "application/octet-stream";
}
// TODO: slice overloads
slice(start, end, contentType) {
return new _S3File(
this.#client,
this.#path,
start ?? void 0,
end ?? void 0,
contentType ?? this.#contentType
);
}
/**
* Get the stat of a file in the bucket. Uses `HEAD` request to check existence.
*
* @remarks Uses [`HeadObject`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
* @throws {S3Error} If the file does not exist or the server has some other issues.
* @throws {Error} If the server returns an invalid response.
*/
async stat({ signal } = {}) {
const response = await this.#client[signedRequest](
"HEAD",
this.#path,
void 0,
void 0,
void 0,
void 0,
void 0,
void 0,
signal
);
await response.body.dump();
if (200 <= response.statusCode && response.statusCode < 300) {
const result = S3Stat.tryParseFromHeaders(response.headers);
if (!result) {
throw new Error(
"S3 server returned an invalid response for `HeadObject`"
);
}
return result;
}
throw fromStatusCode(response.statusCode, this.#path) ?? new Error(
`S3 server returned an unsupported status code for \`HeadObject\`: ${response.statusCode}`
);
}
/**
* Check if a file exists in the bucket. Uses `HEAD` request to check existence.
*
* @remarks Uses [`HeadObject`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
*/
async exists({
signal
} = {}) {
const response = await this.#client[signedRequest](
"HEAD",
this.#path,
void 0,
void 0,
void 0,
void 0,
void 0,
void 0,
signal
);
await response.body.dump();
if (200 <= response.statusCode && response.statusCode < 300) {
return true;
}
if (response.statusCode === 404) {
return false;
}
throw fromStatusCode(response.statusCode, this.#path) ?? new Error(
`S3 server returned an unsupported status code for \`HeadObject\`: ${response.statusCode}`
);
}
/**
* Delete a file from the bucket.
*
* @remarks - Uses [`DeleteObject`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html).
* - `versionId` not supported.
*
* @param {Partial<S3FileDeleteOptions>} [options]
*
* @example
* ```js
* // Simple delete
* await client.unlink("old-file.txt");
*
* // With error handling
* try {
* await client.unlink("file.dat");
* console.log("File deleted");
* } catch (err) {
* console.error("Delete failed:", err);
* }
* ```
*/
async delete({ signal } = {}) {
const response = await this.#client[signedRequest](
"DELETE",
this.#path,
void 0,
void 0,
void 0,
void 0,
void 0,
void 0,
signal
);
if (response.statusCode === 204) {
await response.body.dump();
return;
}
throw await getResponseError(response, this.#path);
}
toString() {
return `S3File { path: "${this.#path}" }`;
}
json() {
return new Response(this.stream()).json();
}
bytes() {
return new Response(this.stream()).arrayBuffer().then((ab) => new Uint8Array(ab));
}
arrayBuffer() {
return new Response(this.stream()).arrayBuffer();
}
text() {
return new Response(this.stream()).text();
}
blob() {
return new Response(this.stream()).blob();
}
/** @returns {ReadableStream<Uint8Array>} */
stream() {
return this.#client[stream](this.#path, void 0, this.#start, this.#end);
}
async #transformData(data) {
if (typeof data === "string") {
const binary = new TextEncoder();
const bytes = binary.encode(data);
return [
bytes,
bytes.byteLength,
sha256(bytes)
// TODO: Maybe use some streaming to compute hash while encoding?
];
}
if (data instanceof Blob) {
const bytes = await data.bytes();
return [
bytes,
bytes.byteLength,
sha256(bytes)
// TODO: Maybe use some streaming to compute hash while encoding?
];
}
if (data instanceof Readable) {
return [data, void 0, void 0];
}
if (data instanceof ArrayBuffer || data instanceof SharedArrayBuffer || ArrayBuffer.isView(data)) {
return [
data,
data.byteLength,
void 0
// TODO: Compute hash some day
];
}
assertNever(data);
}
/**
* @param {ByteSource} data
* @returns {Promise<void>}
*/
async write(data) {
const signal = void 0;
const [bytes, length, hash] = await this.#transformData(data);
return await this.#client[write](
this.#path,
bytes,
this.#contentType,
length,
hash,
this.#start,
this.#end,
signal
);
}
/*
// Future API?
writer(): WritableStream<ArrayBufferLike | ArrayBufferView> {
throw new Error("Not implemented");
}
// Future API?
setTags(): Promise<void> {
throw new Error("Not implemented");
}
getTags(): Promise<unknown> {
throw new Error("Not implemented");
}
*/
};
export {
S3BucketEntry,
S3Client,
S3Error,
S3File,
S3Stat
};