lean-s3
Version:
A server-side S3 API for the regular user.
1,523 lines (1,507 loc) • 56.2 kB
JavaScript
// src/S3File.ts
import { Readable } from "stream";
// src/S3Stat.ts
var S3Stat = class _S3Stat {
etag;
lastModified;
size;
type;
constructor(etag, lastModified, size, type) {
this.etag = etag;
this.lastModified = lastModified;
this.size = size;
this.type = type;
}
static tryParseFromHeaders(headers) {
const lm = headers["last-modified"];
if (lm === null || typeof lm !== "string") {
return void 0;
}
const etag = headers.etag;
if (etag === null || typeof etag !== "string") {
return void 0;
}
const cl = headers["content-length"];
if (cl === null) {
return void 0;
}
const size = Number(cl);
if (!Number.isSafeInteger(size)) {
return void 0;
}
const ct = headers["content-type"];
if (ct === null || typeof ct !== "string") {
return void 0;
}
return new _S3Stat(etag, new Date(lm), size, ct);
}
};
// src/S3Client.ts
import { request, Agent } from "undici";
import { XMLParser as XMLParser2, XMLBuilder } from "fast-xml-parser";
// src/S3Error.ts
var S3Error = class extends Error {
code;
path;
message;
requestId;
hostId;
constructor(code, path, {
message = void 0,
requestId = void 0,
hostId = void 0,
cause = void 0
} = {}) {
super(message, { cause });
this.code = code;
this.path = path;
this.message = message ?? "Some unknown error occurred.";
this.requestId = requestId;
this.hostId = hostId;
}
};
// src/S3BucketEntry.ts
var S3BucketEntry = class _S3BucketEntry {
key;
size;
lastModified;
etag;
storageClass;
checksumAlgorithm;
checksumType;
constructor(key, size, lastModified, etag, storageClass, checksumAlgorithm, checksumType) {
this.key = key;
this.size = size;
this.lastModified = lastModified;
this.etag = etag;
this.storageClass = storageClass;
this.checksumAlgorithm = checksumAlgorithm;
this.checksumType = checksumType;
}
/**
* @internal
*/
// biome-ignore lint/suspicious/noExplicitAny: internal use only, any is ok here
static parse(source) {
return new _S3BucketEntry(
source.Key,
source.Size,
new Date(source.LastModified),
source.ETag,
source.StorageClass,
source.ChecksumAlgorithm,
source.ChecksumType
);
}
};
// src/sign.ts
import { createHmac, createHash } from "crypto";
function deriveSigningKey(date, region, secretAccessKey) {
const key = `AWS4${secretAccessKey}`;
const signedDate = createHmac("sha256", key).update(date).digest();
const signedDateRegion = createHmac("sha256", signedDate).update(region).digest();
const signedDateRegionService = createHmac("sha256", signedDateRegion).update("s3").digest();
return createHmac("sha256", signedDateRegionService).update("aws4_request").digest();
}
function signCanonicalDataHash(signinKey, canonicalDataHash, date, region) {
return createHmac("sha256", signinKey).update(
`AWS4-HMAC-SHA256
${date.dateTime}
${date.date}/${region}/s3/aws4_request
${canonicalDataHash}`
).digest("hex");
}
var unsignedPayload = "UNSIGNED-PAYLOAD";
function createCanonicalDataDigestHostOnly(method, path, query, host) {
return createHash("sha256").update(
`${method}
${path}
${query}
host:${host}
host
UNSIGNED-PAYLOAD`
).digest("hex");
}
function createCanonicalDataDigest(method, path, query, sortedHeaders, contentHashStr) {
const sortedHeaderNames = Object.keys(sortedHeaders);
let canonData = `${method}
${path}
${query}
`;
for (const header of sortedHeaderNames) {
canonData += `${header}:${sortedHeaders[header]}
`;
}
canonData += "\n";
canonData += sortedHeaderNames.length > 0 ? sortedHeaderNames[0] : "";
for (let i = 1; i < sortedHeaderNames.length; ++i) {
canonData += `;${sortedHeaderNames[i]}`;
}
canonData += `
${contentHashStr}`;
return createHash("sha256").update(canonData).digest("hex");
}
function sha256(data) {
return createHash("sha256").update(data).digest();
}
function md5Base64(data) {
return createHash("md5").update(data).digest("base64");
}
// src/KeyCache.ts
var KeyCache = class {
#lastNumericDay = -1;
#keys = /* @__PURE__ */ new Map();
computeIfAbsent(date, region, accessKeyId, secretAccessKey) {
if (date.numericDayStart !== this.#lastNumericDay) {
this.#keys.clear();
this.#lastNumericDay = date.numericDayStart;
}
const cacheKey = `${date.date}:${region}:${accessKeyId}`;
const key = this.#keys.get(cacheKey);
if (key) {
return key;
}
const newKey = deriveSigningKey(date.date, region, secretAccessKey);
this.#keys.set(cacheKey, newKey);
return newKey;
}
};
// src/AmzDate.ts
var ONE_DAY = 1e3 * 60 * 60 * 24;
function getAmzDate(dateTime) {
const date = pad4(dateTime.getUTCFullYear()) + pad2(dateTime.getUTCMonth() + 1) + pad2(dateTime.getUTCDate());
const time = pad2(dateTime.getUTCHours()) + pad2(dateTime.getUTCMinutes()) + pad2(dateTime.getUTCSeconds());
return {
numericDayStart: dateTime.getTime() / ONE_DAY | 0,
date,
dateTime: `${date}T${time}Z`
};
}
function now() {
return getAmzDate(/* @__PURE__ */ new Date());
}
function pad4(v) {
return v < 10 ? `000${v}` : v < 100 ? `00${v}` : v < 1e3 ? `0${v}` : v.toString();
}
function pad2(v) {
return v < 10 ? `0${v}` : v.toString();
}
// src/url.ts
function buildRequestUrl(endpoint, bucket, region, path) {
const normalizedBucket = normalizePath(bucket);
const [endpointWithBucketAndRegion, replacedBucket] = replaceDomainPlaceholders(endpoint, normalizedBucket, region);
const result = new URL(endpointWithBucketAndRegion);
const pathPrefix = result.pathname.endsWith("/") ? result.pathname : `${result.pathname}/`;
const pathSuffix = replacedBucket ? normalizePath(path) : `${normalizedBucket}/${normalizePath(path)}`;
result.pathname = pathPrefix + pathSuffix.replaceAll(":", "%3A").replaceAll("+", "%2B").replaceAll("(", "%28").replaceAll(")", "%29").replaceAll(",", "%2C").replaceAll("'", "%27").replaceAll("*", "%2A");
return result;
}
function replaceDomainPlaceholders(endpoint, bucket, region) {
const replacedBucket = endpoint.includes("{bucket}");
return [
endpoint.replaceAll("{bucket}", bucket).replaceAll("{region}", region),
replacedBucket
];
}
function normalizePath(path) {
const start = path[0] === "/" ? 1 : 0;
const end = path[path.length - 1] === "/" ? path.length - 1 : path.length;
return path.substring(start, end);
}
function prepareHeadersForSigning(unfilteredHeadersUnsorted) {
const result = {};
for (const header of Object.keys(unfilteredHeadersUnsorted).sort()) {
const v = unfilteredHeadersUnsorted[header];
if (v !== void 0 && v !== null) {
result[header] = v;
}
}
return result;
}
function getRangeHeader(start, endExclusive) {
return typeof start === "number" || typeof endExclusive === "number" ? (
// Http-ranges are end-inclusive, we are exclusiv ein our slice
`bytes=${start ?? 0}-${typeof endExclusive === "number" ? endExclusive - 1 : ""}`
) : void 0;
}
// src/error.ts
import { XMLParser } from "fast-xml-parser";
var xmlParser = new XMLParser();
async function getResponseError(response, path) {
let body;
try {
body = await response.body.text();
} catch (cause) {
return new S3Error("Unknown", path, {
message: "Could not read response body.",
cause
});
}
if (response.headers["content-type"] === "application/xml") {
return parseAndGetXmlError(body, path);
}
return new S3Error("Unknown", path, {
message: "Unknown error during S3 request."
});
}
function fromStatusCode(code, path) {
switch (code) {
case 404:
return new S3Error("NoSuchKey", path, {
message: "The specified key does not exist."
});
case 403:
return new S3Error("AccessDenied", path, {
message: "Access denied to the key."
});
// TODO: Add more status codes as needed
default:
return void 0;
}
}
function parseAndGetXmlError(body, path) {
let error;
try {
error = xmlParser.parse(body);
} catch (cause) {
return new S3Error("Unknown", path, {
message: "Could not parse XML error response.",
cause
});
}
if (error.Error) {
const e = error.Error;
return new S3Error(e.Code || "Unknown", path, {
message: e.Message || void 0
// Message might be "",
});
}
return new S3Error(error.Code || "Unknown", path, {
message: error.Message || void 0
// Message might be "",
});
}
// src/request.ts
function getAuthorizationHeader(keyCache, method, path, query, date, sortedSignedHeaders, region, contentHashStr, accessKeyId, secretAccessKey) {
const dataDigest = createCanonicalDataDigest(
method,
path,
query,
sortedSignedHeaders,
contentHashStr
);
const signingKey = keyCache.computeIfAbsent(
date,
region,
accessKeyId,
secretAccessKey
);
const signature = signCanonicalDataHash(
signingKey,
dataDigest,
date,
region
);
const signedHeadersSpec = Object.keys(sortedSignedHeaders).join(";");
const credentialSpec = `${accessKeyId}/${date.date}/${region}/s3/aws4_request`;
return `AWS4-HMAC-SHA256 Credential=${credentialSpec}, SignedHeaders=${signedHeadersSpec}, Signature=${signature}`;
}
// src/branded.ts
function ensureValidBucketName(bucket) {
if (typeof bucket !== "string") {
throw new TypeError("`bucket` is required and must be a `string`.");
}
if (bucket.length < 3 || bucket.length > 63) {
throw new Error("`bucket` must be between 3 and 63 characters long.");
}
if (bucket.startsWith(".") || bucket.endsWith(".")) {
throw new Error("`bucket` must not start or end with a period (.)");
}
if (!/^[a-z0-9.-]+$/.test(bucket)) {
throw new Error(
"`bucket` can only contain lowercase letters, numbers, periods (.), and hyphens (-)."
);
}
if (bucket.includes("..")) {
throw new Error("`bucket` must not contain two adjacent periods (..)");
}
return bucket;
}
function ensureValidAccessKeyId(accessKeyId) {
if (typeof accessKeyId !== "string") {
throw new TypeError("`AccessKeyId` is required and must be a `string`.");
}
if (accessKeyId.length < 1) {
throw new RangeError("`AccessKeyId` must be at least 1 character long.");
}
return accessKeyId;
}
function ensureValidSecretAccessKey(secretAccessKey) {
if (typeof secretAccessKey !== "string") {
throw new TypeError(
"`SecretAccessKey` is required and must be a `string`."
);
}
if (secretAccessKey.length < 1) {
throw new RangeError(
"`SecretAccessKey` must be at least 1 character long."
);
}
return secretAccessKey;
}
function ensureValidPath(path) {
if (typeof path !== "string") {
throw new TypeError("`path` is required and must be a `string`.");
}
if (path.length < 1) {
throw new RangeError("`path` must be at least 1 character long.");
}
return path;
}
function ensureValidEndpoint(endpoint) {
if (typeof endpoint !== "string") {
throw new TypeError("`endpoint` is required and must be a `string`.");
}
if (endpoint.length < 1) {
throw new RangeError("`endpoint` must be at least 1 character long.");
}
return endpoint;
}
function ensureValidRegion(region) {
if (typeof region !== "string") {
throw new TypeError("`region` is required and must be a `string`.");
}
if (region.length < 1) {
throw new RangeError("`region` must be at least 1 character long.");
}
return region;
}
// src/assertNever.ts
function assertNever(v) {
throw new TypeError(`Expected value not to have type ${typeof v}`);
}
// src/encode.ts
function getContentDispositionHeader(value) {
switch (value.type) {
case "inline":
return "inline";
case "attachment": {
const { filename } = value;
if (typeof filename === "undefined") {
return "attachment";
}
const encoded = encodeURIComponent(filename);
return `attachment;filename="${encoded}";filename*=UTF-8''${encoded}`;
}
default:
assertNever(value);
}
}
function encodeURIComponentExtended(value) {
return encodeURIComponent(value).replaceAll(":", "%3A").replaceAll("+", "%2B").replaceAll("(", "%28").replaceAll(")", "%29").replaceAll(",", "%2C").replaceAll("'", "%27").replaceAll("*", "%2A");
}
// src/S3Client.ts
var kWrite = Symbol("kWrite");
var kStream = Symbol("kStream");
var kSignedRequest = Symbol("kSignedRequest");
var kGetEffectiveParams = Symbol("kGetEffectiveParams");
var xmlParser2 = new XMLParser2({
ignoreAttributes: true,
isArray: (_, jPath) => jPath === "ListMultipartUploadsResult.Upload" || jPath === "ListBucketResult.Contents" || jPath === "ListPartsResult.Part" || jPath === "DeleteResult.Deleted" || jPath === "DeleteResult.Error"
});
var xmlBuilder = new XMLBuilder({
attributeNamePrefix: "$",
ignoreAttributes: false
});
var S3Client = class {
#options;
#keyCache = new KeyCache();
// TODO: pass options to this in client? Do we want to expose the internal use of undici?
#dispatcher = new Agent();
/**
* Create a new instance of an S3 bucket so that credentials can be managed from a single instance instead of being passed to every method.
*
* @param options The default options to use for the S3 client.
*/
constructor(options) {
if (!options) {
throw new Error("`options` is required.");
}
this.#options = {
accessKeyId: ensureValidAccessKeyId(options.accessKeyId),
secretAccessKey: ensureValidSecretAccessKey(options.secretAccessKey),
endpoint: ensureValidEndpoint(options.endpoint),
region: ensureValidRegion(options.region),
bucket: ensureValidBucketName(options.bucket),
sessionToken: options.sessionToken
};
}
/** @internal */
[kGetEffectiveParams](options) {
return [
options.region ? ensureValidRegion(options.region) : this.#options.region,
options.endpoint ? ensureValidEndpoint(options.endpoint) : this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket
];
}
/**
* Creates an S3File instance for the given path.
*
* @param {string} path The path to the object in the bucket. Also known as [object key](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html).
* We recommend not using the following characters in a key name because of significant special character handling, which isn't consistent across all applications (see [AWS docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html)):
* - Backslash (`\\`)
* - Left brace (`{`)
* - Non-printable ASCII characters (128–255 decimal characters)
* - Caret or circumflex (`^`)
* - Right brace (`}`)
* - Percent character (`%`)
* - Grave accent or backtick (`\``)
* - Right bracket (`]`)
* - Quotation mark (`"`)
* - Greater than sign (`>`)
* - Left bracket (`[`)
* - Tilde (`~`)
* - Less than sign (`<`)
* - Pound sign (`#`)
* - Vertical bar or pipe (`|`)
*
* lean-s3 does not enforce these restrictions.
*
* @example
* ```js
* const file = client.file("image.jpg");
* await file.write(imageData);
*
* const configFile = client.file("config.json", {
* type: "application/json",
* });
* ```
*/
file(path, options = {}) {
return new S3File(
this,
ensureValidPath(path),
void 0,
void 0,
options.type ?? void 0
);
}
/**
* Generate a presigned URL for temporary access to a file.
* Useful for generating upload/download URLs without exposing credentials.
* @returns The operation on {@link S3Client#presign.path} as a pre-signed URL.
*
* @example
* ```js
* const downloadUrl = client.presign("file.pdf", {
* expiresIn: 3600 // 1 hour
* });
* ```
*
* @example
* ```js
* client.presign("foo.jpg", {
* expiresIn: 3600 // 1 hour
* response: {
* contentDisposition: {
* type: "attachment",
* filename: "download.jpg",
* },
* },
* });
* ```
*/
presign(path, options = {}) {
const contentLength = options.contentLength ?? void 0;
if (typeof contentLength === "number") {
if (contentLength < 0) {
throw new RangeError("`contentLength` must be >= 0.");
}
}
const method = options.method ?? "GET";
const contentType = options.type ?? void 0;
const [region, endpoint, bucket] = this[kGetEffectiveParams](options);
const responseOptions = options.response;
const contentDisposition = responseOptions?.contentDisposition;
const responseContentDisposition = contentDisposition ? getContentDispositionHeader(contentDisposition) : void 0;
const res = buildRequestUrl(
endpoint,
bucket,
region,
ensureValidPath(path)
);
const now2 = /* @__PURE__ */ new Date();
const date = getAmzDate(now2);
const query = buildSearchParams(
`${this.#options.accessKeyId}/${date.date}/${region}/s3/aws4_request`,
date,
options.expiresIn ?? 3600,
typeof contentLength === "number" || typeof contentType === "string" ? typeof contentLength === "number" && typeof contentType === "string" ? "content-length;content-type;host" : typeof contentLength === "number" ? "content-length;host" : typeof contentType === "string" ? "content-type;host" : "" : "host",
unsignedPayload,
options.storageClass,
this.#options.sessionToken,
options.acl,
responseContentDisposition
);
const dataDigest = typeof contentLength === "number" || typeof contentType === "string" ? createCanonicalDataDigest(
method,
res.pathname,
query,
typeof contentLength === "number" && typeof contentType === "string" ? {
"content-length": String(contentLength),
"content-type": contentType,
host: res.host
} : typeof contentLength === "number" ? { "content-length": String(contentLength), host: res.host } : typeof contentType === "string" ? { "content-type": contentType, host: res.host } : {},
unsignedPayload
) : createCanonicalDataDigestHostOnly(
method,
res.pathname,
query,
res.host
);
const signingKey = this.#keyCache.computeIfAbsent(
date,
region,
this.#options.accessKeyId,
this.#options.secretAccessKey
);
const signature = signCanonicalDataHash(
signingKey,
dataDigest,
date,
region
);
res.search = `${query}&X-Amz-Signature=${signature}`;
return res.toString();
}
//#region multipart uploads
async createMultipartUpload(key, options = {}) {
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"POST",
ensureValidPath(key),
"uploads=",
void 0,
void 0,
void 0,
void 0,
options.signal
);
if (response.statusCode !== 200) {
throw await getResponseError(response, key);
}
const text = await response.body.text();
const res = ensureParsedXml(text).InitiateMultipartUploadResult ?? {};
return {
bucket: res.Bucket,
key: res.Key,
uploadId: res.UploadId
};
}
/**
* @remarks Uses [`ListMultipartUploads`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html).
* @throws {RangeError} If `options.maxKeys` is not between `1` and `1000`.
*/
async listMultipartUploads(options = {}) {
let query = "uploads=";
if (options.delimiter) {
if (typeof options.delimiter !== "string") {
throw new TypeError("`delimiter` must be a `string`.");
}
query += `&delimiter=${encodeURIComponent(options.delimiter)}`;
}
if (options.keyMarker) {
if (typeof options.keyMarker !== "string") {
throw new TypeError("`keyMarker` must be a `string`.");
}
query += `&key-marker=${encodeURIComponent(options.keyMarker)}`;
}
if (typeof options.maxUploads !== "undefined") {
if (typeof options.maxUploads !== "number") {
throw new TypeError("`maxUploads` must be a `number`.");
}
if (options.maxUploads < 1 || options.maxUploads > 1e3) {
throw new RangeError("`maxUploads` has to be between 1 and 1000.");
}
query += `&max-uploads=${options.maxUploads}`;
}
if (options.prefix) {
if (typeof options.prefix !== "string") {
throw new TypeError("`prefix` must be a `string`.");
}
query += `&prefix=${encodeURIComponent(options.prefix)}`;
}
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"GET",
"",
query,
void 0,
void 0,
void 0,
void 0,
options.signal
);
if (response.statusCode !== 200) {
throw await getResponseError(response, "");
}
const text = await response.body.text();
const root = ensureParsedXml(text).ListMultipartUploadsResult ?? {};
return {
bucket: root.Bucket || void 0,
delimiter: root.Delimiter || void 0,
prefix: root.Prefix || void 0,
keyMarker: root.KeyMarker || void 0,
uploadIdMarker: root.UploadIdMarker || void 0,
nextKeyMarker: root.NextKeyMarker || void 0,
nextUploadIdMarker: root.NextUploadIdMarker || void 0,
maxUploads: root.MaxUploads ?? 1e3,
// not using || to not override 0; caution: minio supports 10000(!)
isTruncated: root.IsTruncated === "true",
uploads: root.Upload?.map(
// biome-ignore lint/suspicious/noExplicitAny: we're parsing here
(u) => ({
key: u.Key || void 0,
uploadId: u.UploadId || void 0,
// TODO: Initiator
// TODO: Owner
storageClass: u.StorageClass || void 0,
checksumAlgorithm: u.ChecksumAlgorithm || void 0,
checksumType: u.ChecksumType || void 0,
initiated: u.Initiated ? new Date(u.Initiated) : void 0
})
) ?? []
};
}
/**
* @remarks Uses [`AbortMultipartUpload`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html).
* @throws {RangeError} If `key` is not at least 1 character long.
* @throws {Error} If `uploadId` is not provided.
*/
async abortMultipartUpload(path, uploadId, options = {}) {
if (!uploadId) {
throw new Error("`uploadId` is required.");
}
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"DELETE",
ensureValidPath(path),
`uploadId=${encodeURIComponent(uploadId)}`,
void 0,
void 0,
void 0,
void 0,
options.signal
);
if (response.statusCode !== 204 && response.statusCode !== 200) {
throw await getResponseError(response, path);
}
}
/**
* @remarks Uses [`CompleteMultipartUpload`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html).
* @throws {RangeError} If `key` is not at least 1 character long.
* @throws {Error} If `uploadId` is not provided.
*/
async completeMultipartUpload(path, uploadId, parts, options = {}) {
if (!uploadId) {
throw new Error("`uploadId` is required.");
}
const body = xmlBuilder.build({
CompleteMultipartUpload: {
Part: parts.map((part) => ({
PartNumber: part.partNumber,
ETag: part.etag
}))
}
});
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"POST",
ensureValidPath(path),
`uploadId=${encodeURIComponent(uploadId)}`,
body,
void 0,
void 0,
void 0,
options.signal
);
if (response.statusCode !== 200) {
throw await getResponseError(response, path);
}
const text = await response.body.text();
const res = ensureParsedXml(text).CompleteMultipartUploadResult ?? {};
return {
location: res.Location || void 0,
bucket: res.Bucket || void 0,
key: res.Key || void 0,
etag: res.ETag || void 0,
checksumCRC32: res.ChecksumCRC32 || void 0,
checksumCRC32C: res.ChecksumCRC32C || void 0,
checksumCRC64NVME: res.ChecksumCRC64NVME || void 0,
checksumSHA1: res.ChecksumSHA1 || void 0,
checksumSHA256: res.ChecksumSHA256 || void 0,
checksumType: res.ChecksumType || void 0
};
}
/**
* @remarks Uses [`UploadPart`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html).
* @throws {RangeError} If `key` is not at least 1 character long.
* @throws {Error} If `uploadId` is not provided.
*/
async uploadPart(path, uploadId, data, partNumber, options = {}) {
if (!uploadId) {
throw new Error("`uploadId` is required.");
}
if (!data) {
throw new Error("`data` is required.");
}
if (typeof partNumber !== "number" || partNumber <= 0) {
throw new Error("`partNumber` has to be a `number` which is >= 1.");
}
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"PUT",
ensureValidPath(path),
`partNumber=${partNumber}&uploadId=${encodeURIComponent(uploadId)}`,
data,
void 0,
void 0,
void 0,
options.signal
);
if (response.statusCode === 200) {
response.body.dump();
const etag = response.headers.etag;
if (typeof etag !== "string" || etag.length === 0) {
throw new S3Error("Unknown", "", {
message: "Response did not contain an etag."
});
}
return {
partNumber,
etag
};
}
throw await getResponseError(response, "");
}
/**
* @remarks Uses [`ListParts`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html).
* @throws {RangeError} If `key` is not at least 1 character long.
* @throws {Error} If `uploadId` is not provided.
* @throws {TypeError} If `options.maxParts` is not a `number`.
* @throws {RangeError} If `options.maxParts` is <= 0.
* @throws {TypeError} If `options.partNumberMarker` is not a `string`.
*/
async listParts(path, uploadId, options = {}) {
let query = "";
if (options.maxParts) {
if (typeof options.maxParts !== "number") {
throw new TypeError("`maxParts` must be a `number`.");
}
if (options.maxParts <= 0) {
throw new RangeError("`maxParts` must be >= 1.");
}
query += `&max-parts=${options.maxParts}`;
}
if (options.partNumberMarker) {
if (typeof options.partNumberMarker !== "string") {
throw new TypeError("`partNumberMarker` must be a `string`.");
}
query += `&part-number-marker=${encodeURIComponent(options.partNumberMarker)}`;
}
query += `&uploadId=${encodeURIComponent(uploadId)}`;
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"GET",
ensureValidPath(path),
// We always have a leading &, so we can slice the leading & away (this way, we have less conditionals on the hot path); see benchmark-operations.js
query.substring(1),
void 0,
void 0,
void 0,
void 0,
options?.signal
);
if (response.statusCode === 200) {
const text = await response.body.text();
const root = ensureParsedXml(text).ListPartsResult ?? {};
return {
bucket: root.Bucket,
key: root.Key,
uploadId: root.UploadId,
partNumberMarker: root.PartNumberMarker ?? void 0,
nextPartNumberMarker: root.NextPartNumberMarker ?? void 0,
maxParts: root.MaxParts ?? 1e3,
isTruncated: root.IsTruncated ?? false,
parts: (
// biome-ignore lint/suspicious/noExplicitAny: parsing code
root.Part?.map((part) => ({
etag: part.ETag,
lastModified: part.LastModified ? new Date(part.LastModified) : void 0,
partNumber: part.PartNumber ?? void 0,
size: part.Size ?? void 0
})) ?? []
)
};
}
throw await getResponseError(response, path);
}
//#endregion
//#region bucket operations
/**
* Creates a new bucket on the S3 server.
*
* @param name The name of the bucket to create. AWS the name according to [some rules](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html). The most important ones are:
* - Bucket names must be between `3` (min) and `63` (max) characters long.
* - Bucket names can consist only of lowercase letters, numbers, periods (`.`), and hyphens (`-`).
* - Bucket names must begin and end with a letter or number.
* - Bucket names must not contain two adjacent periods.
* - Bucket names must not be formatted as an IP address (for example, `192.168.5.4`).
*
* @throws {Error} If the bucket name is invalid.
* @throws {S3Error} If the bucket could not be created, e.g. if it already exists.
* @remarks Uses [`CreateBucket`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html)
*/
async createBucket(name, options = {}) {
let body;
if (options) {
const location = options.location && (options.location.name || options.location.type) ? {
Name: options.location.name ?? void 0,
Type: options.location.type ?? void 0
} : void 0;
const bucket = options.info && (options.info.dataRedundancy || options.info.type) ? {
DataRedundancy: options.info.dataRedundancy ?? void 0,
Type: options.info.type ?? void 0
} : void 0;
body = location || bucket || options.locationConstraint ? xmlBuilder.build({
CreateBucketConfiguration: {
$xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
LocationConstraint: options.locationConstraint ?? void 0,
Location: location,
Bucket: bucket
}
}) : void 0;
}
const additionalSignedHeaders = body ? { "content-md5": md5Base64(body) } : void 0;
const response = await this[kSignedRequest](
options.region ? ensureValidRegion(options.region) : this.#options.region,
options.endpoint ? ensureValidEndpoint(options.endpoint) : this.#options.endpoint,
ensureValidBucketName(name),
"PUT",
"",
void 0,
body,
additionalSignedHeaders,
void 0,
void 0,
options.signal
);
if (400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
response.body.dump();
if (response.statusCode === 200) {
return;
}
throw new Error(`Response code not supported: ${response.statusCode}`);
}
/**
* Deletes a bucket from the S3 server.
* @param name The name of the bucket to delete. Same restrictions as in {@link S3Client#createBucket}.
* @throws {Error} If the bucket name is invalid.
* @throws {S3Error} If the bucket could not be deleted, e.g. if it is not empty.
* @remarks Uses [`DeleteBucket`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html).
*/
async deleteBucket(name, options) {
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
ensureValidBucketName(name),
"DELETE",
"",
void 0,
void 0,
void 0,
void 0,
void 0,
options?.signal
);
if (400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
response.body.dump();
if (response.statusCode === 204) {
return;
}
throw new Error(`Response code not supported: ${response.statusCode}`);
}
/**
* Checks if a bucket exists.
* @param name The name of the bucket to delete. Same restrictions as in {@link S3Client#createBucket}.
* @throws {Error} If the bucket name is invalid.
* @remarks Uses [`HeadBucket`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html).
*/
async bucketExists(name, options) {
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
ensureValidBucketName(name),
"HEAD",
"",
void 0,
void 0,
void 0,
void 0,
void 0,
options?.signal
);
if (response.statusCode !== 404 && 400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
response.body.dump();
if (response.statusCode === 200) {
return true;
}
if (response.statusCode === 404) {
return false;
}
throw new Error(`Response code not supported: ${response.statusCode}`);
}
//#region bucket cors
/**
* @remarks Uses [`PutBucketCors`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketCors.html).
*/
async putBucketCors(rules, options = {}) {
const body = xmlBuilder.build({
CORSConfiguration: {
CORSRule: rules.map((r) => ({
AllowedOrigin: r.allowedOrigins,
AllowedMethod: r.allowedMethods,
ExposeHeader: r.exposeHeaders,
ID: r.id ?? void 0,
MaxAgeSeconds: r.maxAgeSeconds ?? void 0
}))
}
});
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"PUT",
"",
"cors=",
// "=" is needed by minio for some reason
body,
{
"content-md5": md5Base64(body)
},
void 0,
void 0,
options.signal
);
if (response.statusCode === 200) {
response.body.dump();
return;
}
if (400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
throw new Error(
`Response code not implemented yet: ${response.statusCode}`
);
}
/**
* @remarks Uses [`GetBucketCors`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketCors.html).
*/
async getBucketCors(options = {}) {
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"GET",
"",
"cors=",
// "=" is needed by minio for some reason
void 0,
void 0,
void 0,
void 0,
options.signal
);
if (response.statusCode !== 200) {
response.body.dump();
throw fromStatusCode(response.statusCode, "");
}
throw new Error("Not implemented");
}
/**
* @remarks Uses [`DeleteBucketCors`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketCors.html).
*/
async deleteBucketCors(options = {}) {
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"DELETE",
"",
"cors=",
// "=" is needed by minio for some reason
void 0,
void 0,
void 0,
void 0,
options.signal
);
if (response.statusCode !== 204) {
response.body.dump();
throw fromStatusCode(response.statusCode, "");
}
}
//#endregion
//#region list objects
/**
* Uses [`ListObjectsV2`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) to iterate over all keys. Pagination and continuation is handled internally.
*/
async *listIterating(options) {
const maxKeys = options?.internalPageSize ?? void 0;
let continuationToken;
do {
const res = await this.list({
...options,
maxKeys,
continuationToken
});
if (!res || res.contents.length === 0) {
break;
}
yield* res.contents;
continuationToken = res.nextContinuationToken;
} while (continuationToken);
}
/**
* Implements [`ListObjectsV2`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) to iterate over all keys.
*
* @throws {RangeError} If `maxKeys` is not between `1` and `1000`.
*/
async list(options = {}) {
let query = "";
if (typeof options.continuationToken !== "undefined") {
if (typeof options.continuationToken !== "string") {
throw new TypeError("`continuationToken` must be a `string`.");
}
query += `continuation-token=${encodeURIComponent(options.continuationToken)}&`;
}
query += "list-type=2";
if (typeof options.maxKeys !== "undefined") {
if (typeof options.maxKeys !== "number") {
throw new TypeError("`maxKeys` must be a `number`.");
}
if (options.maxKeys < 1 || options.maxKeys > 1e3) {
throw new RangeError("`maxKeys` has to be between 1 and 1000.");
}
query += `&max-keys=${options.maxKeys}`;
}
if (typeof options.delimiter !== "undefined") {
if (typeof options.delimiter !== "string") {
throw new TypeError("`delimiter` must be a `string`.");
}
query += `&delimiter=${options.delimiter === "/" ? "/" : encodeURIComponent(options.delimiter)}`;
}
if (options.prefix) {
if (typeof options.prefix !== "string") {
throw new TypeError("`prefix` must be a `string`.");
}
query += `&prefix=${encodeURIComponent(options.prefix)}`;
}
if (typeof options.startAfter !== "undefined") {
if (typeof options.startAfter !== "string") {
throw new TypeError("`startAfter` must be a `string`.");
}
query += `&start-after=${encodeURIComponent(options.startAfter)}`;
}
const response = await this[kSignedRequest](
ensureValidRegion(this.#options.region),
ensureValidEndpoint(this.#options.endpoint),
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"GET",
"",
query,
void 0,
void 0,
void 0,
void 0,
options.signal
);
if (response.statusCode !== 200) {
response.body.dump();
throw new Error(
`Response code not implemented yet: ${response.statusCode}`
);
}
const text = await response.body.text();
const res = ensureParsedXml(text).ListBucketResult ?? {};
if (!res) {
throw new S3Error("Unknown", "", {
message: "Could not read bucket contents."
});
}
return {
name: res.Name,
prefix: res.Prefix,
startAfter: res.StartAfter,
isTruncated: res.IsTruncated,
continuationToken: res.ContinuationToken,
maxKeys: res.MaxKeys,
keyCount: res.KeyCount,
nextContinuationToken: res.NextContinuationToken,
contents: res.Contents?.map(S3BucketEntry.parse) ?? []
};
}
//#endregion
/**
* Uses [`DeleteObjects`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) to delete multiple objects in a single request.
*/
async deleteObjects(objects, options = {}) {
const body = xmlBuilder.build({
Delete: {
Quiet: true,
Object: objects.map((o) => ({
Key: typeof o === "string" ? o : o.key
}))
}
});
const response = await this[kSignedRequest](
this.#options.region,
this.#options.endpoint,
options.bucket ? ensureValidBucketName(options.bucket) : this.#options.bucket,
"POST",
"",
"delete=",
// "=" is needed by minio for some reason
body,
{
"content-md5": md5Base64(body)
},
void 0,
void 0,
options.signal
);
if (response.statusCode === 200) {
const text = await response.body.text();
let deleteResult;
try {
deleteResult = ensureParsedXml(text).DeleteResult ?? {};
} catch (cause) {
throw new S3Error("Unknown", "", {
message: "S3 service responded with invalid XML.",
cause
});
}
const errors = (
// biome-ignore lint/suspicious/noExplicitAny: parsing
deleteResult.Error?.map((e) => ({
code: e.Code,
key: e.Key,
message: e.Message,
versionId: e.VersionId
})) ?? []
);
return { errors };
}
if (400 <= response.statusCode && response.statusCode < 500) {
throw await getResponseError(response, "");
}
response.body.dump();
throw new Error(
`Response code not implemented yet: ${response.statusCode}`
);
}
/**
* Do not use this. This is an internal method.
* TODO: Maybe move this into a separate free function?
* @internal
*/
async [kSignedRequest](region, endpoint, bucket, method, pathWithoutBucket, query, body, additionalSignedHeaders, additionalUnsignedHeaders, contentHash, signal) {
const url = buildRequestUrl(endpoint, bucket, region, pathWithoutBucket);
if (query) {
url.search = query;
}
const now2 = now();
const contentHashStr = contentHash?.toString("hex") ?? unsignedPayload;
const headersToBeSigned = prepareHeadersForSigning({
host: url.host,
"x-amz-date": now2.dateTime,
"x-amz-content-sha256": contentHashStr,
...additionalSignedHeaders
});
try {
return await request(url, {
method,
signal,
dispatcher: this.#dispatcher,
headers: {
...headersToBeSigned,
authorization: getAuthorizationHeader(
this.#keyCache,
method,
url.pathname,
query ?? "",
now2,
headersToBeSigned,
region,
contentHashStr,
this.#options.accessKeyId,
this.#options.secretAccessKey
),
...additionalUnsignedHeaders,
"user-agent": "lean-s3"
},
body
});
} catch (cause) {
signal?.throwIfAborted();
throw new S3Error("Unknown", pathWithoutBucket, {
message: "Unknown error during S3 request.",
cause
});
}
}
/** @internal */
async [kWrite](path, data, contentType, contentLength, contentHash, rageStart, rangeEndExclusive, signal = void 0) {
const bucket = this.#options.bucket;
const endpoint = this.#options.endpoint;
const region = this.#options.region;
const url = buildRequestUrl(endpoint, bucket, region, path);
const now2 = now();
const contentHashStr = contentHash?.toString("hex") ?? unsignedPayload;
const headersToBeSigned = prepareHeadersForSigning({
"content-length": contentLength?.toString() ?? void 0,
"content-type": contentType,
host: url.host,
range: getRangeHeader(rageStart, rangeEndExclusive),
"x-amz-content-sha256": contentHashStr,
"x-amz-date": now2.dateTime
});
let response;
try {
response = await request(url, {
method: "PUT",
signal,
dispatcher: this.#dispatcher,
headers: {
...headersToBeSigned,
authorization: getAuthorizationHeader(
this.#keyCache,
"PUT",
url.pathname,
url.search,
now2,
headersToBeSigned,
region,
contentHashStr,
this.#options.accessKeyId,
this.#options.secretAccessKey
),
"user-agent": "lean-s3"
},
body: data
});
} catch (cause) {
signal?.throwIfAborted();
throw new S3Error("Unknown", path, {
message: "Unknown error during S3 request.",
cause
});
}
const status = response.statusCode;
if (200 <= status && status < 300) {
return;
}
throw await getResponseError(response, path);
}
// TODO: Support abortSignal
/**
* @internal
*/
[kStream](path, contentHash, rageStart, rangeEndExclusive) {
const bucket = this.#options.bucket;
const endpoint = this.#options.endpoint;
const region = this.#options.region;
const now2 = now();
const url = buildRequestUrl(endpoint, bucket, region, path);
const range = getRangeHeader(rageStart, rangeEndExclusive);
const contentHashStr = contentHash?.toString("hex") ?? unsignedPayload;
const headersToBeSigned = prepareHeadersForSigning({
"amz-sdk-invocation-id": crypto.randomUUID(),
host: url.host,
range,
// Hetzner doesnt care if the x-amz-content-sha256 header is missing, R2 requires it to be present
"x-amz-content-sha256": contentHashStr,
"x-amz-date": now2.dateTime
});
const ac = new AbortController();
return new ReadableStream({
type: "bytes",
start: (controller) => {
const onNetworkError = (cause) => {
controller.error(
new S3Error("Unknown", path, {
message: void 0,
cause
})
);
};
request(url, {
method: "GET",
signal: ac.signal,
dispatcher: this.#dispatcher,
headers: {
...headersToBeSigned,
authorization: getAuthorizationHeader(
this.#keyCache,
"GET",
url.pathname,
url.search,
now2,
headersToBeSigned,
region,
contentHashStr,
this.#options.accessKeyId,
this.#options.secretAccessKey
),
"user-agent": "lean-s3"
}
}).then((response) => {
const onData = controller.enqueue.bind(controller);
const onClose = controller.close.bind(controller);
const expectPartialResponse = range !== void 0;
const status = response.statusCode;
if (status === 200) {
if (expectPartialResponse) {
return controller.error(
new S3Error("Unknown", path, {
message: "Expected partial response to range request."
})
);
}
response.body.on("data", onData);
response.body.once("error", onNetworkError);
response.body.once("end", onClose);
return;
}
if (status === 206) {
if (!expectPartialResponse) {
return controller.error(
new S3Error("Unknown", path, {
message: "Received partial response but expected a full response."
})
);
}
response.body.on("data", onData);
response.body.once("error", onNetworkError);
response.body.once("end", onClose);
return;
}
if (400 <= status && status < 500) {
const responseText = void 0;
if (response.headers["content-type"] === "application/xml") {
return response.body.text().then((body) => {
let error;
try {
error = xmlParser2.parse(body);
} catch (cause) {
return controller.error(
new S3Error("Unknown", path, {
message: "Could not parse XML error response.",
cause
})
);
}
return controller.error(
new S3Error(error.Code || "Unknown", path, {
message: error.Message || void 0
// Message might be "",
})
);
}, onNetworkError);
}
return controller.error(
new S3Error("Unknown", path, {
message: void 0,
cause: responseText
})
);
}
return controller.error(
new Error(
`Handling for status code ${status} not implemented yet. You might want to open an issue and describe your situation.`
)
);
}, onNetworkError);
},
cancel(reason) {
ac.abort(reason);
}
});
}
};
function buildSearchParams(amzCredential, date, expiresIn, headerList, contentHashStr, storageClass, sessionToken, acl, responseContentDisposition) {
let res = "";
if (acl) {
res += `X-Amz-Acl=${encodeURIComponent(acl)}&`;
}
res += "X-Amz-Algorithm=AWS4-HMAC-SHA256";
if (contentHashStr) {
res += `&X-Amz-Content-Sha256=${contentHashStr}`;
}
res += `&X-Amz-Credential=${encodeURIComponentExtended(amzCredential)}`;
res += `&X-Amz-Date=${date.dateTime}`;
res += `&X-Amz-Expires=${expiresIn}`;
if (sessionToken) {
res += `&X-Amz-Security-Token=${encodeURIComponent(sessionToken)}`;
}
res += `&X-Amz-SignedHeaders=${encodeURIComponent(headerList)}`;
if (storageClass) {
res += `&X-Amz-Storage-Class=${storageClass}`;
}
if (responseContentDisposition) {
res += `&response-content-disposition=${encodeURIComponentExtended(responseContentDisposition)}`;
}
return res;
}
function ensureParsedXml(text) {
try {
const r = xmlParser2.parse(text);
if (!r) {
throw new S3Error("Unknown", "", {
message: "S3 service responded with empty XML."
});
}
return r;
} catch (cause) {
throw new S3Error("Unknown", "", {
message: "S3 service responded with invalid XML.",
cause
});
}
}
// src/S3File.ts
var S3File = class _S3File {
#client;
#path;
#start;
#end;
#contentType;
/**