UNPKG

miniflare

Version:

Fun, full-featured, fully-local simulator for Cloudflare Workers

1,135 lines (1,127 loc) • 43.9 kB
var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; var __decorateClass = (decorators, target, key, kind) => { for (var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target, i = decorators.length - 1, decorator; i >= 0; i--) (decorator = decorators[i]) && (result = (kind ? decorator(target, key, result) : decorator(result)) || result); return kind && result && __defProp(target, key, result), result; }; // src/workers/r2/bucket.worker.ts import assert2 from "node:assert"; import { Buffer as Buffer3 } from "node:buffer"; import { createHash } from "node:crypto"; import { all, base64Decode, base64Encode, DeferredPromise, GET, get, maybeApply, MiniflareDurableObject, PUT, readPrefix, WaitGroup } from "miniflare:shared"; // src/workers/r2/constants.ts var R2Limits = { MAX_LIST_KEYS: 1e3, MAX_KEY_SIZE: 1024, // https://developers.cloudflare.com/r2/platform/limits/ MAX_VALUE_SIZE: 5363466240, // 5 GiB - 5 MiB MAX_METADATA_SIZE: 2048, // 2048 B MIN_MULTIPART_PART_SIZE: 5242880, MIN_MULTIPART_PART_SIZE_TEST: 50 }, R2Headers = { ERROR: "cf-r2-error", REQUEST: "cf-r2-request", METADATA_SIZE: "cf-r2-metadata-size" }; // src/workers/r2/errors.worker.ts import { HttpError } from "miniflare:shared"; var R2ErrorCode = { INTERNAL_ERROR: 10001, NO_SUCH_OBJECT_KEY: 10007, ENTITY_TOO_LARGE: 100100, ENTITY_TOO_SMALL: 10011, METADATA_TOO_LARGE: 10012, INVALID_OBJECT_NAME: 10020, INVALID_MAX_KEYS: 10022, NO_SUCH_UPLOAD: 10024, INVALID_PART: 10025, INVALID_ARGUMENT: 10029, PRECONDITION_FAILED: 10031, BAD_DIGEST: 10037, INVALID_RANGE: 10039, BAD_UPLOAD: 10048 }, R2Error = class extends HttpError { constructor(code, message, v4Code) { super(code, message); this.v4Code = v4Code; } object; toResponse() { if (this.object !== void 0) { let { metadataSize, value } = this.object.encode(); return new Response(value, { status: this.code, headers: { [R2Headers.METADATA_SIZE]: `${metadataSize}`, "Content-Type": "application/json", [R2Headers.ERROR]: JSON.stringify({ message: this.message, version: 1, // Note the lowercase 'c', which the runtime expects v4code: this.v4Code }) } }); } return new Response(null, { status: this.code, headers: { [R2Headers.ERROR]: JSON.stringify({ message: this.message, version: 1, // Note the lowercase 'c', which the runtime expects v4code: this.v4Code }) } }); } context(info) { return this.message += ` (${info})`, this; } attach(object) { return this.object = object, this; } }, InvalidMetadata = class extends R2Error { constructor() { super(400, "Metadata missing or invalid", R2ErrorCode.INVALID_ARGUMENT); } }, InternalError = class extends R2Error { constructor() { super( 500, "We encountered an internal error. Please try again.", R2ErrorCode.INTERNAL_ERROR ); } }, NoSuchKey = class extends R2Error { constructor() { super( 404, "The specified key does not exist.", R2ErrorCode.NO_SUCH_OBJECT_KEY ); } }, EntityTooLarge = class extends R2Error { constructor() { super( 400, "Your proposed upload exceeds the maximum allowed object size.", R2ErrorCode.ENTITY_TOO_LARGE ); } }, EntityTooSmall = class extends R2Error { constructor() { super( 400, "Your proposed upload is smaller than the minimum allowed object size.", R2ErrorCode.ENTITY_TOO_SMALL ); } }, MetadataTooLarge = class extends R2Error { constructor() { super( 400, "Your metadata headers exceed the maximum allowed metadata size.", R2ErrorCode.METADATA_TOO_LARGE ); } }, BadDigest = class extends R2Error { constructor(algorithm, provided, calculated) { super( 400, [ `The ${algorithm} checksum you specified did not match what we received.`, `You provided a ${algorithm} checksum with value: ${provided.toString( "hex" )}`, `Actual ${algorithm} was: ${calculated.toString("hex")}` ].join(` `), R2ErrorCode.BAD_DIGEST ); } }, InvalidObjectName = class extends R2Error { constructor() { super( 400, "The specified object name is not valid.", R2ErrorCode.INVALID_OBJECT_NAME ); } }, InvalidMaxKeys = class extends R2Error { constructor() { super( 400, "MaxKeys params must be positive integer <= 1000.", R2ErrorCode.INVALID_MAX_KEYS ); } }, NoSuchUpload = class extends R2Error { constructor() { super( 400, "The specified multipart upload does not exist.", R2ErrorCode.NO_SUCH_UPLOAD ); } }, InvalidPart = class extends R2Error { constructor() { super( 400, "One or more of the specified parts could not be found.", R2ErrorCode.INVALID_PART ); } }, PreconditionFailed = class extends R2Error { constructor() { super( 412, "At least one of the pre-conditions you specified did not hold.", R2ErrorCode.PRECONDITION_FAILED ); } }, InvalidRange = class extends R2Error { constructor() { super( 416, "The requested range is not satisfiable", R2ErrorCode.INVALID_RANGE ); } }, BadUpload = class extends R2Error { constructor() { super( 500, "There was a problem with the multipart upload.", R2ErrorCode.BAD_UPLOAD ); } }; // src/workers/r2/r2Object.worker.ts import { HEX_REGEXP } from "miniflare:zod"; var InternalR2Object = class { key; version; size; etag; uploaded; httpMetadata; customMetadata; range; checksums; constructor(row, range) { this.key = row.key, this.version = row.version, this.size = row.size, this.etag = row.etag, this.uploaded = row.uploaded, this.httpMetadata = JSON.parse(row.http_metadata), this.customMetadata = JSON.parse(row.custom_metadata), this.range = range; let checksums = JSON.parse(row.checksums); this.etag.length === 32 && HEX_REGEXP.test(this.etag) && (checksums.md5 = row.etag), this.checksums = checksums; } // Format for return to the Workers Runtime #rawProperties() { return { name: this.key, version: this.version, size: this.size, etag: this.etag, uploaded: this.uploaded, httpFields: this.httpMetadata, customFields: Object.entries(this.customMetadata).map(([k, v]) => ({ k, v })), range: this.range, checksums: { 0: this.checksums.md5, 1: this.checksums.sha1, 2: this.checksums.sha256, 3: this.checksums.sha384, 4: this.checksums.sha512 } }; } encode() { let json = JSON.stringify(this.#rawProperties()), blob = new Blob([json]); return { metadataSize: blob.size, value: blob.stream(), size: blob.size }; } static encodeMultiple(objects) { let json = JSON.stringify({ ...objects, objects: objects.objects.map((o) => o.#rawProperties()) }), blob = new Blob([json]); return { metadataSize: blob.size, value: blob.stream(), size: blob.size }; } }, InternalR2ObjectBody = class extends InternalR2Object { constructor(metadata, body, range) { super(metadata, range); this.body = body; } encode() { let { metadataSize, value: metadata } = super.encode(), size = this.range?.length ?? this.size, identity2 = new FixedLengthStream(size + metadataSize); return metadata.pipeTo(identity2.writable, { preventClose: !0 }).then(() => this.body.pipeTo(identity2.writable)), { metadataSize, value: identity2.readable, size }; } }; // src/workers/r2/schemas.worker.ts import { Base64DataSchema, HexDataSchema, z } from "miniflare:zod"; var MultipartUploadState = { IN_PROGRESS: 0, COMPLETED: 1, ABORTED: 2 }, SQL_SCHEMA = ` CREATE TABLE IF NOT EXISTS _mf_objects ( key TEXT PRIMARY KEY, blob_id TEXT, version TEXT NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, uploaded INTEGER NOT NULL, checksums TEXT NOT NULL, http_metadata TEXT NOT NULL, custom_metadata TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS _mf_multipart_uploads ( upload_id TEXT PRIMARY KEY, key TEXT NOT NULL, http_metadata TEXT NOT NULL, custom_metadata TEXT NOT NULL, state TINYINT DEFAULT 0 NOT NULL ); CREATE TABLE IF NOT EXISTS _mf_multipart_parts ( upload_id TEXT NOT NULL REFERENCES _mf_multipart_uploads(upload_id), part_number INTEGER NOT NULL, blob_id TEXT NOT NULL, size INTEGER NOT NULL, etag TEXT NOT NULL, checksum_md5 TEXT NOT NULL, object_key TEXT REFERENCES _mf_objects(key) DEFERRABLE INITIALLY DEFERRED, PRIMARY KEY (upload_id, part_number) ); `, DateSchema = z.coerce.number().transform((value) => new Date(value)), RecordSchema = z.object({ k: z.string(), v: z.string() }).array().transform( (entries) => Object.fromEntries(entries.map(({ k, v }) => [k, v])) ), R2RangeSchema = z.object({ offset: z.coerce.number().optional(), length: z.coerce.number().optional(), suffix: z.coerce.number().optional() }), R2EtagSchema = z.discriminatedUnion("type", [ z.object({ type: z.literal("strong"), value: z.string() }), z.object({ type: z.literal("weak"), value: z.string() }), z.object({ type: z.literal("wildcard") }) ]), R2EtagMatchSchema = R2EtagSchema.array().min(1).optional(), R2ConditionalSchema = z.object({ // Performs the operation if the object's ETag matches the given string etagMatches: R2EtagMatchSchema, // "If-Match" // Performs the operation if the object's ETag does NOT match the given string etagDoesNotMatch: R2EtagMatchSchema, // "If-None-Match" // Performs the operation if the object was uploaded BEFORE the given date uploadedBefore: DateSchema.optional(), // "If-Unmodified-Since" // Performs the operation if the object was uploaded AFTER the given date uploadedAfter: DateSchema.optional(), // "If-Modified-Since" // Truncates dates to seconds before performing comparisons secondsGranularity: z.oboolean() }), R2ChecksumsSchema = z.object({ 0: HexDataSchema.optional(), 1: HexDataSchema.optional(), 2: HexDataSchema.optional(), 3: HexDataSchema.optional(), 4: HexDataSchema.optional() }).transform((checksums) => ({ md5: checksums[0], sha1: checksums[1], sha256: checksums[2], sha384: checksums[3], sha512: checksums[4] })), R2PublishedPartSchema = z.object({ etag: z.string(), part: z.number() }), R2HttpFieldsSchema = z.object({ contentType: z.ostring(), contentLanguage: z.ostring(), contentDisposition: z.ostring(), contentEncoding: z.ostring(), cacheControl: z.ostring(), cacheExpiry: z.coerce.number().optional() }), R2HeadRequestSchema = z.object({ method: z.literal("head"), object: z.string() }), R2GetRequestSchema = z.object({ method: z.literal("get"), object: z.string(), // Specifies that only a specific length (from an optional offset) or suffix // of bytes from the object should be returned. Refer to // https://developers.cloudflare.com/r2/runtime-apis/#ranged-reads. range: R2RangeSchema.optional(), rangeHeader: z.ostring(), // Specifies that the object should only be returned given satisfaction of // certain conditions in the R2Conditional. Refer to R2Conditional above. onlyIf: R2ConditionalSchema.optional() }), R2PutRequestSchema = z.object({ method: z.literal("put"), object: z.string(), customFields: RecordSchema.optional(), // (renamed in transform) httpFields: R2HttpFieldsSchema.optional(), // (renamed in transform) onlyIf: R2ConditionalSchema.optional(), md5: Base64DataSchema.optional(), // (intentionally base64, not hex) sha1: HexDataSchema.optional(), sha256: HexDataSchema.optional(), sha384: HexDataSchema.optional(), sha512: HexDataSchema.optional() }).transform((value) => ({ method: value.method, object: value.object, customMetadata: value.customFields, httpMetadata: value.httpFields, onlyIf: value.onlyIf, md5: value.md5, sha1: value.sha1, sha256: value.sha256, sha384: value.sha384, sha512: value.sha512 })), R2CreateMultipartUploadRequestSchema = z.object({ method: z.literal("createMultipartUpload"), object: z.string(), customFields: RecordSchema.optional(), // (renamed in transform) httpFields: R2HttpFieldsSchema.optional() // (renamed in transform) }).transform((value) => ({ method: value.method, object: value.object, customMetadata: value.customFields, httpMetadata: value.httpFields })), R2UploadPartRequestSchema = z.object({ method: z.literal("uploadPart"), object: z.string(), uploadId: z.string(), partNumber: z.number() }), R2CompleteMultipartUploadRequestSchema = z.object({ method: z.literal("completeMultipartUpload"), object: z.string(), uploadId: z.string(), parts: R2PublishedPartSchema.array() }), R2AbortMultipartUploadRequestSchema = z.object({ method: z.literal("abortMultipartUpload"), object: z.string(), uploadId: z.string() }), R2ListRequestSchema = z.object({ method: z.literal("list"), limit: z.onumber(), prefix: z.ostring(), cursor: z.ostring(), delimiter: z.ostring(), startAfter: z.ostring(), include: z.union([z.literal(0), z.literal(1)]).transform((value) => value === 0 ? "httpMetadata" : "customMetadata").array().optional() }), R2DeleteRequestSchema = z.intersection( z.object({ method: z.literal("delete") }), z.union([ z.object({ object: z.string() }), z.object({ objects: z.string().array() }) ]) ), R2BindingRequestSchema = z.union([ R2HeadRequestSchema, R2GetRequestSchema, R2PutRequestSchema, R2CreateMultipartUploadRequestSchema, R2UploadPartRequestSchema, R2CompleteMultipartUploadRequestSchema, R2AbortMultipartUploadRequestSchema, R2ListRequestSchema, R2DeleteRequestSchema ]); // src/workers/r2/validator.worker.ts import assert from "node:assert"; import { Buffer as Buffer2 } from "node:buffer"; import { parseRanges } from "miniflare:shared"; function identity(ms) { return ms; } function truncateToSeconds(ms) { return Math.floor(ms / 1e3) * 1e3; } function includesEtag(conditions, etag, comparison) { for (let condition of conditions) if (condition.type === "wildcard" || condition.value === etag && (condition.type === "strong" || comparison === "weak")) return !0; return !1; } function _testR2Conditional(cond, metadata) { if (metadata === void 0) { let ifMatch2 = cond.etagMatches === void 0, ifModifiedSince2 = cond.uploadedAfter === void 0; return ifMatch2 && ifModifiedSince2; } let { etag, uploaded: lastModifiedRaw } = metadata, ifMatch = cond.etagMatches === void 0 || includesEtag(cond.etagMatches, etag, "strong"), ifNoneMatch = cond.etagDoesNotMatch === void 0 || !includesEtag(cond.etagDoesNotMatch, etag, "weak"), maybeTruncate = cond.secondsGranularity ? truncateToSeconds : identity, lastModified = maybeTruncate(lastModifiedRaw), ifModifiedSince = cond.uploadedAfter === void 0 || maybeTruncate(cond.uploadedAfter.getTime()) < lastModified || cond.etagDoesNotMatch !== void 0 && ifNoneMatch, ifUnmodifiedSince = cond.uploadedBefore === void 0 || lastModified < maybeTruncate(cond.uploadedBefore.getTime()) || cond.etagMatches !== void 0 && ifMatch; return ifMatch && ifNoneMatch && ifModifiedSince && ifUnmodifiedSince; } var R2_HASH_ALGORITHMS = [ { name: "MD5", field: "md5" }, { name: "SHA-1", field: "sha1" }, { name: "SHA-256", field: "sha256" }, { name: "SHA-384", field: "sha384" }, { name: "SHA-512", field: "sha512" } ]; function serialisedLength(x) { for (let i = 0; i < x.length; i++) if (x.charCodeAt(i) >= 256) return x.length * 2; return x.length; } var Validator = class { hash(digests, hashes) { let checksums = {}; for (let { name, field } of R2_HASH_ALGORITHMS) { let providedHash = hashes[field]; if (providedHash !== void 0) { let computedHash = digests.get(name); if (assert(computedHash !== void 0), !providedHash.equals(computedHash)) throw new BadDigest(name, providedHash, computedHash); checksums[field] = computedHash.toString("hex"); } } return checksums; } condition(meta, onlyIf) { if (onlyIf !== void 0 && !_testR2Conditional(onlyIf, meta)) throw new PreconditionFailed(); return this; } range(options, size) { if (options.rangeHeader !== void 0) { let ranges = parseRanges(options.rangeHeader, size); if (ranges?.length === 1) return ranges[0]; } else if (options.range !== void 0) { let { offset, length, suffix } = options.range; if (suffix !== void 0) { if (suffix <= 0) throw new InvalidRange(); suffix > size && (suffix = size), offset = size - suffix, length = suffix; } if (offset === void 0 && (offset = 0), length === void 0 && (length = size - offset), offset < 0 || offset > size || length <= 0) throw new InvalidRange(); return offset + length > size && (length = size - offset), { start: offset, end: offset + length - 1 }; } } size(size) { if (size > R2Limits.MAX_VALUE_SIZE) throw new EntityTooLarge(); return this; } metadataSize(customMetadata) { if (customMetadata === void 0) return this; let metadataLength = 0; for (let [key, value] of Object.entries(customMetadata)) metadataLength += serialisedLength(key) + serialisedLength(value); if (metadataLength > R2Limits.MAX_METADATA_SIZE) throw new MetadataTooLarge(); return this; } key(key) { if (Buffer2.byteLength(key) > R2Limits.MAX_KEY_SIZE) throw new InvalidObjectName(); return this; } limit(limit) { if (limit !== void 0 && (limit < 1 || limit > R2Limits.MAX_LIST_KEYS)) throw new InvalidMaxKeys(); return this; } }; // src/workers/r2/bucket.worker.ts var DigestingStream = class extends TransformStream { digests; constructor(algorithms) { let digests = new DeferredPromise(), hashes = algorithms.map((alg) => { let stream = new crypto.DigestStream(alg), writer = stream.getWriter(); return { stream, writer }; }); super({ async transform(chunk, controller) { for (let hash of hashes) await hash.writer.write(chunk); controller.enqueue(chunk); }, async flush() { let result = /* @__PURE__ */ new Map(); for (let i = 0; i < hashes.length; i++) await hashes[i].writer.close(), result.set(algorithms[i], Buffer3.from(await hashes[i].stream.digest)); digests.resolve(result); } }), this.digests = digests; } }, validate = new Validator(), decoder = new TextDecoder(); function generateVersion() { return Buffer3.from(crypto.getRandomValues(new Uint8Array(16))).toString( "hex" ); } function generateId() { return Buffer3.from(crypto.getRandomValues(new Uint8Array(128))).toString( "base64url" ); } function generateMultipartEtag(md5Hexes) { let hash = createHash("md5"); for (let md5Hex of md5Hexes) hash.update(md5Hex, "hex"); return `${hash.digest("hex")}-${md5Hexes.length}`; } function rangeOverlaps(a, b) { return a.start <= b.end && b.start <= a.end; } async function decodeMetadata(req) { let metadataSize = parseInt(req.headers.get(R2Headers.METADATA_SIZE)); if (Number.isNaN(metadataSize)) throw new InvalidMetadata(); assert2(req.body !== null); let body = req.body, [metadataBuffer, value] = await readPrefix(body, metadataSize), metadataJson = decoder.decode(metadataBuffer); return { metadata: R2BindingRequestSchema.parse(JSON.parse(metadataJson)), metadataSize, value }; } function decodeHeaderMetadata(req) { let header = req.headers.get(R2Headers.REQUEST); if (header === null) throw new InvalidMetadata(); return R2BindingRequestSchema.parse(JSON.parse(header)); } function encodeResult(result) { let encoded; return result instanceof InternalR2Object ? encoded = result.encode() : encoded = InternalR2Object.encodeMultiple(result), new Response(encoded.value, { headers: { [R2Headers.METADATA_SIZE]: `${encoded.metadataSize}`, "Content-Type": "application/json", "Content-Length": `${encoded.size}` } }); } function encodeJSONResult(result) { let encoded = JSON.stringify(result); return new Response(encoded, { headers: { [R2Headers.METADATA_SIZE]: `${Buffer3.byteLength(encoded)}`, "Content-Type": "application/json" } }); } function sqlStmts(db) { let stmtGetPreviousByKey = db.stmt("SELECT blob_id, etag, uploaded FROM _mf_objects WHERE key = :key"), stmtGetByKey = db.stmt(` SELECT key, blob_id, version, size, etag, uploaded, checksums, http_metadata, custom_metadata FROM _mf_objects WHERE key = :key `), stmtPut = db.stmt(` INSERT OR REPLACE INTO _mf_objects (key, blob_id, version, size, etag, uploaded, checksums, http_metadata, custom_metadata) VALUES (:key, :blob_id, :version, :size, :etag, :uploaded, :checksums, :http_metadata, :custom_metadata) `), stmtDelete = db.stmt("DELETE FROM _mf_objects WHERE key = :key RETURNING blob_id"); function stmtListWithoutDelimiter(...extraColumns) { let columns = [ "key", "version", "size", "etag", "uploaded", "checksums", ...extraColumns ]; return db.stmt(` SELECT ${columns.join(", ")} FROM _mf_objects WHERE substr(key, 1, length(:prefix)) = :prefix AND (:start_after IS NULL OR key > :start_after) ORDER BY key LIMIT :limit `); } let stmtGetUploadState = db.stmt( // For checking current upload state "SELECT state FROM _mf_multipart_uploads WHERE upload_id = :upload_id AND key = :key" ), stmtGetUploadMetadata = db.stmt( // For checking current upload state, and getting metadata for completion "SELECT http_metadata, custom_metadata, state FROM _mf_multipart_uploads WHERE upload_id = :upload_id AND key = :key" ), stmtUpdateUploadState = db.stmt( // For completing/aborting uploads "UPDATE _mf_multipart_uploads SET state = :state WHERE upload_id = :upload_id" ), stmtGetPreviousPartByNumber = db.stmt( // For getting part number's previous blob ID to garbage collect "SELECT blob_id FROM _mf_multipart_parts WHERE upload_id = :upload_id AND part_number = :part_number" ), stmtPutPart = db.stmt( // For recording metadata when uploading parts `INSERT OR REPLACE INTO _mf_multipart_parts (upload_id, part_number, blob_id, size, etag, checksum_md5) VALUES (:upload_id, :part_number, :blob_id, :size, :etag, :checksum_md5)` ), stmtLinkPart = db.stmt( // For linking parts with an object when completing uploads `UPDATE _mf_multipart_parts SET object_key = :object_key WHERE upload_id = :upload_id AND part_number = :part_number` ), stmtDeletePartsByUploadId = db.stmt( // For deleting parts when aborting uploads "DELETE FROM _mf_multipart_parts WHERE upload_id = :upload_id RETURNING blob_id" ), stmtDeleteUnlinkedPartsByUploadId = db.stmt( // For deleting unused parts when completing uploads "DELETE FROM _mf_multipart_parts WHERE upload_id = :upload_id AND object_key IS NULL RETURNING blob_id" ), stmtDeletePartsByKey = db.stmt( // For deleting dangling parts when overwriting an existing key "DELETE FROM _mf_multipart_parts WHERE object_key = :object_key RETURNING blob_id" ), stmtListPartsByUploadId = db.stmt( // For getting part metadata when completing uploads `SELECT upload_id, part_number, blob_id, size, etag, checksum_md5, object_key FROM _mf_multipart_parts WHERE upload_id = :upload_id` ), stmtListPartsByKey = db.stmt( // For getting part metadata when getting values, size included for range // requests, so we only need to read blobs containing the required data "SELECT blob_id, size FROM _mf_multipart_parts WHERE object_key = :object_key ORDER BY part_number" ); return { getByKey: stmtGetByKey, getPartsByKey: db.txn((key) => { let row = get(stmtGetByKey({ key })); if (row !== void 0) if (row.blob_id === null) { let partsRows = all(stmtListPartsByKey({ object_key: key })); return { row, parts: partsRows }; } else return { row }; }), put: db.txn((newRow, onlyIf) => { let key = newRow.key, row = get(stmtGetPreviousByKey({ key })); onlyIf !== void 0 && validate.condition(row, onlyIf), stmtPut(newRow); let maybeOldBlobId = row?.blob_id; return maybeOldBlobId === void 0 ? [] : maybeOldBlobId === null ? all(stmtDeletePartsByKey({ object_key: key })).map(({ blob_id }) => blob_id) : [maybeOldBlobId]; }), deleteByKeys: db.txn((keys) => { let oldBlobIds = []; for (let key of keys) { let maybeOldBlobId = get(stmtDelete({ key }))?.blob_id; if (maybeOldBlobId === null) { let partRows = stmtDeletePartsByKey({ object_key: key }); for (let partRow of partRows) oldBlobIds.push(partRow.blob_id); } else maybeOldBlobId !== void 0 && oldBlobIds.push(maybeOldBlobId); } return oldBlobIds; }), listWithoutDelimiter: stmtListWithoutDelimiter(), listHttpMetadataWithoutDelimiter: stmtListWithoutDelimiter("http_metadata"), listCustomMetadataWithoutDelimiter: stmtListWithoutDelimiter("custom_metadata"), listHttpCustomMetadataWithoutDelimiter: stmtListWithoutDelimiter( "http_metadata", "custom_metadata" ), listMetadata: db.stmt(` SELECT -- When grouping by a delimited prefix, this will give us the last key with that prefix. -- NOTE: we'll use this for the next cursor. If we didn't return the last key, the next page may return the -- same delimited prefix. Essentially, we're skipping over all keys with this group's delimited prefix. -- When grouping by a key, this will just give us the key. max(key) AS last_key, iif( -- Try get 1-indexed position \`i\` of :delimiter in rest of key after :prefix... instr(substr(key, length(:prefix) + 1), :delimiter), -- ...if found, we have a delimited prefix of the :prefix followed by the rest of key up to and including the :delimiter 'dlp:' || substr(key, 1, length(:prefix) + instr(substr(key, length(:prefix) + 1), :delimiter) + length(:delimiter) - 1), -- ...otherwise, we just have a regular key 'key:' || key ) AS delimited_prefix_or_key, -- NOTE: we'll ignore metadata for delimited prefix rows, so it doesn't matter which keys' we return version, size, etag, uploaded, checksums, http_metadata, custom_metadata FROM _mf_objects WHERE substr(key, 1, length(:prefix)) = :prefix AND (:start_after IS NULL OR key > :start_after) GROUP BY delimited_prefix_or_key -- Group keys with same delimited prefix into a row, leaving others in their own rows ORDER BY last_key LIMIT :limit; `), createMultipartUpload: db.stmt(` INSERT INTO _mf_multipart_uploads (upload_id, key, http_metadata, custom_metadata) VALUES (:upload_id, :key, :http_metadata, :custom_metadata) `), putPart: db.txn( (key, newRow) => { if (get( stmtGetUploadState({ key, upload_id: newRow.upload_id }) )?.state !== MultipartUploadState.IN_PROGRESS) throw new NoSuchUpload(); let partRow = get( stmtGetPreviousPartByNumber({ upload_id: newRow.upload_id, part_number: newRow.part_number }) ); return stmtPutPart(newRow), partRow?.blob_id; } ), completeMultipartUpload: db.txn( (key, upload_id, selectedParts, minPartSize) => { let uploadRow = get(stmtGetUploadMetadata({ key, upload_id })); if (uploadRow === void 0) throw new InternalError(); if (uploadRow.state > MultipartUploadState.IN_PROGRESS) throw new NoSuchUpload(); let partNumberSet = /* @__PURE__ */ new Set(); for (let { part } of selectedParts) { if (partNumberSet.has(part)) throw new InternalError(); partNumberSet.add(part); } let uploadedPartRows = stmtListPartsByUploadId({ upload_id }), uploadedParts = /* @__PURE__ */ new Map(); for (let row of uploadedPartRows) uploadedParts.set(row.part_number, row); let parts = selectedParts.map((selectedPart) => { let uploadedPart = uploadedParts.get(selectedPart.part); if (uploadedPart?.etag !== selectedPart.etag) throw new InvalidPart(); return uploadedPart; }); for (let part of parts.slice(0, -1)) if (part.size < minPartSize) throw new EntityTooSmall(); parts.sort((a, b) => a.part_number - b.part_number); let partSize; for (let part of parts.slice(0, -1)) if (partSize ??= part.size, part.size < minPartSize || part.size !== partSize) throw new BadUpload(); if (partSize !== void 0 && parts[parts.length - 1].size > partSize) throw new BadUpload(); let oldBlobIds = [], maybeOldBlobId = get(stmtGetPreviousByKey({ key }))?.blob_id; if (maybeOldBlobId === null) { let partRows2 = stmtDeletePartsByKey({ object_key: key }); for (let partRow of partRows2) oldBlobIds.push(partRow.blob_id); } else maybeOldBlobId !== void 0 && oldBlobIds.push(maybeOldBlobId); let totalSize = parts.reduce((acc, { size }) => acc + size, 0), etag = generateMultipartEtag( parts.map(({ checksum_md5 }) => checksum_md5) ), newRow = { key, blob_id: null, version: generateVersion(), size: totalSize, etag, uploaded: Date.now(), checksums: "{}", http_metadata: uploadRow.http_metadata, custom_metadata: uploadRow.custom_metadata }; stmtPut(newRow); for (let part of parts) stmtLinkPart({ upload_id, part_number: part.part_number, object_key: key }); let partRows = stmtDeleteUnlinkedPartsByUploadId({ upload_id }); for (let partRow of partRows) oldBlobIds.push(partRow.blob_id); return stmtUpdateUploadState({ upload_id, state: MultipartUploadState.COMPLETED }), { newRow, oldBlobIds }; } ), abortMultipartUpload: db.txn((key, upload_id) => { let uploadRow = get(stmtGetUploadState({ key, upload_id })); if (uploadRow === void 0) throw new InternalError(); if (uploadRow.state > MultipartUploadState.IN_PROGRESS) return []; let oldBlobIds = all(stmtDeletePartsByUploadId({ upload_id })).map(({ blob_id }) => blob_id); return stmtUpdateUploadState({ upload_id, state: MultipartUploadState.ABORTED }), oldBlobIds; }) }; } var R2BucketObject = class extends MiniflareDurableObject { #stmts; // Multipart uploads are stored as multiple blobs. Therefore, when reading a // multipart upload, we'll be reading multiple blobs. When an object is // deleted, all its blobs are deleted in the background. // // Normally for single part objects, this is fine, since we'd open a handle to // a single blob, which we'd have until we closed it, at which point the blob // may be deleted. With multipart, we don't want to open handles for all blobs // as we could hit open file descriptor limits. Similarly, we don't want to // read all blobs first, as we'd have to buffer them. // // Instead, we set up in-process locking on blobs needed for multipart reads. // When we start a multipart read, we acquire all the blobs we need, then // release them as we've streamed each part. Multiple multipart reads may be // in-progress at any given time, so we use a wait group. // // This assumes we only ever have a single Miniflare instance operating on a // blob store, which is always true for in-memory stores, and usually true for // on-disk ones. If we really wanted to do this properly, we could store the // bookkeeping for the wait group in SQLite, but then we'd have to implement // some inter-process signalling/subscription system. #inUseBlobs = /* @__PURE__ */ new Map(); constructor(state, env) { super(state, env), this.db.exec("PRAGMA case_sensitive_like = TRUE"), this.db.exec(SQL_SCHEMA), this.#stmts = sqlStmts(this.db); } #acquireBlob(blobId) { let waitGroup = this.#inUseBlobs.get(blobId); waitGroup === void 0 ? (waitGroup = new WaitGroup(), this.#inUseBlobs.set(blobId, waitGroup), waitGroup.add(), waitGroup.wait().then(() => this.#inUseBlobs.delete(blobId))) : waitGroup.add(); } #releaseBlob(blobId) { this.#inUseBlobs.get(blobId)?.done(); } #backgroundDelete(blobId) { this.timers.queueMicrotask(async () => (await this.#inUseBlobs.get(blobId)?.wait(), this.blob.delete(blobId).catch((e) => { console.error("R2BucketObject##backgroundDelete():", e); }))); } #assembleMultipartValue(parts, queryRange) { let requiredParts = [], start = 0; for (let part of parts) { let partRange = { start, end: start + part.size - 1 }; if (rangeOverlaps(partRange, queryRange)) { let range = { start: Math.max(partRange.start, queryRange.start) - partRange.start, end: Math.min(partRange.end, queryRange.end) - partRange.start }; this.#acquireBlob(part.blob_id), requiredParts.push({ blobId: part.blob_id, range }); } start = partRange.end + 1; } let identity2 = new TransformStream(); return (async () => { let i = 0; try { for (; i < requiredParts.length; i++) { let { blobId, range } = requiredParts[i], value = await this.blob.get(blobId, range), msg = `Expected to find blob "${blobId}" for multipart value`; assert2(value !== null, msg), await value.pipeTo(identity2.writable, { preventClose: !0 }), this.#releaseBlob(blobId); } await identity2.writable.close(); } catch (e) { await identity2.writable.abort(e); } finally { for (; i < requiredParts.length; i++) this.#releaseBlob(requiredParts[i].blobId); } })(), identity2.readable; } async #head(key) { validate.key(key); let row = get(this.#stmts.getByKey({ key })); if (row === void 0) throw new NoSuchKey(); let range = { offset: 0, length: row.size }; return new InternalR2Object(row, range); } async #get(key, opts) { validate.key(key); let result = this.#stmts.getPartsByKey(key); if (result === void 0) throw new NoSuchKey(); let { row, parts } = result, defaultR2Range = { offset: 0, length: row.size }; try { validate.condition(row, opts.onlyIf); } catch (e) { throw e instanceof PreconditionFailed && e.attach(new InternalR2Object(row, defaultR2Range)), e; } let range = validate.range(opts, row.size), r2Range; if (range === void 0) r2Range = defaultR2Range; else { let start = range.start, end = Math.min(range.end, row.size); r2Range = { offset: start, length: end - start + 1 }; } let value; if (row.blob_id === null) { assert2(parts !== void 0); let defaultRange = { start: 0, end: row.size - 1 }; value = this.#assembleMultipartValue(parts, range ?? defaultRange); } else if (value = await this.blob.get(row.blob_id, range), value === null) throw new NoSuchKey(); return new InternalR2ObjectBody(row, value, r2Range); } async #put(key, value, valueSize, opts) { let algorithms = []; for (let { name, field } of R2_HASH_ALGORITHMS) (field === "md5" || opts[field] !== void 0) && algorithms.push(name); let digesting = new DigestingStream(algorithms), blobId = await this.blob.put(value.pipeThrough(digesting)), digests = await digesting.digests, md5Digest = digests.get("MD5"); assert2(md5Digest !== void 0); let md5DigestHex = md5Digest.toString("hex"), checksums = validate.key(key).size(valueSize).metadataSize(opts.customMetadata).hash(digests, opts), row = { key, blob_id: blobId, version: generateVersion(), size: valueSize, etag: md5DigestHex, uploaded: Date.now(), checksums: JSON.stringify(checksums), http_metadata: JSON.stringify(opts.httpMetadata ?? {}), custom_metadata: JSON.stringify(opts.customMetadata ?? {}) }, oldBlobIds; try { oldBlobIds = this.#stmts.put(row, opts.onlyIf); } catch (e) { throw this.#backgroundDelete(blobId), e; } if (oldBlobIds !== void 0) for (let blobId2 of oldBlobIds) this.#backgroundDelete(blobId2); return new InternalR2Object(row); } #delete(keys) { Array.isArray(keys) || (keys = [keys]); for (let key of keys) validate.key(key); let oldBlobIds = this.#stmts.deleteByKeys(keys); for (let blobId of oldBlobIds) this.#backgroundDelete(blobId); } #listWithoutDelimiterQuery(excludeHttp, excludeCustom) { return excludeHttp && excludeCustom ? this.#stmts.listWithoutDelimiter : excludeHttp ? this.#stmts.listCustomMetadataWithoutDelimiter : excludeCustom ? this.#stmts.listHttpMetadataWithoutDelimiter : this.#stmts.listHttpCustomMetadataWithoutDelimiter; } async #list(opts) { let prefix = opts.prefix ?? "", limit = opts.limit ?? R2Limits.MAX_LIST_KEYS; validate.limit(limit); let include = opts.include ?? []; include.length > 0 && (limit = Math.min(limit, 100)); let excludeHttp = !include.includes("httpMetadata"), excludeCustom = !include.includes("customMetadata"), rowObject = (row) => ((row.http_metadata === void 0 || excludeHttp) && (row.http_metadata = "{}"), (row.custom_metadata === void 0 || excludeCustom) && (row.custom_metadata = "{}"), new InternalR2Object(row)), startAfter = opts.startAfter; if (opts.cursor !== void 0) { let cursorStartAfter = base64Decode(opts.cursor); (startAfter === void 0 || cursorStartAfter > startAfter) && (startAfter = cursorStartAfter); } let delimiter = opts.delimiter; delimiter === "" && (delimiter = void 0); let params = { prefix, start_after: startAfter ?? null, // Increase the queried limit by 1, if we return this many results, we // know there are more rows. We'll truncate to the original limit before // returning results. limit: limit + 1 }, objects, delimitedPrefixes = [], nextCursorStartAfter; if (delimiter !== void 0) { let rows = all(this.#stmts.listMetadata({ ...params, delimiter })), hasMoreRows = rows.length === limit + 1; rows.splice(limit, 1), objects = []; for (let row of rows) row.delimited_prefix_or_key.startsWith("dlp:") ? delimitedPrefixes.push(row.delimited_prefix_or_key.substring(4)) : objects.push(rowObject({ ...row, key: row.last_key })); hasMoreRows && (nextCursorStartAfter = rows[limit - 1].last_key); } else { let query = this.#listWithoutDelimiterQuery(excludeHttp, excludeCustom), rows = all(query(params)), hasMoreRows = rows.length === limit + 1; rows.splice(limit, 1), objects = rows.map(rowObject), hasMoreRows && (nextCursorStartAfter = rows[limit - 1].key); } let nextCursor = maybeApply(base64Encode, nextCursorStartAfter); return { objects, truncated: nextCursor !== void 0, cursor: nextCursor, delimitedPrefixes }; } async #createMultipartUpload(key, opts) { validate.key(key); let uploadId = generateId(); return this.#stmts.createMultipartUpload({ key, upload_id: uploadId, http_metadata: JSON.stringify(opts.httpMetadata ?? {}), custom_metadata: JSON.stringify(opts.customMetadata ?? {}) }), { uploadId }; } async #uploadPart(key, uploadId, partNumber, value, valueSize) { validate.key(key); let digesting = new DigestingStream(["MD5"]), blobId = await this.blob.put(value.pipeThrough(digesting)), md5Digest = (await digesting.digests).get("MD5"); assert2(md5Digest !== void 0); let etag = generateId(), maybeOldBlobId; try { maybeOldBlobId = this.#stmts.putPart(key, { upload_id: uploadId, part_number: partNumber, blob_id: blobId, size: valueSize, etag, checksum_md5: md5Digest.toString("hex") }); } catch (e) { throw this.#backgroundDelete(blobId), e; } return maybeOldBlobId !== void 0 && this.#backgroundDelete(maybeOldBlobId), { etag }; } async #completeMultipartUpload(key, uploadId, parts) { validate.key(key); let minPartSize = this.beingTested ? R2Limits.MIN_MULTIPART_PART_SIZE_TEST : R2Limits.MIN_MULTIPART_PART_SIZE, { newRow, oldBlobIds } = this.#stmts.completeMultipartUpload( key, uploadId, parts, minPartSize ); for (let blobId of oldBlobIds) this.#backgroundDelete(blobId); return new InternalR2Object(newRow); } async #abortMultipartUpload(key, uploadId) { validate.key(key); let oldBlobIds = this.#stmts.abortMultipartUpload(key, uploadId); for (let blobId of oldBlobIds) this.#backgroundDelete(blobId); } get = async (req) => { let metadata = decodeHeaderMetadata(req), result; if (metadata.method === "head") result = await this.#head(metadata.object); else if (metadata.method === "get") result = await this.#get(metadata.object, metadata); else if (metadata.method === "list") result = await this.#list(metadata); else throw new InternalError(); return encodeResult(result); }; put = async (req) => { let { metadata, metadataSize, value } = await decodeMetadata(req); if (metadata.method === "delete") return await this.#delete( "object" in metadata ? metadata.object : metadata.objects ), new Response(); if (metadata.method === "put") { let contentLength = parseInt(req.headers.get("Content-Length")); assert2(!isNaN(contentLength)); let valueSize = contentLength - metadataSize, result = await this.#put( metadata.object, value, valueSize, metadata ); return encodeResult(result); } else if (metadata.method === "createMultipartUpload") { let result = await this.#createMultipartUpload( metadata.object, metadata ); return encodeJSONResult(result); } else if (metadata.method === "uploadPart") { let contentLength = parseInt(req.headers.get("Content-Length")); assert2(!isNaN(contentLength)); let valueSize = contentLength - metadataSize, result = await this.#uploadPart( metadata.object, metadata.uploadId, metadata.partNumber, value, valueSize ); return encodeJSONResult(result); } else if (metadata.method === "completeMultipartUpload") { let result = await this.#completeMultipartUpload( metadata.object, metadata.uploadId, metadata.parts ); return encodeResult(result); } else { if (metadata.method === "abortMultipartUpload") return await this.#abortMultipartUpload(metadata.object, metadata.uploadId), new Response(); throw new InternalError(); } }; }; __decorateClass([ GET("/") ], R2BucketObject.prototype, "get", 2), __decorateClass([ PUT("/") ], R2BucketObject.prototype, "put", 2); export { R2BucketObject }; //# sourceMappingURL=bucket.worker.js.map